From afea93b2c07fff48b9a423b03568a1b1fd840cdb Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 21 Mar 2022 19:28:49 -0700 Subject: [PATCH 01/14] Revert "[build] remove Maven POM files (#3009)" This reverts commit e089b51ab5e1cf5f061c81463d27f33a21198271. --- .github/workflows/pr-validation.yml | 2 +- bin/bookkeeper_gradle | 182 +++ bin/common.sh | 41 +- bin/common_gradle.sh | 346 +++++ bookkeeper-benchmark/bin/benchmark | 14 +- bookkeeper-benchmark/build.gradle | 1 - bookkeeper-benchmark/pom.xml | 85 ++ bookkeeper-common-allocator/pom.xml | 60 + bookkeeper-common/pom.xml | 129 ++ bookkeeper-dist/all/pom.xml | 159 +++ bookkeeper-dist/bkctl/pom.xml | 96 ++ bookkeeper-dist/build.gradle | 1 + bookkeeper-dist/pom.xml | 70 + bookkeeper-dist/server/pom.xml | 146 +++ bookkeeper-dist/src/assemble/src.xml | 1 + bookkeeper-http/http-server/pom.xml | 48 + bookkeeper-http/pom.xml | 34 + bookkeeper-http/servlet-http-server/pom.xml | 57 + bookkeeper-http/vertx-http-server/pom.xml | 49 + bookkeeper-proto/pom.xml | 66 + bookkeeper-server/build.gradle | 8 +- bookkeeper-server/pom.xml | 336 +++++ .../codahale-metrics-provider/pom.xml | 53 + bookkeeper-stats-providers/pom.xml | 33 + .../prometheus-metrics-provider/pom.xml | 72 + bookkeeper-stats/pom.xml | 63 + build.gradle | 8 - buildtools/pom.xml | 28 + circe-checksum/pom.xml | 236 ++++ cpu-affinity/pom.xml | 226 ++++ dev/check-all-licenses | 6 +- dev/check-all-licenses-gradle | 33 + dev/common.sh | 6 +- dev/docker/ci.sh | 64 + dev/update-snapshot-version.sh | 38 + metadata-drivers/etcd/pom.xml | 147 +++ metadata-drivers/pom.xml | 32 + microbenchmarks/pom.xml | 129 ++ pom.xml | 1164 +++++++++++++++++ settings.gradle | 2 - shaded/bookkeeper-server-shaded/pom.xml | 128 ++ shaded/bookkeeper-server-tests-shaded/pom.xml | 147 +++ shaded/distributedlog-core-shaded/pom.xml | 251 ++++ shaded/pom.xml | 34 + stats/pom.xml | 38 + stats/utils/pom.xml | 54 + stream/api/pom.xml | 49 + stream/bin/streamstorage | 11 +- stream/bin/streamstorage-cli | 10 +- stream/bk-grpc-name-resolver/pom.xml | 80 ++ stream/clients/java/all/pom.xml | 109 ++ stream/clients/java/base/pom.xml | 56 + stream/clients/java/kv/pom.xml | 44 + stream/clients/java/pom.xml | 34 + stream/clients/pom.xml | 32 + stream/common/pom.xml | 77 ++ stream/distributedlog/common/pom.xml | 109 ++ stream/distributedlog/core/build.gradle | 1 - stream/distributedlog/core/pom.xml | 135 ++ stream/distributedlog/io/dlfs/pom.xml | 91 ++ stream/distributedlog/io/pom.xml | 31 + stream/distributedlog/pom.xml | 91 ++ stream/distributedlog/protocol/pom.xml | 58 + stream/pom.xml | 92 ++ stream/proto/pom.xml | 115 ++ stream/server/build.gradle | 6 +- stream/server/pom.xml | 79 ++ stream/statelib/pom.xml | 169 +++ stream/storage/api/pom.xml | 58 + stream/storage/impl/pom.xml | 160 +++ stream/storage/pom.xml | 33 + stream/tests-common/pom.xml | 118 ++ tests/backward-compat/bc-non-fips/pom.xml | 79 ++ .../current-server-old-clients/pom.xml | 32 + .../hierarchical-ledger-manager/pom.xml | 32 + .../backward-compat/hostname-bookieid/pom.xml | 32 + .../old-cookie-new-cluster/pom.xml | 32 + tests/backward-compat/pom.xml | 41 + .../recovery-no-password/pom.xml | 39 + tests/backward-compat/upgrade-direct/pom.xml | 32 + tests/backward-compat/upgrade/pom.xml | 32 + .../yahoo-custom-version/pom.xml | 32 + .../all-released-versions-image/pom.xml | 71 + .../docker-images/all-versions-image/pom.xml | 111 ++ .../current-version-image/pom.xml | 147 +++ tests/docker-images/pom.xml | 34 + tests/integration-tests-base-groovy/pom.xml | 118 ++ tests/integration-tests-base/pom.xml | 98 ++ tests/integration-tests-topologies/pom.xml | 49 + tests/integration-tests-utils/pom.xml | 93 ++ .../utils/BookKeeperClusterUtils.java | 15 +- tests/integration/cluster/pom.xml | 79 ++ tests/integration/pom.xml | 73 ++ tests/integration/smoke/pom.xml | 75 ++ tests/integration/standalone/pom.xml | 64 + tests/pom.xml | 57 + tests/scripts/pom.xml | 75 ++ .../test/bash/gradle/bk_test_bin_common.sh | 16 +- .../bookkeeper-server-shaded-test/pom.xml | 62 + .../pom.xml | 78 ++ .../distributedlog-core-shaded-test/pom.xml | 82 ++ tests/shaded/pom.xml | 35 + tools/all/build.gradle | 1 - tools/all/pom.xml | 89 ++ tools/framework/pom.xml | 44 + tools/ledger/pom.xml | 67 + tools/perf/build.gradle | 1 - tools/perf/pom.xml | 52 + tools/pom.xml | 35 + tools/stream/pom.xml | 52 + 110 files changed, 8872 insertions(+), 55 deletions(-) create mode 100755 bin/bookkeeper_gradle create mode 100755 bin/common_gradle.sh create mode 100644 bookkeeper-benchmark/pom.xml create mode 100644 bookkeeper-common-allocator/pom.xml create mode 100644 bookkeeper-common/pom.xml create mode 100644 bookkeeper-dist/all/pom.xml create mode 100644 bookkeeper-dist/bkctl/pom.xml create mode 100644 bookkeeper-dist/pom.xml create mode 100644 bookkeeper-dist/server/pom.xml create mode 100644 bookkeeper-http/http-server/pom.xml create mode 100644 bookkeeper-http/pom.xml create mode 100644 bookkeeper-http/servlet-http-server/pom.xml create mode 100644 bookkeeper-http/vertx-http-server/pom.xml create mode 100644 bookkeeper-proto/pom.xml create mode 100644 bookkeeper-server/pom.xml create mode 100644 bookkeeper-stats-providers/codahale-metrics-provider/pom.xml create mode 100644 bookkeeper-stats-providers/pom.xml create mode 100644 bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml create mode 100644 bookkeeper-stats/pom.xml create mode 100644 buildtools/pom.xml create mode 100644 circe-checksum/pom.xml create mode 100644 cpu-affinity/pom.xml create mode 100755 dev/check-all-licenses-gradle create mode 100755 dev/docker/ci.sh create mode 100755 dev/update-snapshot-version.sh create mode 100644 metadata-drivers/etcd/pom.xml create mode 100644 metadata-drivers/pom.xml create mode 100644 microbenchmarks/pom.xml create mode 100644 pom.xml create mode 100644 shaded/bookkeeper-server-shaded/pom.xml create mode 100644 shaded/bookkeeper-server-tests-shaded/pom.xml create mode 100644 shaded/distributedlog-core-shaded/pom.xml create mode 100644 shaded/pom.xml create mode 100644 stats/pom.xml create mode 100644 stats/utils/pom.xml create mode 100644 stream/api/pom.xml create mode 100644 stream/bk-grpc-name-resolver/pom.xml create mode 100644 stream/clients/java/all/pom.xml create mode 100644 stream/clients/java/base/pom.xml create mode 100644 stream/clients/java/kv/pom.xml create mode 100644 stream/clients/java/pom.xml create mode 100644 stream/clients/pom.xml create mode 100644 stream/common/pom.xml create mode 100644 stream/distributedlog/common/pom.xml create mode 100644 stream/distributedlog/core/pom.xml create mode 100644 stream/distributedlog/io/dlfs/pom.xml create mode 100644 stream/distributedlog/io/pom.xml create mode 100644 stream/distributedlog/pom.xml create mode 100644 stream/distributedlog/protocol/pom.xml create mode 100644 stream/pom.xml create mode 100644 stream/proto/pom.xml create mode 100644 stream/server/pom.xml create mode 100644 stream/statelib/pom.xml create mode 100644 stream/storage/api/pom.xml create mode 100644 stream/storage/impl/pom.xml create mode 100644 stream/storage/pom.xml create mode 100644 stream/tests-common/pom.xml create mode 100644 tests/backward-compat/bc-non-fips/pom.xml create mode 100644 tests/backward-compat/current-server-old-clients/pom.xml create mode 100644 tests/backward-compat/hierarchical-ledger-manager/pom.xml create mode 100644 tests/backward-compat/hostname-bookieid/pom.xml create mode 100644 tests/backward-compat/old-cookie-new-cluster/pom.xml create mode 100644 tests/backward-compat/pom.xml create mode 100644 tests/backward-compat/recovery-no-password/pom.xml create mode 100644 tests/backward-compat/upgrade-direct/pom.xml create mode 100644 tests/backward-compat/upgrade/pom.xml create mode 100644 tests/backward-compat/yahoo-custom-version/pom.xml create mode 100644 tests/docker-images/all-released-versions-image/pom.xml create mode 100644 tests/docker-images/all-versions-image/pom.xml create mode 100644 tests/docker-images/current-version-image/pom.xml create mode 100644 tests/docker-images/pom.xml create mode 100644 tests/integration-tests-base-groovy/pom.xml create mode 100644 tests/integration-tests-base/pom.xml create mode 100644 tests/integration-tests-topologies/pom.xml create mode 100644 tests/integration-tests-utils/pom.xml create mode 100644 tests/integration/cluster/pom.xml create mode 100644 tests/integration/pom.xml create mode 100644 tests/integration/smoke/pom.xml create mode 100644 tests/integration/standalone/pom.xml create mode 100644 tests/pom.xml create mode 100644 tests/scripts/pom.xml create mode 100644 tests/shaded/bookkeeper-server-shaded-test/pom.xml create mode 100644 tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml create mode 100644 tests/shaded/distributedlog-core-shaded-test/pom.xml create mode 100644 tests/shaded/pom.xml create mode 100644 tools/all/pom.xml create mode 100644 tools/framework/pom.xml create mode 100644 tools/ledger/pom.xml create mode 100644 tools/perf/pom.xml create mode 100644 tools/pom.xml create mode 100644 tools/stream/pom.xml diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 00b8a1ae8df..f00c33587d0 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -55,7 +55,7 @@ jobs: run: ./gradlew build -x signDistTar -x test ${GRADLE_ARGS} - name: Check license files - run: dev/check-all-licenses + run: dev/check-all-licenses-gradle - name: Generate Javadoc run: ./gradlew generateApiJavadoc diff --git a/bin/bookkeeper_gradle b/bin/bookkeeper_gradle new file mode 100755 index 00000000000..22545b29cbc --- /dev/null +++ b/bin/bookkeeper_gradle @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +set -e + +BINDIR=`dirname "$0"` +BK_HOME=`cd ${BINDIR}/..;pwd` + +source ${BK_HOME}/bin/common_gradle.sh + +# default variables +DEFAULT_CONF=${BK_HOME}/conf/bk_server.conf +DEFAULT_ZK_CONF=${BK_HOME}/conf/zookeeper.conf + +if [ -z "$BOOKIE_CONF" ]; then + BOOKIE_CONF_TO_CHECK=${DEFAULT_CONF} +else + BOOKIE_CONF_TO_CHECK=${BOOKIE_CONF} +fi + +FIND_TABLE_SERVICE_RESULT=$(find_table_service ${BOOKIE_CONF_TO_CHECK} $1) + +if [ "x${FIND_TABLE_SERVICE_RESULT}" == "xtrue" ]; then + BOOKIE_MODULE_PATH=stream/server + BOOKIE_MODULE_NAME=${TABLE_SERVICE_MODULE_NAME} +elif [ "x${FIND_TABLE_SERVICE_RESULT}" == "xfalse" ]; then + BOOKIE_MODULE_PATH=bookkeeper-server + BOOKIE_MODULE_NAME=${BOOKIE_SERVER_MODULE_NAME} +else + echo ${FIND_TABLE_SERVICE_RESULT} + exit 1 +fi + +# find the module jar +BOOKIE_JAR=$(find_module_jar ${BOOKIE_MODULE_PATH} ${BOOKIE_MODULE_NAME}) + +# set up the classpath +BOOKIE_CLASSPATH=$(set_module_classpath ${BOOKIE_MODULE_PATH}) + +bookkeeper_help() { + cat < +where command is one of: + +[service commands] + + bookie Run a bookie server + autorecovery Run AutoRecovery service daemon + zookeeper Run zookeeper server + +[development commands] + + localbookie Run a test ensemble of bookies locally + standalone Run a standalone cluster (with all service components) locally + +[tooling commands] + + upgrade Upgrade bookie filesystem + shell Run shell for admin commands + +[other commands] + + help This help message + +or command is the full name of a class with a defined main() method. + +Environment variables: + BOOKIE_LOG_CONF Log4j configuration file (default ${DEFAULT_LOG_CONF}) + BOOKIE_CONF Configuration file (default: ${DEFAULT_CONF}) + BOOKIE_ZK_CONF Configuration file for zookeeper (default: $DEFAULT_ZK_CONF) + BOOKIE_EXTRA_OPTS Extra options to be passed to the jvm + BOOKIE_EXTRA_CLASSPATH Add extra paths to the bookkeeper classpath + ENTRY_FORMATTER_CLASS Entry formatter class to format entries. + BOOKIE_PID_DIR Folder where the Bookie server PID file should be stored + BOOKIE_STOP_TIMEOUT Wait time before forcefully kill the Bookie server instance, if the stop is not successful + +These variable can also be set in conf/bkenv.sh +EOF +} + +# if no args specified, show usage +if [ $# = 0 ]; then + bookkeeper_help; + exit 1; +fi + +# get arguments +COMMAND=$1 +shift + +LOCALBOOKIES_CONFIG_DIR="${LOCALBOOKIES_CONFIG_DIR:-/tmp/localbookies-config}" +if [ ${COMMAND} == "shell" ]; then + DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j.shell.properties + if [[ $1 == "-localbookie" ]]; then + if [[ $2 == *:* ]]; + then + BOOKIE_CONF=${LOCALBOOKIES_CONFIG_DIR}/$2.conf + shift 2 + else + BOOKIE_CONF=${LOCALBOOKIES_CONFIG_DIR}/baseconf.conf + shift + fi + fi +fi + +if [ -z "$BOOKIE_ZK_CONF" ]; then + BOOKIE_ZK_CONF=$DEFAULT_ZK_CONF +fi + +if [ -z "$BOOKIE_CONF" ]; then + BOOKIE_CONF=${DEFAULT_CONF} +fi + +# Configure logging +if [ -z "$BOOKIE_LOG_CONF" ]; then + BOOKIE_LOG_CONF=${DEFAULT_LOG_CONF} +fi +BOOKIE_LOG_DIR=${BOOKIE_LOG_DIR:-"$BK_HOME/logs"} +BOOKIE_LOG_FILE=${BOOKIE_LOG_FILE:-"bookkeeper-server.log"} +BOOKIE_ROOT_LOGGER=${BOOKIE_ROOT_LOGGER:-"INFO,CONSOLE"} + +# Configure the classpath +BOOKIE_CLASSPATH="$BOOKIE_JAR:$BOOKIE_CLASSPATH:$BOOKIE_EXTRA_CLASSPATH" +BOOKIE_CLASSPATH="`dirname $BOOKIE_LOG_CONF`:$BOOKIE_CLASSPATH" + +# Build the OPTS +BOOKIE_OPTS=$(build_bookie_opts) +GC_OPTS=$(build_bookie_jvm_opts ${BOOKIE_LOG_DIR} "gc_%p.log") +NETTY_OPTS=$(build_netty_opts) +LOGGING_OPTS=$(build_logging_opts ${BOOKIE_LOG_CONF} ${BOOKIE_LOG_DIR} ${BOOKIE_LOG_FILE} ${BOOKIE_ROOT_LOGGER}) + +BOOKIE_EXTRA_OPTS="${BOOKIE_EXTRA_OPTS} -Dorg.bouncycastle.fips.approved_only=true" +OPTS="${OPTS} -cp ${BOOKIE_CLASSPATH} ${BOOKIE_OPTS} ${GC_OPTS} ${NETTY_OPTS} ${LOGGING_OPTS} ${BOOKIE_EXTRA_OPTS}" + +# Create log dir if it doesn't exist +if [ ! -d ${BOOKIE_LOG_DIR} ]; then + mkdir ${BOOKIE_LOG_DIR} +fi + +#Change to BK_HOME to support relative paths +cd "$BK_HOME" +if [ ${COMMAND} == "bookie" ]; then + exec "${JAVA}" ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.server.Main --conf ${BOOKIE_CONF} $@ +elif [ ${COMMAND} == "autorecovery" ]; then + exec "${JAVA}" ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.replication.AutoRecoveryMain --conf ${BOOKIE_CONF} $@ +elif [ ${COMMAND} == "localbookie" ]; then + NUMBER=$1 + shift + exec "${JAVA}" ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.bookkeeper.util.LocalBookKeeper ${NUMBER} ${BOOKIE_CONF} $@ +elif [ ${COMMAND} == "standalone" ]; then + exec "${JAVA}" ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.bookkeeper.stream.cluster.StandaloneStarter --conf ${BK_HOME}/conf/standalone.conf $@ +elif [ ${COMMAND} == "upgrade" ]; then + exec "${JAVA}" ${OPTS} org.apache.bookkeeper.bookie.FileSystemUpgrade --conf ${BOOKIE_CONF} $@ +elif [ $COMMAND == "zookeeper" ]; then + BOOKIE_LOG_FILE=${BOOKIE_LOG_FILE:-"zookeeper.log"} + exec "${JAVA}" $OPTS -Dbookkeeper.log.file=$BOOKIE_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $BOOKIE_ZK_CONF $@ +elif [ ${COMMAND} == "shell" ]; then + ENTRY_FORMATTER_ARG="-DentryFormatterClass=${ENTRY_FORMATTER_CLASS:-org.apache.bookkeeper.util.StringEntryFormatter}" + exec "${JAVA}" ${OPTS} ${ENTRY_FORMATTER_ARG} org.apache.bookkeeper.bookie.BookieShell -conf ${BOOKIE_CONF} $@ +elif [ ${COMMAND} == "help" ]; then + bookkeeper_help; +else + exec "${JAVA}" ${OPTS} ${COMMAND} $@ +fi + diff --git a/bin/common.sh b/bin/common.sh index 1fbcd885332..a61edcfb064 100755 --- a/bin/common.sh +++ b/bin/common.sh @@ -155,7 +155,7 @@ is_released_binary() { find_module_jar_at() { DIR=$1 MODULE=$2 - REGEX="^${MODULE}[-0-9\\.]*((-[a-zA-Z]*(-[0-9]*)?)|(-SNAPSHOT))?.jar$" + REGEX="^${MODULE}-[0-9\\.]*((-[a-zA-Z]*(-[0-9]*)?)|(-SNAPSHOT))?.jar$" if [ -d ${DIR} ]; then cd ${DIR} for f in *.jar; do @@ -196,13 +196,23 @@ find_module_jar() { fi if [ -z "${MODULE_JAR}" ]; then - BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/build/libs ${MODULE_NAME}) + BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/target ${MODULE_NAME}) if [ -z "${BUILT_JAR}" ]; then - echo "${BK_HOME}/${MODULE_PATH}/build/libs" >&2; echo "Couldn't find module '${MODULE_NAME}' jar." >&2 - ## read -p "Do you want me to run \`mvn package -DskipTests\` for you ? (y|n) " answer - - BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/build/libs ${MODULE_NAME}) + read -p "Do you want me to run \`mvn package -DskipTests\` for you ? (y|n) " answer + case "${answer:0:1}" in + y|Y ) + mkdir -p ${BK_HOME}/logs + output="${BK_HOME}/logs/build.out" + echo "see output at ${output} for the progress ..." >&2 + mvn package -DskipTests &> ${output} + ;; + * ) + exit 1 + ;; + esac + + BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/target ${MODULE_NAME}) fi if [ -n "${BUILT_JAR}" ]; then MODULE_JAR=${BUILT_JAR} @@ -213,19 +223,28 @@ find_module_jar() { echo "Could not find module '${MODULE_JAR}' jar." >&2 exit 1 fi - echo "${MODULE_JAR}" + echo ${MODULE_JAR} return } add_maven_deps_to_classpath() { MODULE_PATH=$1 + MVN="mvn" + if [ "$MAVEN_HOME" != "" ]; then + MVN=${MAVEN_HOME}/bin/mvn + fi + # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. - f="${BK_HOME}/${MODULE_PATH}/build/classpath.txt" + f="${BK_HOME}/${MODULE_PATH}/target/cached_classpath.txt" + output="${BK_HOME}/${MODULE_PATH}/target/build_classpath.out" + if [ ! -f ${f} ]; then - echo "no classpath.txt found at ${BK_HOME}/${MODULE_PATH}/build" - exit 1 + echo "the classpath of module '${MODULE_PATH}' is not found, generating it ..." >&2 + echo "see output at ${output} for the progress ..." >&2 + ${MVN} -f "${BK_HOME}/${MODULE_PATH}/pom.xml" dependency:build-classpath -Dmdep.outputFile="target/cached_classpath.txt" &> ${output} + echo "the classpath of module '${MODULE_PATH}' is generated at '${f}'." >&2 fi } @@ -239,7 +258,7 @@ set_module_classpath() { echo ${BK_CLASSPATH} else add_maven_deps_to_classpath ${MODULE_PATH} >&2 - cat ${BK_HOME}/${MODULE_PATH}/build/classpath.txt + cat ${BK_HOME}/${MODULE_PATH}/target/cached_classpath.txt fi return } diff --git a/bin/common_gradle.sh b/bin/common_gradle.sh new file mode 100755 index 00000000000..ec704499468 --- /dev/null +++ b/bin/common_gradle.sh @@ -0,0 +1,346 @@ +#!/usr/bin/env bash +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Check net.ipv6.bindv6only +if [ -f /sbin/sysctl ] && [ -f /proc/sys/net/ipv6/bindv6only ]; then + # check if net.ipv6.bindv6only is set to 1 + bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) + if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ] + then + echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken" + echo "For more info (the following page also applies to bookkeeper): http://wiki.apache.org/hadoop/HadoopIPv6" + exit 1 + fi +fi + +# See the following page for extensive details on setting +# up the JVM to accept JMX remote management: +# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# by default we allow local JMX connections +if [ "x$JMXLOCALONLY" = "x" ] +then + JMXLOCALONLY=false +fi + +if [ "x$JMXDISABLE" = "x" ] +then + # for some reason these two options are necessary on jdk6 on Ubuntu + # accord to the docs they are not necessary, but otw jconsole cannot + # do a local attach + JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" +else + echo "JMX disabled by user request" >&2 +fi + +# Check for the java to use +if [[ -z ${JAVA_HOME} ]]; then + JAVA=$(which java) + if [ $? = 0 ]; then + echo "JAVA_HOME not set, using java from PATH. ($JAVA)" + else + echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2 + exit 1 + fi +else + JAVA=${JAVA_HOME}/bin/java +fi + +BINDIR=${BK_BINDIR:-"`dirname "$0"`"} +BK_HOME=${BK_HOME:-"`cd ${BINDIR}/..;pwd`"} +BK_CONFDIR=${BK_HOME}/conf +DEFAULT_LOG_CONF=${BK_CONFDIR}/log4j.properties + +source ${BK_CONFDIR}/nettyenv.sh +source ${BK_CONFDIR}/bkenv.sh +source ${BK_CONFDIR}/bk_cli_env.sh + +detect_jdk8() { + + if [ -f "$JAVA_HOME/lib/modules" ]; then + echo "0" + else + echo "1" + fi + return +} + +# default netty settings +NETTY_LEAK_DETECTION_LEVEL=${NETTY_LEAK_DETECTION_LEVEL:-"disabled"} +NETTY_RECYCLER_MAXCAPACITY=${NETTY_RECYCLER_MAXCAPACITY:-"1000"} +NETTY_RECYCLER_LINKCAPACITY=${NETTY_RECYCLER_LINKCAPACITY:-"1024"} + +USING_JDK8=$(detect_jdk8) + +if [ "$USING_JDK8" -ne "1" ]; then + DEFAULT_BOOKIE_GC_OPTS="-XX:+UseG1GC \ + -XX:MaxGCPauseMillis=10 \ + -XX:+ParallelRefProcEnabled \ + -XX:+DisableExplicitGC" + DEFAULT_BOOKIE_GC_LOGGING_OPTS="" +else + DEFAULT_BOOKIE_GC_OPTS="-XX:+UseG1GC \ + -XX:MaxGCPauseMillis=10 \ + -XX:+ParallelRefProcEnabled \ + -XX:+UnlockExperimentalVMOptions \ + -XX:+AggressiveOpts \ + -XX:+DoEscapeAnalysis \ + -XX:ParallelGCThreads=32 \ + -XX:ConcGCThreads=32 \ + -XX:G1NewSizePercent=50 \ + -XX:+DisableExplicitGC \ + -XX:-ResizePLAB" + DEFAULT_BOOKIE_GC_LOGGING_OPTS="-XX:+PrintGCDetails \ + -XX:+PrintGCApplicationStoppedTime \ + -XX:+UseGCLogFileRotation \ + -XX:NumberOfGCLogFiles=5 \ + -XX:GCLogFileSize=64m" +fi + +BOOKIE_MAX_HEAP_MEMORY=${BOOKIE_MAX_HEAP_MEMORY:-"1g"} +BOOKIE_MIN_HEAP_MEMORY=${BOOKIE_MIN_HEAP_MEMORY:-"1g"} +BOOKIE_MAX_DIRECT_MEMORY=${BOOKIE_MAX_DIRECT_MEMORY:-"2g"} +BOOKIE_MEM_OPTS=${BOOKIE_MEM_OPTS:-"-Xms${BOOKIE_MIN_HEAP_MEMORY} -Xmx${BOOKIE_MAX_HEAP_MEMORY} -XX:MaxDirectMemorySize=${BOOKIE_MAX_DIRECT_MEMORY}"} +BOOKIE_GC_OPTS=${BOOKIE_GC_OPTS:-"${DEFAULT_BOOKIE_GC_OPTS}"} +BOOKIE_GC_LOGGING_OPTS=${BOOKIE_GC_LOGGING_OPTS:-"${DEFAULT_BOOKIE_GC_LOGGING_OPTS}"} + +# default CLI JVM settings +DEFAULT_CLI_GC_OPTS="-XX:+UseG1GC \ + -XX:MaxGCPauseMillis=10" +if [ "$USING_JDK8" -ne "1" ]; then + DEFAULT_CLI_GC_LOGGING_OPTS="" +else + DEFAULT_CLI_GC_LOGGING_OPTS="-XX:+PrintGCDetails \ + -XX:+PrintGCApplicationStoppedTime \ + -XX:+UseGCLogFileRotation \ + -XX:NumberOfGCLogFiles=5 \ + -XX:GCLogFileSize=64m" +fi + +CLI_MAX_HEAP_MEMORY=${CLI_MAX_HEAP_MEMORY:-"512M"} +CLI_MIN_HEAP_MEMORY=${CLI_MIN_HEAP_MEMORY:-"256M"} +CLI_MEM_OPTS=${CLI_MEM_OPTS:-"-Xms${CLI_MIN_HEAP_MEMORY} -Xmx${CLI_MAX_HEAP_MEMORY}"} +CLI_GC_OPTS=${CLI_GC_OPTS:-"${DEFAULT_CLI_GC_OPTS}"} +CLI_GC_LOGGING_OPTS=${CLI_GC_LOGGING_OPTS:-"${DEFAULT_CLI_GC_LOGGING_OPTS}"} + +# module names +BOOKIE_SERVER_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-server" +TABLE_SERVICE_MODULE_NAME="(org.apache.bookkeeper-)?stream-storage-server" + +is_released_binary() { + if [ -d ${BK_HOME}/lib ]; then + echo "true" + return + else + echo "false" + return + fi +} + +find_module_jar_at() { + DIR=$1 + MODULE=$2 + REGEX="^${MODULE}[-0-9\\.]*((-[a-zA-Z]*(-[0-9]*)?)|(-SNAPSHOT))?.jar$" + if [ -d ${DIR} ]; then + cd ${DIR} + for f in *.jar; do + if [[ ${f} =~ ${REGEX} ]]; then + echo ${DIR}/${f} + return + fi + done + fi +} + +find_module_release_jar() { + MODULE_NAME=$1 + RELEASE_JAR=$(find_module_jar_at ${BK_HOME} ${MODULE_NAME}) + if [ -n "${RELEASE_JAR}" ]; then + MODULE_JAR=${RELEASE_JAR} + else + RELEASE_JAR=$(find_module_jar_at ${BK_HOME}/lib ${MODULE_NAME}) + if [ -n "${RELEASE_JAR}" ]; then + MODULE_JAR=${RELEASE_JAR} + fi + fi + echo ${RELEASE_JAR} + return +} + +find_module_jar() { + MODULE_PATH=$1 + MODULE_NAME=$2 + RELEASE_JAR=$(find_module_jar_at ${BK_HOME} ${MODULE_NAME}) + if [ -n "${RELEASE_JAR}" ]; then + MODULE_JAR=${RELEASE_JAR} + else + RELEASE_JAR=$(find_module_jar_at ${BK_HOME}/lib ${MODULE_NAME}) + if [ -n "${RELEASE_JAR}" ]; then + MODULE_JAR=${RELEASE_JAR} + fi + fi + + if [ -z "${MODULE_JAR}" ]; then + BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/build/libs ${MODULE_NAME}) + if [ -z "${BUILT_JAR}" ]; then + echo "${BK_HOME}/${MODULE_PATH}/build/libs" >&2; + echo "Couldn't find module '${MODULE_NAME}' jar." >&2 + ## read -p "Do you want me to run \`mvn package -DskipTests\` for you ? (y|n) " answer + + BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/build/libs ${MODULE_NAME}) + fi + if [ -n "${BUILT_JAR}" ]; then + MODULE_JAR=${BUILT_JAR} + fi + fi + + if [ ! -e "${MODULE_JAR}" ]; then + echo "Could not find module '${MODULE_JAR}' jar." >&2 + exit 1 + fi + echo "${MODULE_JAR}" + return +} + +add_maven_deps_to_classpath() { + MODULE_PATH=$1 + # Need to generate classpath from maven pom. This is costly so generate it + # and cache it. Save the file into our target dir so a mvn clean will get + # clean it up and force us create a new one. + f="${BK_HOME}/${MODULE_PATH}/build/classpath.txt" + if [ ! -f ${f} ]; then + echo "the classpath of module '${MODULE_PATH}' is not found, generating it ..." >&2 + ${BK_HOME}/gradlew ${MODULE_PATH}:jar + echo "the classpath of module '${MODULE_PATH}' is generated at '${f}'." >&2 + fi +} + +set_module_classpath() { + MODULE_PATH=$1 + if [ -d "${BK_HOME}/lib" ]; then + BK_CLASSPATH="" + for i in ${BK_HOME}/lib/*.jar; do + BK_CLASSPATH=${BK_CLASSPATH}:${i} + done + echo ${BK_CLASSPATH} + else + add_maven_deps_to_classpath ${MODULE_PATH} >&2 + cat ${BK_HOME}/${MODULE_PATH}/build/classpath.txt + fi + return +} + +build_bookie_jvm_opts() { + LOG_DIR=$1 + GC_LOG_FILENAME=$2 + if [ "$USING_JDK8" -eq "1" ]; then + echo "$BOOKIE_MEM_OPTS $BOOKIE_GC_OPTS $BOOKIE_GC_LOGGING_OPTS $BOOKIE_PERF_OPTS -Xloggc:${LOG_DIR}/${GC_LOG_FILENAME}" + else + echo "$BOOKIE_MEM_OPTS $BOOKIE_GC_OPTS $BOOKIE_GC_LOGGING_OPTS $BOOKIE_PERF_OPTS -Xlog:gc=info:file=${LOG_DIR}/${GC_LOG_FILENAME}::filecount=5,filesize=64m" + fi + return +} + +build_cli_jvm_opts() { + LOG_DIR=$1 + GC_LOG_FILENAME=$2 + if [ "$USING_JDK8" -eq "1" ]; then + echo "$CLI_MEM_OPTS $CLI_GC_OPTS $CLI_GC_LOGGING_OPTS -Xloggc:${LOG_DIR}/${GC_LOG_FILENAME}" + else + echo "$CLI_MEM_OPTS $CLI_GC_OPTS $CLI_GC_LOGGING_OPTS -Xlog:gc=info:file=${LOG_DIR}/${GC_LOG_FILENAME}::filecount=5,filesize=64m" + fi + return +} + +build_netty_opts() { + echo "-Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL} \ + -Dio.netty.recycler.maxCapacity.default=${NETTY_RECYCLER_MAXCAPACITY} \ + -Dio.netty.recycler.linkCapacity=${NETTY_RECYCLER_LINKCAPACITY}" +} + +build_logging_opts() { + CONF_FILE=$1 + LOG_DIR=$2 + LOG_FILE=$3 + LOGGER=$4 + + echo "-Dlog4j.configuration=`basename ${CONF_FILE}` \ + -Dbookkeeper.root.logger=${LOGGER} \ + -Dbookkeeper.log.dir=${LOG_DIR} \ + -Dbookkeeper.log.file=${LOG_FILE}" +} + +build_cli_logging_opts() { + CONF_FILE=$1 + LOG_DIR=$2 + LOG_FILE=$3 + LOGGER=$4 + + echo "-Dlog4j.configuration=`basename ${CONF_FILE}` \ + -Dbookkeeper.cli.root.logger=${LOGGER} \ + -Dbookkeeper.cli.log.dir=${LOG_DIR} \ + -Dbookkeeper.cli.log.file=${LOG_FILE}" +} + +build_bookie_opts() { + echo "-Djava.net.preferIPv4Stack=true" +} + +find_table_service() { + BOOKIE_CONF_TO_CHECK=$1 + SERVICE_COMMAND=$2 + + # check if it is a released binary + IS_RELEASED_BINARY=$(is_released_binary) + + # check if table service is released + TABLE_SERVICE_RELEASED="true" + if [ "x${IS_RELEASED_BINARY}" == "xtrue" ]; then + TABLE_SERVICE_RELEASE_JAR=$(find_module_release_jar ${TABLE_SERVICE_MODULE_NAME}) + if [ "x${TABLE_SERVICE_RELEASE_JAR}" == "x" ]; then + TABLE_SERVICE_RELEASED="false" + fi + fi + + # check the configuration to see if table service is enabled or not. + if [ -z "${ENABLE_TABLE_SERVICE}" ]; then + # mask exit code if the configuration file doesn't contain `StreamStorageLifecycleComponent` + TABLE_SERVICE_SETTING=$(grep StreamStorageLifecycleComponent ${BOOKIE_CONF_TO_CHECK} | cat) + if [[ "${TABLE_SERVICE_SETTING}" =~ ^extraServerComponents.* ]]; then + if [ "x${TABLE_SERVICE_RELEASED}" == "xfalse" ]; then + echo "The release binary is built without table service. Please disable \`StreamStorageLifecycleComponent\` in your bookie configuration at '${BOOKIE_CONF_TO_CHECK}'." + return + fi + ENABLE_TABLE_SERVICE="true" + fi + fi + + # standalone only run + if [ \( "x${SERVICE_COMMAND}" == "xstandalone" \) -a \( "x${TABLE_SERVICE_RELEASED}" == "xfalse" \) ]; then + echo "The release binary is built without table service. Use \`localbookie \` instead of \`standalone\` for local development." + return + fi + + if [ \( "x${SERVICE_COMMAND}" == "xstandalone" \) -o \( "x${ENABLE_TABLE_SERVICE}" == "xtrue" \) ]; then + echo "true" + return + else + echo "false" + return + fi +} diff --git a/bookkeeper-benchmark/bin/benchmark b/bookkeeper-benchmark/bin/benchmark index 72753f1931d..3158aac140f 100755 --- a/bookkeeper-benchmark/bin/benchmark +++ b/bookkeeper-benchmark/bin/benchmark @@ -31,7 +31,6 @@ fi BINDIR=`dirname "$0"` BENCH_HOME=`cd $BINDIR/..;pwd` -BK_HOME=${BK_HOME:-"`cd ${BINDIR}/../..;pwd`"} RELEASE_JAR=`ls $BENCH_HOME/bookkeeper-benchmark-*.jar 2> /dev/null | tail -1` if [ $? == 0 ]; then @@ -70,11 +69,18 @@ EOF } add_maven_deps_to_classpath() { - f="${BENCH_HOME}/build/classpath.txt" + MVN="mvn" + if [ "$MAVEN_HOME" != "" ]; then + MVN=${MAVEN_HOME}/bin/mvn + fi + + # Need to generate classpath from maven pom. This is costly so generate it + # and cache it. Save the file into our target dir so a mvn clean will get + # clean it up and force us create a new one. + f="${BENCH_HOME}/target/cached_classpath.txt" if [ ! -f "${f}" ] then - echo "no classpath.txt found at ${BENCH_HOME}/build" - exit 1 + ${MVN} -f "${BENCH_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null fi BENCHMARK_CLASSPATH=${CLASSPATH}:`cat "${f}"` } diff --git a/bookkeeper-benchmark/build.gradle b/bookkeeper-benchmark/build.gradle index d0089873813..3dec69e5229 100644 --- a/bookkeeper-benchmark/build.gradle +++ b/bookkeeper-benchmark/build.gradle @@ -74,6 +74,5 @@ test { } jar { - dependsOn tasks.named("writeClasspath") archiveBaseName = 'bookkeeper-benchmark' } diff --git a/bookkeeper-benchmark/pom.xml b/bookkeeper-benchmark/pom.xml new file mode 100644 index 00000000000..fc7ed6f5d47 --- /dev/null +++ b/bookkeeper-benchmark/pom.xml @@ -0,0 +1,85 @@ + + + + + 4.0.0 + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + + org.apache.bookkeeper + bookkeeper-benchmark + Apache BookKeeper :: Benchmark + + UTF-8 + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + target/latencyDump.dat + + + + + + + + org.apache.zookeeper + zookeeper + test-jar + test + + + + org.xerial.snappy + snappy-java + test + + + + io.dropwizard.metrics + metrics-core + test + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + compile + + + org.apache.bookkeeper + bookkeeper-common + ${project.version} + test-jar + test + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + test-jar + test + + + diff --git a/bookkeeper-common-allocator/pom.xml b/bookkeeper-common-allocator/pom.xml new file mode 100644 index 00000000000..31315773665 --- /dev/null +++ b/bookkeeper-common-allocator/pom.xml @@ -0,0 +1,60 @@ + + + + 4.0.0 + + org.apache.bookkeeper + bookkeeper + 4.15.0-SNAPSHOT + + bookkeeper-common-allocator + Apache BookKeeper :: Common :: Allocator + + + io.netty + netty-buffer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + diff --git a/bookkeeper-common/pom.xml b/bookkeeper-common/pom.xml new file mode 100644 index 00000000000..e7709f4597a --- /dev/null +++ b/bookkeeper-common/pom.xml @@ -0,0 +1,129 @@ + + + + 4.0.0 + + org.apache.bookkeeper + bookkeeper + 4.15.0-SNAPSHOT + + bookkeeper-common + Apache BookKeeper :: Common + + + org.apache.bookkeeper.stats + bookkeeper-stats-api + ${project.parent.version} + + + org.apache.bookkeeper + cpu-affinity + ${project.parent.version} + + + com.google.guava + guava + + + io.netty + netty-common + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-annotations + + + org.jctools + jctools-core + + + com.google.code.findbugs + jsr305 + provided + + + com.google.errorprone + error_prone_annotations + provided + + + org.apache.commons + commons-lang3 + test + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version} + test + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + test + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + test + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + diff --git a/bookkeeper-dist/all/pom.xml b/bookkeeper-dist/all/pom.xml new file mode 100644 index 00000000000..146dafa3271 --- /dev/null +++ b/bookkeeper-dist/all/pom.xml @@ -0,0 +1,159 @@ + + + 4.0.0 + + bookkeeper-dist + org.apache.bookkeeper + 4.15.0-SNAPSHOT + .. + + + bookkeeper-dist-all + jar + Apache BookKeeper :: Dist (All) + + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + + + + + org.apache.bookkeeper.stats + bookkeeper-stats-api + ${project.version} + + + + org.apache.bookkeeper.stats + codahale-metrics-provider + ${project.version} + + + org.apache.bookkeeper.stats + prometheus-metrics-provider + ${project.version} + + + + + org.apache.bookkeeper.http + http-server + ${project.version} + + + org.apache.bookkeeper.http + vertx-http-server + ${project.version} + + + + + org.apache.bookkeeper + bookkeeper-tools + ${project.version} + + + + + org.apache.distributedlog + distributedlog-core + ${project.version} + + + + + + org.apache.bookkeeper + stream-storage-server + ${project.version} + + + + + org.apache.bookkeeper + bookkeeper-benchmark + ${project.version} + + + + + org.apache.logging.log4j + log4j-1.2-api + + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + + org.xerial.snappy + snappy-java + + + + io.dropwizard.metrics + metrics-core + + + + + + + maven-assembly-plugin + ${maven-assembly-plugin.version} + + bookkeeper-all-${project.version} + false + + ../src/assemble/bin-all.xml + + posix + + + + package + + single + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + ${maven-deploy-plugin.version} + + true + + + + + diff --git a/bookkeeper-dist/bkctl/pom.xml b/bookkeeper-dist/bkctl/pom.xml new file mode 100644 index 00000000000..8ac6545851e --- /dev/null +++ b/bookkeeper-dist/bkctl/pom.xml @@ -0,0 +1,96 @@ + + + 4.0.0 + + bookkeeper-dist + org.apache.bookkeeper + 4.15.0-SNAPSHOT + .. + + + bkctl + jar + Apache BookKeeper :: Dist (Bkctl) + + + + + org.apache.bookkeeper + bookkeeper-tools + ${project.version} + + + org.rocksdb + rocksdbjni + + + + + + + org.apache.logging.log4j + log4j-1.2-api + + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + + + + maven-assembly-plugin + ${maven-assembly-plugin.version} + + bkctl-${project.version} + true + + ../src/assemble/bkctl.xml + + posix + + + + package + + single + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + ${maven-deploy-plugin.version} + + true + + + + + diff --git a/bookkeeper-dist/build.gradle b/bookkeeper-dist/build.gradle index d1e9445174b..5fc27cf8a5e 100644 --- a/bookkeeper-dist/build.gradle +++ b/bookkeeper-dist/build.gradle @@ -38,6 +38,7 @@ distributions { "**/README.md", "**/bin/**", "**/conf/**", + "**/pom.xml", "**/src/**", "deploy/**", "doc/**", diff --git a/bookkeeper-dist/pom.xml b/bookkeeper-dist/pom.xml new file mode 100644 index 00000000000..5285131dbcb --- /dev/null +++ b/bookkeeper-dist/pom.xml @@ -0,0 +1,70 @@ + + + + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + + 4.0.0 + bookkeeper-dist + pom + Apache BookKeeper :: Dist (Parent) + + all + server + bkctl + + + UTF-8 + UTF-8 + + + + + maven-jar-plugin + ${maven-jar-plugin.version} + + + default-jar + none + + + + + maven-assembly-plugin + ${maven-assembly-plugin.version} + + bookkeeper-${project.version} + + src/assemble/src.xml + + posix + + + + package + + single + + + + + + + diff --git a/bookkeeper-dist/server/pom.xml b/bookkeeper-dist/server/pom.xml new file mode 100644 index 00000000000..83777f5f342 --- /dev/null +++ b/bookkeeper-dist/server/pom.xml @@ -0,0 +1,146 @@ + + + 4.0.0 + + bookkeeper-dist + org.apache.bookkeeper + 4.15.0-SNAPSHOT + .. + + + bookkeeper-dist-server + jar + Apache BookKeeper :: Dist (Server) + + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + + + + + org.apache.bookkeeper.stats + bookkeeper-stats-api + ${project.version} + + + org.apache.bookkeeper.stats + prometheus-metrics-provider + ${project.version} + + + + + org.apache.bookkeeper.http + http-server + ${project.version} + + + org.apache.bookkeeper.http + vertx-http-server + ${project.version} + + + + + org.apache.bookkeeper + stream-storage-server + ${project.version} + + + + + org.apache.bookkeeper + bookkeeper-tools + ${project.version} + + + + + org.apache.distributedlog + distributedlog-core + ${project.version} + + + + + org.apache.logging.log4j + log4j-1.2-api + + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + + org.xerial.snappy + snappy-java + + + + io.dropwizard.metrics + metrics-core + + + + + + + maven-assembly-plugin + ${maven-assembly-plugin.version} + + bookkeeper-server-${project.version} + true + + ../src/assemble/bin-server.xml + + posix + + + + package + + single + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + ${maven-deploy-plugin.version} + + true + + + + + + diff --git a/bookkeeper-dist/src/assemble/src.xml b/bookkeeper-dist/src/assemble/src.xml index 0301c8a9bde..146061e28a4 100644 --- a/bookkeeper-dist/src/assemble/src.xml +++ b/bookkeeper-dist/src/assemble/src.xml @@ -30,6 +30,7 @@ **/README.md **/LICENSE **/NOTICE + **/pom.xml **/*gradle* **/src/** **/conf/** diff --git a/bookkeeper-http/http-server/pom.xml b/bookkeeper-http/http-server/pom.xml new file mode 100644 index 00000000000..044a5c02850 --- /dev/null +++ b/bookkeeper-http/http-server/pom.xml @@ -0,0 +1,48 @@ + + + + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + ../.. + + 4.0.0 + org.apache.bookkeeper.http + http-server + Apache BookKeeper :: Http :: Http Server + http://maven.apache.org + + + commons-configuration + commons-configuration + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-annotations + + + diff --git a/bookkeeper-http/pom.xml b/bookkeeper-http/pom.xml new file mode 100644 index 00000000000..bb9307322bf --- /dev/null +++ b/bookkeeper-http/pom.xml @@ -0,0 +1,34 @@ + + + + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + + 4.0.0 + org.apache.bookkeeper.http + bookkeeper-http + pom + Apache BookKeeper :: Http + + http-server + vertx-http-server + servlet-http-server + + diff --git a/bookkeeper-http/servlet-http-server/pom.xml b/bookkeeper-http/servlet-http-server/pom.xml new file mode 100644 index 00000000000..2efa234a518 --- /dev/null +++ b/bookkeeper-http/servlet-http-server/pom.xml @@ -0,0 +1,57 @@ + + + + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + ../.. + + 4.0.0 + org.apache.bookkeeper.http + servlet-http-server + Apache BookKeeper :: Bookkeeper Http :: Servlet Http Server + http://maven.apache.org + + + + org.apache.bookkeeper.http + http-server + ${project.version} + + + javax.servlet + javax.servlet-api + + + commons-io + commons-io + + + org.eclipse.jetty + jetty-server + test + + + org.eclipse.jetty + jetty-webapp + test + + + + diff --git a/bookkeeper-http/vertx-http-server/pom.xml b/bookkeeper-http/vertx-http-server/pom.xml new file mode 100644 index 00000000000..c8914241361 --- /dev/null +++ b/bookkeeper-http/vertx-http-server/pom.xml @@ -0,0 +1,49 @@ + + + + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + ../.. + + 4.0.0 + org.apache.bookkeeper.http + vertx-http-server + Apache BookKeeper :: Bookkeeper Http :: Vertx Http Server + http://maven.apache.org + + + io.vertx + vertx-core + + + io.vertx + vertx-web + + + com.google.guava + guava + + + org.apache.bookkeeper.http + http-server + ${project.version} + + + diff --git a/bookkeeper-proto/pom.xml b/bookkeeper-proto/pom.xml new file mode 100644 index 00000000000..0894fd064ec --- /dev/null +++ b/bookkeeper-proto/pom.xml @@ -0,0 +1,66 @@ + + + + 4.0.0 + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + + bookkeeper-proto + Apache BookKeeper :: Protocols + + + com.google.protobuf + protobuf-java + + + + + + org.apache.rat + apache-rat-plugin + + + + **/DataFormats.java + **/BookkeeperProtocol.java + **/DbLedgerStorageDataFormats.java + **/.checkstyle + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} + true + + + + + compile + + + + + + + diff --git a/bookkeeper-server/build.gradle b/bookkeeper-server/build.gradle index 53187915aba..2d48d6b2779 100644 --- a/bookkeeper-server/build.gradle +++ b/bookkeeper-server/build.gradle @@ -80,6 +80,11 @@ dependencies { testImplementation depLibs.log4jCore } +task writeClasspath { + buildDir.mkdirs() + new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" +} + test { retry { maxFailures = 200 @@ -96,6 +101,3 @@ test.doFirst { jvmArgs("-Djunit.timeout.test=600000", "-Djunit.max.retry=3", "-Djava.net.preferIPv4Stack=true", "-Dio.netty.leakDetection.level=paranoid") } -jar { - dependsOn tasks.named("writeClasspath") -} diff --git a/bookkeeper-server/pom.xml b/bookkeeper-server/pom.xml new file mode 100644 index 00000000000..469aab161d9 --- /dev/null +++ b/bookkeeper-server/pom.xml @@ -0,0 +1,336 @@ + + + + 4.0.0 + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + + bookkeeper-server + Apache BookKeeper :: Server + + + org.apache.bookkeeper + bookkeeper-common + ${project.parent.version} + + + org.apache.bookkeeper + bookkeeper-common-allocator + ${project.parent.version} + + + org.apache.bookkeeper + bookkeeper-proto + ${project.parent.version} + + + org.apache.bookkeeper + bookkeeper-tools-framework + ${project.parent.version} + + + org.rocksdb + rocksdbjni + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version} + test + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + test + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + test + + + org.apache.zookeeper + zookeeper + + + io.netty + netty-handler + + + io.netty + netty-transport-native-epoll + linux-x86_64 + + + io.netty + netty-tcnative-boringssl-static + + + org.apache.bookkeeper.http + http-server + ${project.version} + + + org.apache.bookkeeper + circe-checksum + ${project.version} + + + commons-cli + commons-cli + + + commons-codec + commons-codec + + + commons-io + commons-io + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-collections4 + + + org.bouncycastle + bc-fips + + + com.beust + jcommander + + + net.java.dev.jna + jna + + + org.apache.httpcomponents + httpclient + + + + org.apache.bookkeeper + bookkeeper-common + ${project.parent.version} + test-jar + test + + + org.apache.kerby + kerby-config + ${kerby.version} + test + + + org.slf4j + * + + + + + org.apache.kerby + kerb-simplekdc + ${kerby.version} + test + + + org.slf4j + * + + + + + org.apache.zookeeper + zookeeper + test-jar + test + + + + org.xerial.snappy + snappy-java + test + + + + io.dropwizard.metrics + metrics-core + test + + + org.apache.bookkeeper.stats + prometheus-metrics-provider + ${project.parent.version} + test + + + org.apache.bookkeeper.http + vertx-http-server + ${project.parent.version} + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + test-jar + + + + + + org.apache.rat + apache-rat-plugin + + + + **/target/**/* + **/.classpath + **/.gitignore + **/.project + **/.checkstyle + **/.settings/* + src/test/resources/server-key.pem + src/test/resources/server-key.p12 + src/test/resources/server-key.jks + src/test/resources/server-cert.pem + src/test/resources/client-key.pem + src/test/resources/client-key.p12 + src/test/resources/client-key.jks + src/test/resources/client-cert.pem + src/test/resources/keyStoreClientPassword.txt + src/test/resources/keyStoreServerPassword.txt + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + false + + + listener + org.apache.bookkeeper.common.testing.util.TimedOutTestsListener + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + + none + org.apache.bookkeeper.client:org.apache.bookkeeper.conf:org.apache.bookkeeper.feature + + + Bookkeeper + org.apache.bookkeeper* + + + + + + attach-javadocs + + jar + + + + + + + + + vertx-http-server + + + org.apache.bookkeeper.http + vertx-http-server + ${project.parent.version} + + + + + tls-certs + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + Generate Self-Signed Certificates + generate-test-resources + + exec + + + ${basedir}/src/test/resources + ${basedir}/src/test/resources/generateKeysAndCerts.sh + + + + + + + + + + skipBookKeeperServerTests + + + skipBookKeeperServerTests + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + + + diff --git a/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml b/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml new file mode 100644 index 00000000000..7ba68aa96e2 --- /dev/null +++ b/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml @@ -0,0 +1,53 @@ + + + + 4.0.0 + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + ../.. + + org.apache.bookkeeper.stats + codahale-metrics-provider + Apache BookKeeper :: Stats Providers :: Codahale Metrics + http://maven.apache.org + + + org.apache.bookkeeper.stats + bookkeeper-stats-api + ${project.parent.version} + + + io.dropwizard.metrics + metrics-core + + + io.dropwizard.metrics + metrics-jvm + + + io.dropwizard.metrics + metrics-graphite + + + com.google.guava + guava + + + diff --git a/bookkeeper-stats-providers/pom.xml b/bookkeeper-stats-providers/pom.xml new file mode 100644 index 00000000000..faed3ad46de --- /dev/null +++ b/bookkeeper-stats-providers/pom.xml @@ -0,0 +1,33 @@ + + + + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + + 4.0.0 + bookkeeper-stats-providers + pom + Apache BookKeeper :: Stats Providers + + codahale-metrics-provider + prometheus-metrics-provider + + + diff --git a/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml b/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml new file mode 100644 index 00000000000..3dfe19c0b2a --- /dev/null +++ b/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml @@ -0,0 +1,72 @@ + + + + 4.0.0 + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + ../.. + + org.apache.bookkeeper.stats + prometheus-metrics-provider + Apache BookKeeper :: Stats Providers :: Prometheus + + + org.apache.bookkeeper.stats + bookkeeper-stats-api + ${project.parent.version} + + + + io.prometheus + simpleclient + + + + io.prometheus + simpleclient_hotspot + + + + io.prometheus + simpleclient_servlet + + + + io.netty + netty-common + + + + org.eclipse.jetty + jetty-servlet + + + + com.google.guava + guava + + + + com.yahoo.datasketches + sketches-core + + + + diff --git a/bookkeeper-stats/pom.xml b/bookkeeper-stats/pom.xml new file mode 100644 index 00000000000..b89cb4bcc97 --- /dev/null +++ b/bookkeeper-stats/pom.xml @@ -0,0 +1,63 @@ + + + + 4.0.0 + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + + org.apache.bookkeeper.stats + bookkeeper-stats-api + Apache BookKeeper :: Stats API + http://maven.apache.org + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + + none + org.apache.bookkeeper.stats + + + Bookkeeper Stats API + org.apache.bookkeeper.stats + + + + + + attach-javadocs + + jar + + + + + + + + + commons-configuration + commons-configuration + + + diff --git a/build.gradle b/build.gradle index a8bd1398f00..344335d9fdb 100644 --- a/build.gradle +++ b/build.gradle @@ -158,11 +158,9 @@ allprojects { tasks.withType(Tar) { duplicatesStrategy = DuplicatesStrategy.EXCLUDE } - tasks.withType(Zip) { duplicatesStrategy = DuplicatesStrategy.EXCLUDE } - task testJar(type: Jar, dependsOn: testClasses) { archiveClassifier = 'tests' from sourceSets.test.output @@ -313,12 +311,6 @@ allprojects { implementation(enforcedPlatform(depLibs.nettyBom)) testImplementation depLibs.log4jSlf4jImpl } - tasks.register('writeClasspath') { - doLast { - buildDir.mkdirs() - new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" - } - } dependencies { implementation(enforcedPlatform(depLibs.guavaBom)) diff --git a/buildtools/pom.xml b/buildtools/pom.xml new file mode 100644 index 00000000000..1dced8306c6 --- /dev/null +++ b/buildtools/pom.xml @@ -0,0 +1,28 @@ + + + + 4.0.0 + + org.apache.bookkeeper + bookkeeper + 4.15.0-SNAPSHOT + + buildtools + Apache BookKeeper :: Build Tools + 4.15.0-SNAPSHOT + diff --git a/circe-checksum/pom.xml b/circe-checksum/pom.xml new file mode 100644 index 00000000000..75fac546587 --- /dev/null +++ b/circe-checksum/pom.xml @@ -0,0 +1,236 @@ + + + + 4.0.0 + + org.apache.bookkeeper + bookkeeper + 4.15.0-SNAPSHOT + .. + + + circe-checksum + nar + Apache BookKeeper :: Circe Checksum Library + Circe Checksum Library + + + dynamic + -msse4.2 -mpclmul + + + + + + com.google.guava + guava + + + + io.netty + netty-buffer + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + 1.8 + 1.8 + + + + -Xlint:deprecation + -Xlint:unchecked + + -Xpkginfo:always + + + + + com.github.maven-nar + nar-maven-plugin + ${nar-maven-plugin.version} + true + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven-assembly-plugin.version} + + + src/main/assembly/assembly.xml + + false + posix + + + + make-assembly + package + + single + + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco-maven-plugin.version} + + + + com/scurrilous/circe/checksum/NarSystem* + + + + + + + + + + jdk-without-javah + + [10,) + + + + + com.github.maven-nar + nar-maven-plugin + ${nar-maven-plugin.version} + true + + + + default-nar-javah + none + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + 1.8 + 1.8 + + + + -Xlint:deprecation + -Xlint:unchecked + + -Xpkginfo:always + + -h + ${project.build.directory}/nar/javah-include + + + + + + + + mac + + + Mac OS X + + + + + + com.github.maven-nar + nar-maven-plugin + ${nar-maven-plugin.version} + true + + ${nar.runtime} + circe-checksum + + + jni + com.scurrilous.circe.checksum + + + + ${nar.cpp.optionSet} + false + false + full + + + + + + + + Linux + + + Linux + + + + + + com.github.maven-nar + nar-maven-plugin + ${nar-maven-plugin.version} + true + + ${nar.runtime} + circe-checksum + + + jni + com.scurrilous.circe.checksum + + + + ${nar.cpp.optionSet} + false + false + full + + + rt + + + + + + + + + diff --git a/cpu-affinity/pom.xml b/cpu-affinity/pom.xml new file mode 100644 index 00000000000..c1f8eba9d92 --- /dev/null +++ b/cpu-affinity/pom.xml @@ -0,0 +1,226 @@ + + + 4.0.0 + + org.apache.bookkeeper + bookkeeper + 4.15.0-SNAPSHOT + .. + + + cpu-affinity + nar + Apache BookKeeper :: CPU Affinity Library + CPU Affinity Library + + + dynamic + + + + + com.google.guava + guava + + + org.apache.commons + commons-lang3 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + -Xlint:deprecation + -Xlint:unchecked + + -Xpkginfo:always + + + + + com.github.maven-nar + nar-maven-plugin + ${nar-maven-plugin.version} + true + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven-assembly-plugin.version} + + + src/main/assembly/assembly.xml + + false + posix + + + + make-assembly + package + + single + + + + + + org.apache.rat + apache-rat-plugin + + + + **/src/test/resources/proc_cpuinfo.txt + + + + + + + + + + jdk-without-javah + + [10,) + + + + + com.github.maven-nar + nar-maven-plugin + ${nar-maven-plugin.version} + true + + + + default-nar-javah + none + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + 1.8 + 1.8 + + + + -Xlint:deprecation + -Xlint:unchecked + + -Xpkginfo:always + + -h + ${project.build.directory}/nar/javah-include + + + + + + + + mac + + + Mac OS X + + + + + + com.github.maven-nar + nar-maven-plugin + ${nar-maven-plugin.version} + true + + ${nar.runtime} + cpu-affinity + + + jni + org.apache.bookkeeper.utils.affinity + + + + ${nar.cpp.optionSet} + false + false + full + + + + + + + + + Linux + + + Linux + + + + + + com.github.maven-nar + nar-maven-plugin + ${nar-maven-plugin.version} + true + + ${nar.runtime} + cpu-affinity + + + jni + org.apache.bookkeeper.utils.affinity + + + + ${nar.cpp.optionSet} + false + false + full + + + rt + + + + + + + + + diff --git a/dev/check-all-licenses b/dev/check-all-licenses index 03eeb51c65b..1f548c3cef1 100755 --- a/dev/check-all-licenses +++ b/dev/check-all-licenses @@ -27,7 +27,7 @@ set -e -x HERE=$(dirname $0) BOOKKEEPER_DIST=$HERE/../bookkeeper-dist -$HERE/check-binary-license $BOOKKEEPER_DIST/server/build/distributions/bookkeeper-server-*.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/all/build/distributions/bookkeeper-all-*.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/bkctl/build/distributions/bkctl-*.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/server/target/bookkeeper-server-*-bin.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/all/target/bookkeeper-all-*-bin.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/bkctl/target/bkctl-*-bin.tar.gz diff --git a/dev/check-all-licenses-gradle b/dev/check-all-licenses-gradle new file mode 100755 index 00000000000..03eeb51c65b --- /dev/null +++ b/dev/check-all-licenses-gradle @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Script to check licenses on a binary tarball. +# It extracts the list of bundled jars, the NOTICE, and the LICENSE +# files. It checked that every non-bk jar bundled is mentioned in the +# LICENSE file. It checked that all jar files mentioned in NOTICE and +# LICENSE are actually bundled. + +# all error fatal +set -e -x + +HERE=$(dirname $0) +BOOKKEEPER_DIST=$HERE/../bookkeeper-dist +$HERE/check-binary-license $BOOKKEEPER_DIST/server/build/distributions/bookkeeper-server-*.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/all/build/distributions/bookkeeper-all-*.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/bkctl/build/distributions/bkctl-*.tar.gz + diff --git a/dev/common.sh b/dev/common.sh index 49825406a55..d4f3d3f2efa 100644 --- a/dev/common.sh +++ b/dev/common.sh @@ -21,7 +21,11 @@ # function get_bk_version() { - bk_version=$(${BK_HOME}/gradlew --no-daemon properties -q | grep "version:" | awk '{print $2}') + bk_version=$(mvn -q \ + -Dexec.executable="echo" \ + -Dexec.args='${project.version}' \ + --non-recursive \ + org.codehaus.mojo:exec-maven-plugin:1.3.1:exec) echo ${bk_version} } diff --git a/dev/docker/ci.sh b/dev/docker/ci.sh new file mode 100755 index 00000000000..75588a3d3e2 --- /dev/null +++ b/dev/docker/ci.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -x -u + +SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + +export IMAGE_NAME="bookkeeper/dev" + +pushd ${SCRIPT_DIR} + +docker build --rm=true -t ${IMAGE_NAME} . + +popd + +if [ "$(uname -s)" == "Linux" ]; then + USER_NAME=${SUDO_USER:=$USER} + USER_ID=$(id -u "${USER_NAME}") + GROUP_ID=$(id -g "${USER_NAME}") + LOCAL_HOME="/home/${USER_NAME}" +else # boot2docker uid and gid + USER_NAME=$USER + USER_ID=1000 + GROUP_ID=50 + LOCAL_HOME="/Users/${USER_NAME}" +fi + +docker build -t "${IMAGE_NAME}-${USER_NAME}" - < + + + + org.apache.bookkeeper.metadata.drivers + metadata-drivers-parent + 4.15.0-SNAPSHOT + .. + + 4.0.0 + org.apache.bookkeeper.metadata.drivers + metadata-stores-etcd + Apache BookKeeper :: Metadata Drivers:: Etcd + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + + + + io.etcd + jetcd-core + ${etcd.version} + + + io.grpc + * + + + + + + io.grpc + grpc-all + ${grpc.version} + + + org.bouncycastle + bcpkix-jdk15on + + + io.grpc + grpc-okhttp + + + + + + org.arquillian.cube + arquillian-cube-docker + ${arquillian-cube.version} + + + com.github.docker-java + * + + + test + + + org.jboss.arquillian.junit + arquillian-junit-standalone + ${arquillian-junit.version} + + + com.github.docker-java + * + + + test + + + org.testcontainers + testcontainers + test + + + org.apache.bookkeeper + bookkeeper-common + ${project.version} + test-jar + test + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + test-jar + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + + integrationTests + + + integrationTests + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + + + + + + diff --git a/metadata-drivers/pom.xml b/metadata-drivers/pom.xml new file mode 100644 index 00000000000..e44e116edeb --- /dev/null +++ b/metadata-drivers/pom.xml @@ -0,0 +1,32 @@ + + + + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + + 4.0.0 + org.apache.bookkeeper.metadata.drivers + metadata-drivers-parent + pom + Apache BookKeeper :: Metadata Drivers :: Parent + + etcd + + diff --git a/microbenchmarks/pom.xml b/microbenchmarks/pom.xml new file mode 100644 index 00000000000..dc0093954db --- /dev/null +++ b/microbenchmarks/pom.xml @@ -0,0 +1,129 @@ + + + + 4.0.0 + + org.apache.bookkeeper + bookkeeper + 4.15.0-SNAPSHOT + + microbenchmarks + Apache BookKeeper :: microbenchmarks + http://maven.apache.org + + benchmarks + + + + org.openjdk.jmh + jmh-core + + + org.openjdk.jmh + jmh-generator-annprocess + provided + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-1.2-api + + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.apache.bookkeeper + bookkeeper-server + ${project.parent.version} + compile + jar + + + org.apache.bookkeeper.stats + prometheus-metrics-provider + ${project.version} + + + org.apache.bookkeeper.stats + codahale-metrics-provider + ${project.version} + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + ${javac.target} + ${javac.target} + ${javac.target} + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + package + + shade + + + ${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + + diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000000..ac7658821b0 --- /dev/null +++ b/pom.xml @@ -0,0 +1,1164 @@ + + + + + org.apache + apache + 19 + + 4.0.0 + org.apache.bookkeeper + 4.15.0-SNAPSHOT + bookkeeper + pom + Apache BookKeeper :: Parent + http://bookkeeeper.apache.org + 2011 + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + scm:git:https://github.com/apache/bookkeeper.git + scm:git:https://github.com/apache/bookkeeper.git + https://github.com/apache/bookkeeper + branch-4.13 + + + JIRA + https://issues.apache.org/jira/browse/BOOKKEEPER + + + Jenkins + https://builds.apache.org/job/bookkeeper-master + + + buildtools + circe-checksum + bookkeeper-common + bookkeeper-common-allocator + stats + + bookkeeper-stats + bookkeeper-proto + bookkeeper-server + bookkeeper-benchmark + bookkeeper-stats-providers + bookkeeper-http + stream + tools + cpu-affinity + metadata-drivers + bookkeeper-dist + shaded + microbenchmarks + tests + + + + BookKeeper User + user-subscribe@bookkeeper.apache.org + user-unsubscribe@bookkeeper.apache.org + user@bookkeeper.apache.org + http://www.mail-archive.com/user@bookkeeper.apache.org + + + BookKeeper Dev + dev-subscribe@bookkeeper.apache.org + dev-unsubscribe@bookkeeper.apache.org + dev@bookkeeper.apache.org + http://www.mail-archive.com/dev@bookkeeper.apache.org + + + BookKeeper Commits + commits-subscribe@bookkeeper.apache.org + commits-unsubscribe@bookkeeper.apache.org + commits@bookkeeper.apache.org + http://www.mail-archive.com/commits@bookkeeper.apache.org + + + + + The Apache BookKeeper Team + dev@bookkeeper.apache.org + http://bookkeeper.apache.org + Apache Software Foundation + http://www.apache.org + + + + UTF-8 + UTF-8 + 1.8 + true + 2 + + + 1.18.2 + 1.6.0.Final + 3.0.1 + 1.2 + 4.1 + 1.6 + 1.10 + 1.19 + 2.6 + 3.6 + 2.7 + 1.0.2.1 + 5.1.0 + 3.2.5 + 0.5.11 + 2.7.0 + 3.0.2 + 2.4.0 + 1.42.1 + 30.0-jre + 1.1.1 + 2.10.0 + 1.3 + 2.1.10 + 2.11.0 + 1.78 + 9.4.43.v20210629 + 1.19 + 2.8.2 + 3.2.7 + 4.12 + 0.14.2 + 1.18.20 + 2.17.1 + 1.3.0 + 3.0.0 + 4.1.72.Final + 2.0.46.Final + 9.1.3 + 2.0.2 + 0.8.1 + 0.8.3 + 4.5.13 + 3.14.0 + 3.14.0 + ${grpc.version} + 0.9.11 + 6.22.1.1 + 3.0.1 + 1.7.32 + 1.19 + 3.1.8 + 1.3.2 + 1.15.1 + 3.9.8 + 3.6.2 + 1.1.7 + 2.1.2 + + 0.12 + 2.7 + 4.3.0 + 1.4.13 + 1.6.0 + 1.6 + 0.8.0 + 1.8 + 3.1.0 + 3.0.0 + 2.5 + 3.8.1 + 3.0.2 + 2.7 + 2.5.1 + 2.4 + 3.1.1 + 3.2.0 + 2.2.1 + 3.0.0-M5 + 3.5.2 + 1.4.1.Final + 0.6.1 + 6.19 + 3.1.8 + 1 + 4.0.0 + + + + + + + + com.github.spotbugs + spotbugs-annotations + ${spotbugs-annotations.version} + + + javax.annotation + javax.annotation-api + ${javax-annotations-api.version} + + + com.google.code.findbugs + jsr305 + ${google.code.version} + + + com.google.errorprone + error_prone_annotations + ${google.errorprone.version} + + + org.projectlombok + lombok + ${lombok.version} + + + org.inferred + freebuilder + ${freebuilder.version} + true + + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + + + commons-cli + commons-cli + ${commons-cli.version} + + + commons-codec + commons-codec + ${commons-codec.version} + + + commons-configuration + commons-configuration + ${commons-configuration.version} + + + commons-io + commons-io + ${commons-io.version} + + + commons-lang + commons-lang + ${commons-lang.version} + + + com.google.guava + guava + ${guava.version} + + + org.apache.commons + commons-collections4 + ${commons-collections4.version} + + + org.apache.commons + commons-compress + ${commons-compress.version} + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + + org.bouncycastle + bc-fips + ${bouncycastle.version} + + + + org.reflections + reflections + ${reflections.version} + + + + + net.jpountz.lz4 + lz4 + ${lz4.version} + + + + + net.java.dev.jna + jna + ${jna.version} + + + + + org.yaml + snakeyaml + ${snakeyaml.version} + + + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + javax.servlet + javax.servlet-api + ${servlet-api.version} + + + com.fasterxml.jackson.module + jackson-module-paranamer + ${jackson.version} + + + com.fasterxml.jackson.module + jackson-module-scala_2.11 + ${jackson.version} + + + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + + + org.apache.thrift + libthrift + ${libthrift.version} + + + org.apache.tomcat.embed + tomcat-embed-core + + + javax.annotation + javax.annotation-api + + + + + + + io.netty + netty-common + ${netty.version} + + + io.netty + netty-buffer + ${netty.version} + + + io.netty + netty-transport + ${netty.version} + + + io.netty + netty-handler + ${netty.version} + + + io.netty + netty-transport-native-epoll + ${netty.version} + + + io.netty + netty-transport-native-epoll + ${netty.version} + linux-x86_64 + + + io.netty + netty-codec-dns + ${netty.version} + + + io.netty + netty-codec-http + ${netty.version} + + + io.netty + netty-codec-http2 + ${netty.version} + + + io.netty + netty-codec-socks + ${netty.version} + + + io.netty + netty-handler-proxy + ${netty.version} + + + io.netty + netty-resolver + ${netty.version} + + + io.netty + netty-resolver-dns + ${netty.version} + + + io.netty + netty-tcnative-boringssl-static + ${netty-boringssl.version} + + + + + io.grpc + grpc-bom + ${grpc.version} + pom + import + + + + + org.rocksdb + rocksdbjni + ${rocksdb.version} + + + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + net.java.dev.javacc + javacc + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-api + + + log4j + log4j + + + io.netty + * + + + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + test-jar + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-api + + + log4j + log4j + + + io.netty + * + + + + + + + org.xerial.snappy + snappy-java + ${snappy.version} + + + + + org.apache.curator + curator-recipes + ${curator.version} + + + org.apache.zookeeper + zookeeper + + + + + + + io.vertx + vertx-core + ${vertx.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + + + io.vertx + vertx-web + ${vertx.version} + + + org.eclipse.jetty + jetty-server + ${jetty.version} + + + org.eclipse.jetty + jetty-webapp + ${jetty.version} + + + org.eclipse.jetty + jetty-servlet + ${jetty.version} + + + + + org.jctools + jctools-core + ${jctools.version} + + + + + io.dropwizard.metrics + metrics-core + ${dropwizard.version} + + + + + io.dropwizard.metrics + metrics-jvm + ${dropwizard.version} + + + io.dropwizard.metrics + metrics-graphite + ${dropwizard.version} + + + + io.prometheus + simpleclient + ${prometheus.version} + + + io.prometheus + simpleclient_hotspot + ${prometheus.version} + + + io.prometheus + simpleclient_servlet + ${prometheus.version} + + + + com.yahoo.datasketches + sketches-core + ${datasketches.version} + + + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + + + com.beust + jcommander + ${jcommander.version} + + + + + org.hdrhistogram + HdrHistogram + ${hdrhistogram.version} + + + + + junit + junit + ${junit.version} + + + org.hamcrest + hamcrest-all + ${hamcrest.version} + + + org.jmock + jmock + ${jmock.version} + + + org.mockito + mockito-core + ${mockito.version} + + + org.powermock + powermock-api-mockito2 + ${powermock.version} + + + org.powermock + powermock-module-junit4 + ${powermock.version} + + + org.apache.hadoop + hadoop-minikdc + ${hadoop.minikdc.version} + + + org.arquillian.cube + arquillian-cube-docker + ${arquillian-cube.version} + + + com.github.docker-java + * + + + + + org.jboss.arquillian.junit + arquillian-junit-standalone + ${arquillian-junit.version} + + + com.github.docker-java + * + + + + + org.codehaus.groovy + groovy-all + ${groovy.version} + pom + + + org.jboss.shrinkwrap.resolver + shrinkwrap-resolver-impl-maven + ${shrinkwrap.version} + + + org.jboss.shrinkwrap.resolver + shrinkwrap-resolver-api + ${shrinkwrap.version} + + + org.testcontainers + testcontainers + ${testcontainers.version} + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + + + + + + org.projectlombok + lombok + provided + + + com.github.spotbugs + spotbugs-annotations + provided + + + + + org.slf4j + slf4j-api + + + commons-configuration + commons-configuration + + + + + junit + junit + test + + + org.hamcrest + hamcrest-all + test + + + org.apache.logging.log4j + log4j-core + test + + + org.apache.logging.log4j + log4j-slf4j-impl + test + + + org.mockito + mockito-core + test + + + org.powermock + powermock-api-mockito2 + test + + + org.powermock + powermock-module-junit4 + test + + + + + + + kr.motd.maven + os-maven-plugin + ${os-maven-plugin.version} + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${maven-checkstyle-plugin.version} + + + com.puppycrawl.tools + checkstyle + ${puppycrawl.checkstyle.version} + + + + buildtools/src/main/resources/bookkeeper/checkstyle.xml + buildtools/src/main/resources/bookkeeper/suppressions.xml + UTF-8 + true + true + false + true + + + + checkstyle + validate + + check + + + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${session.executionRootDirectory}/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml + + + + maven-compiler-plugin + ${maven-compiler-plugin.version} + + ${javac.target} + ${javac.target} + + -Werror + -Xlint:deprecation + -Xlint:unchecked + + -Xpkginfo:always + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid + ${redirectTestOutputToFile} + ${forkCount.variable} + false + false + 1800 + ${testRetryCount} + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + -notimestamp + + none + org.apache.bookkeeper.client:org.apache.bookkeeper.client.api:org.apache.bookkeeper.common.annotation:org.apache.bookkeeper.conf:org.apache.bookkeeper.feature:org.apache.bookkeeper.stats + + + Bookkeeper Client + org.apache.bookkeeper.client:org.apache.bookkeeper.common.annotation:org.apache.bookkeeper.conf:org.apache.bookkeeper.feature + + + Bookkeeper Client (New Fluent API - Experimental) + org.apache.bookkeeper.client.api + + + Bookkeeper Stats API + + org.apache.bookkeeper.stats + + + Bookkeeper Stats Providers + org.apache.bookkeeper.stats.codahale:org.apache.bookkeeper.stats.prometheus + + + BookKeeper Java API (version ${project.version}) + site/_site/overview/index.html + package + false + + + + aggregate + + aggregate + + site + + + + + org.apache.maven.plugins + maven-source-plugin + ${maven-source-plugin.version} + + + attach-sources + + jar + + + + + + org.apache.rat + apache-rat-plugin + ${apache-rat-plugin.version} + + + + **/.idea/** + + + .git/**/* + .github/**/* + **/.gitignore + + + **/.svn/**/* + + + **/target/**/* + + + **/README.md + **/README.rst + **/apidocs/* + **/src/main/resources/deps/** + **/META-INF/** + + + **/.classpath + **/.project + **/.checkstyle + **/.settings/* + **/*.iml + **/*.iws + **/*.ipr + + + .repository/** + + + site/** + site2/** + + + **/org/apache/distributedlog/thrift/* + + + **/*.log + + + data/** + + + dev/.vagrant/** + + + **/proto/**.py + + **/python/.coverage + **/python/.Python + **/python/bin/** + **/python/include/** + **/python/lib/** + **/**.pyc + **/.nox/** + **/.pytest_cache/** + **/__pycache__/** + **/bookkeeper.egg-info/** + **/pip-selfcheck.json + + + **/test_conf_2.conf + + true + + + + + + + + code-coverage + + + + org.eluder.coveralls + coveralls-maven-plugin + ${coveralls-maven-plugin.version} + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + @{argLine} -Xmx2G -Djava.net.preferIPv4Stack=true + ${redirectTestOutputToFile} + ${forkCount.variable} + false + 1800 + + true + + + + org.jacoco + jacoco-maven-plugin + 0.8.0 + + + + prepare-agent + + + + + + + + + + + dev + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + -Xmx2G -Djava.net.preferIPv4Stack=true + false + ${forkCount.variable} + false + 1800 + false + 0 + + + + + + + dev-debug + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + -Xmx2G -Djava.net.preferIPv4Stack=true -Dbookkeeper.root.logger=DEBUG,CONSOLE + false + ${forkCount.variable} + false + 1800 + false + + + + + + + + jdk11-no-spotbugs + + [11,) + + + + + com.github.spotbugs + spotbugs-maven-plugin + + true + + + + + + + + apache-release + + + + maven-assembly-plugin + + + source-release-assembly + + + true + + + + + + + + + diff --git a/settings.gradle b/settings.gradle index b275534b0f3..550476e3f3b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -32,8 +32,6 @@ pluginManagement { rootProject.name = 'bookkeeper' - - include(':bookkeeper-benchmark', ':bookkeeper-tools', ':bookkeeper-tools-framework', diff --git a/shaded/bookkeeper-server-shaded/pom.xml b/shaded/bookkeeper-server-shaded/pom.xml new file mode 100644 index 00000000000..0644f91bcf9 --- /dev/null +++ b/shaded/bookkeeper-server-shaded/pom.xml @@ -0,0 +1,128 @@ + + + + 4.0.0 + + org.apache.bookkeeper + shaded-parent + 4.15.0-SNAPSHOT + .. + + bookkeeper-server-shaded + Apache BookKeeper :: Shaded :: bookkeeper-server-shaded + + UTF-8 + + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + package + + shade + + + true + true + false + + + com.google.guava:guava + com.google.protobuf:protobuf-java + org.apache.bookkeeper:bookkeeper-common + org.apache.bookkeeper:bookkeeper-common-allocator + org.apache.bookkeeper:cpu-affinity + org.apache.bookkeeper:bookkeeper-tools-framework + org.apache.bookkeeper:bookkeeper-proto + org.apache.bookkeeper:bookkeeper-server + org.apache.bookkeeper:circe-checksum + org.apache.bookkeeper.stats:bookkeeper-stats-api + + + + + com.google + org.apache.bookkeeper.shaded.com.google + + + + + + + + org.codehaus.mojo + license-maven-plugin + ${license-maven-plugin.version} + + false + ${project.basedir} + + + + update-pom-license + + update-file-header + + package + + apache_v2 + + dependency-reduced-pom.xml + + + + + + + maven-clean-plugin + ${maven-clean-plugin.version} + + + + ${project.basedir} + + dependency-reduced-pom.xml + + + + + + + + diff --git a/shaded/bookkeeper-server-tests-shaded/pom.xml b/shaded/bookkeeper-server-tests-shaded/pom.xml new file mode 100644 index 00000000000..a5cd9f6a177 --- /dev/null +++ b/shaded/bookkeeper-server-tests-shaded/pom.xml @@ -0,0 +1,147 @@ + + + + 4.0.0 + + org.apache.bookkeeper + shaded-parent + 4.15.0-SNAPSHOT + .. + + bookkeeper-server-tests-shaded + Apache BookKeeper :: Shaded :: bookkeeper-server-tests-shaded + + UTF-8 + + + + org.apache.bookkeeper + bookkeeper-server + test-jar + ${project.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + org.apache.bookkeeper + bookkeeper-common + + + org.apache.bookkeeper + bookkeeper-tools-framework + + + org.apache.bookkeeper + bookkeeper-proto + + + org.apache.bookkeeper + bookkeeper-server + + + org.apache.bookkeeper + circe-checksum + + + org.apache.bookkeeper.stats + bookkeeper-stats-api + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + package + + shade + + + true + true + false + + + com.google.guava:guava + com.google.protobuf:protobuf-java + org.apache.bookkeeper:bookkeeper-server:test-jar:tests + + + + + + com.google + org.apache.bookkeeper.shaded.com.google + + + + + + + + org.codehaus.mojo + license-maven-plugin + ${license-maven-plugin.version} + + false + ${project.basedir} + + + + update-pom-license + + update-file-header + + package + + apache_v2 + + dependency-reduced-pom.xml + + + + + + + maven-clean-plugin + ${maven-clean-plugin.version} + + + + ${project.basedir} + + dependency-reduced-pom.xml + + + + + + + + diff --git a/shaded/distributedlog-core-shaded/pom.xml b/shaded/distributedlog-core-shaded/pom.xml new file mode 100644 index 00000000000..c1cf1fef955 --- /dev/null +++ b/shaded/distributedlog-core-shaded/pom.xml @@ -0,0 +1,251 @@ + + + + 4.0.0 + + org.apache.bookkeeper + shaded-parent + 4.15.0-SNAPSHOT + .. + + org.apache.distributedlog + distributedlog-core-shaded + Apache BookKeeper :: Shaded :: distributedlog-core-shaded + + UTF-8 + + + + org.apache.distributedlog + distributedlog-core + ${project.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + io.netty + netty-common + + + io.netty + netty-buffer + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + package + + shade + + + true + true + false + + + commons-codec:commons-codec + commons-cli:commons-cli + commons-io:commons-io + commons-lang:commons-lang + commons-logging:commons-logging + com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.core:jackson-databind + com.fasterxml.jackson.core:jackson-annotations + com.google.guava:guava + com.google.protobuf:protobuf-java + net.java.dev.jna:jna + net.jpountz.lz4:lz4 + org.apache.bookkeeper:bookkeeper-common + org.apache.bookkeeper:bookkeeper-common-allocator + org.apache.bookkeeper:cpu-affinity + org.apache.bookkeeper:bookkeeper-tools-framework + org.apache.bookkeeper:bookkeeper-proto + org.apache.bookkeeper:bookkeeper-server + org.apache.bookkeeper:circe-checksum + org.apache.bookkeeper.http:http-server + org.apache.bookkeeper.stats:bookkeeper-stats-api + org.apache.commons:commons-collections4 + org.apache.commons:commons-lang3 + org.apache.distributedlog:distributedlog-common + org.apache.distributedlog:distributedlog-core + org.apache.distributedlog:distributedlog-protocol + org.apache.httpcomponents:httpclient + org.apache.httpcomponents:httpcore + org.apache.thrift:libthrift + org.apache.zookeeper:zookeeper + org.apache.zookeeper:zookeeper-jute + org.rocksdb:rocksdbjni + + + + + + org.apache.commons.cli + dlshade.org.apache.commons.cli + + + org.apache.commons.codec + dlshade.org.apache.commons.codec + + + org.apache.commons.collections4 + dlshade.org.apache.commons.collections4 + + + org.apache.commons.lang + dlshade.org.apache.commons.lang + + + org.apache.commons.lang3 + dlshade.org.apache.commons.lang3 + + + org.apache.commons.logging + dlshade.org.apache.commons.logging + + + org.apache.commons.io + dlshade.org.apache.commons.io + + + + org.apache.httpcomponents + dlshade.org.apache.httpcomponents + + + org.apache.http + dlshade.org.apache.http + + + + org.apache.thrift + dlshade.org.apache.thrift + + + + org.apache.zookeeper + dlshade.org.apache.zookeeper + + + org.apache.jute + dlshade.org.apache.jute + + + + com.fasterxml.jackson + dlshade.com.fasterxml.jackson + + + + com.sun.jna + dlshade.com.sun.jna + + + + com.google + dlshade.com.google + + + + org.jboss.netty + dlshade.org.jboss.netty + + + + net.jpountz + dlshade.net.jpountz + + + + org.rocksdb + dlshade.org.rocksdb + + + + com.scurrilous.circe + dlshade.com.scurrilous.circe + + + org.apache.bookkeeper + dlshade.org.apache.bookkeeper + + + + org.apache.distributedlog + org.apache.distributedlog + + + + + + + + org.codehaus.mojo + license-maven-plugin + ${license-maven-plugin.version} + + false + ${project.basedir} + + + + update-pom-license + + update-file-header + + package + + apache_v2 + + dependency-reduced-pom.xml + + + + + + + maven-clean-plugin + ${maven-clean-plugin.version} + + + + ${project.basedir} + + dependency-reduced-pom.xml + + + + + + + + diff --git a/shaded/pom.xml b/shaded/pom.xml new file mode 100644 index 00000000000..254de71d961 --- /dev/null +++ b/shaded/pom.xml @@ -0,0 +1,34 @@ + + + + pom + 4.0.0 + + org.apache.bookkeeper + bookkeeper + 4.15.0-SNAPSHOT + + org.apache.bookkeeper + shaded-parent + Apache BookKeeper :: Shaded :: Parent + + bookkeeper-server-shaded + bookkeeper-server-tests-shaded + distributedlog-core-shaded + + diff --git a/stats/pom.xml b/stats/pom.xml new file mode 100644 index 00000000000..d2c89aa671f --- /dev/null +++ b/stats/pom.xml @@ -0,0 +1,38 @@ + + + + 4.0.0 + + org.apache.bookkeeper + bookkeeper + 4.15.0-SNAPSHOT + .. + + pom + org.apache.bookkeeper.stats + bookkeeper-stats-parent + Apache BookKeeper :: Stats :: Parent + + + utils + + + + diff --git a/stats/utils/pom.xml b/stats/utils/pom.xml new file mode 100644 index 00000000000..cdfebcd08cc --- /dev/null +++ b/stats/utils/pom.xml @@ -0,0 +1,54 @@ + + + + 4.0.0 + + bookkeeper-stats-parent + org.apache.bookkeeper.stats + 4.15.0-SNAPSHOT + .. + + org.apache.bookkeeper.stats + bookkeeper-stats-utils + Apache BookKeeper :: Stats :: Utils + + + org.apache.bookkeeper.stats + bookkeeper-stats-api + ${project.version} + + + org.reflections + reflections + + + org.yaml + snakeyaml + + + com.beust + jcommander + + + com.fasterxml.jackson.core + jackson-annotations + + + diff --git a/stream/api/pom.xml b/stream/api/pom.xml new file mode 100644 index 00000000000..82906fdb484 --- /dev/null +++ b/stream/api/pom.xml @@ -0,0 +1,49 @@ + + + + 4.0.0 + + stream-storage-parent + org.apache.bookkeeper + 4.15.0-SNAPSHOT + .. + + org.apache.bookkeeper + stream-storage-api + Apache BookKeeper :: Stream Storage :: API + + + org.apache.bookkeeper + bookkeeper-common + ${project.parent.version} + + + org.apache.bookkeeper + stream-storage-common + ${project.parent.version} + + + io.netty + netty-buffer + + + org.inferred + freebuilder + + + diff --git a/stream/bin/streamstorage b/stream/bin/streamstorage index 1f41b2258ac..a85c6e70d32 100755 --- a/stream/bin/streamstorage +++ b/stream/bin/streamstorage @@ -20,7 +20,6 @@ BINDIR=$(dirname "$0") SS_HOME=`cd $BINDIR/..;pwd` -BK_HOME=${BK_HOME:-"`cd ${BINDIR}/../..;pwd`"} DEFAULT_STANDALONE_CONF=$SS_HOME/conf/standalone.conf DEFAULT_LOG_CONF=$SS_HOME/conf/log4j.properties @@ -80,14 +79,18 @@ EOF } add_maven_deps_to_classpath() { + MVN="mvn" + if [ "$MAVEN_HOME" != "" ]; then + MVN=${MAVEN_HOME}/bin/mvn + fi + # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. - f="${SS_HOME}/server/build/classpath.txt" + f="${SS_HOME}/server/target/classpath.txt" if [ ! -f "${f}" ] then - echo "no classpath.txt found at ${SS_HOME}/server/build" - exit 1 + ${MVN} -f "${SS_HOME}/server/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null fi SS_CLASSPATH=${CLASSPATH}:`cat "${f}"` } diff --git a/stream/bin/streamstorage-cli b/stream/bin/streamstorage-cli index 365c50dcf16..6a1df7810d8 100755 --- a/stream/bin/streamstorage-cli +++ b/stream/bin/streamstorage-cli @@ -57,14 +57,18 @@ elif [ -e "$BUILT_JAR" ]; then fi add_maven_deps_to_classpath() { + MVN="mvn" + if [ "$MAVEN_HOME" != "" ]; then + MVN=${MAVEN_HOME}/bin/mvn + fi + # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. - f="${SS_HOME}/cli/build/classpath.txt" + f="${SS_HOME}/cli/target/classpath.txt" if [ ! -f "${f}" ] then - echo "no classpath.txt found at ${SS_HOME}/cli/build" - exit 1 + ${MVN} -f "${SS_HOME}/cli/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null fi SS_CLASSPATH=${CLASSPATH}:`cat "${f}"` } diff --git a/stream/bk-grpc-name-resolver/pom.xml b/stream/bk-grpc-name-resolver/pom.xml new file mode 100644 index 00000000000..37feb436205 --- /dev/null +++ b/stream/bk-grpc-name-resolver/pom.xml @@ -0,0 +1,80 @@ + + + + 4.0.0 + + stream-storage-parent + org.apache.bookkeeper + 4.15.0-SNAPSHOT + .. + + org.apache.bookkeeper + bk-grpc-name-resolver + Apache BookKeeper :: Stream Storage :: Common :: BK Grpc Name Resolver + + + org.apache.bookkeeper + stream-storage-common + ${project.version} + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + + + org.apache.bookkeeper + stream-storage-java-client-base + ${project.version} + test + + + org.apache.bookkeeper + bookkeeper-common + ${project.version} + test-jar + test + + + org.apache.zookeeper + zookeeper + test-jar + test + + + + org.xerial.snappy + snappy-java + test + + + + io.dropwizard.metrics + metrics-core + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + test-jar + test + + + diff --git a/stream/clients/java/all/pom.xml b/stream/clients/java/all/pom.xml new file mode 100644 index 00000000000..1aa66fd526a --- /dev/null +++ b/stream/clients/java/all/pom.xml @@ -0,0 +1,109 @@ + + + + 4.0.0 + + org.apache.bookkeeper + stream-storage-java-client-parent + 4.15.0-SNAPSHOT + + stream-storage-java-client + Apache BookKeeper :: Stream Storage :: Clients :: Java Client + + + + org.apache.bookkeeper + stream-storage-java-kv-client + ${project.parent.version} + + + org.apache.bookkeeper + stream-storage-java-client-base + ${project.parent.version} + test-jar + test + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + package + + shade + + + true + true + false + + + org.apache.bookkeeper:stream-storage-* + + + + + + + + org.codehaus.mojo + license-maven-plugin + ${license-maven-plugin.version} + + false + + ${project.basedir} + + + + + update-pom-license + + update-file-header + + package + + apache_v2 + + dependency-reduced-pom.xml + + + + + + + maven-clean-plugin + ${maven-clean-plugin.version} + + + + ${project.basedir} + + dependency-reduced-pom.xml + + + + + + + + diff --git a/stream/clients/java/base/pom.xml b/stream/clients/java/base/pom.xml new file mode 100644 index 00000000000..c8d8388420b --- /dev/null +++ b/stream/clients/java/base/pom.xml @@ -0,0 +1,56 @@ + + + + 4.0.0 + + org.apache.bookkeeper + stream-storage-java-client-parent + 4.15.0-SNAPSHOT + + stream-storage-java-client-base + Apache BookKeeper :: Stream Storage :: Clients :: Java Client :: Base + + + + org.apache.bookkeeper + stream-storage-api + ${project.parent.version} + + + org.apache.bookkeeper + stream-storage-proto + ${project.parent.version} + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + test-jar + + + + + + + diff --git a/stream/clients/java/kv/pom.xml b/stream/clients/java/kv/pom.xml new file mode 100644 index 00000000000..d9c9e359324 --- /dev/null +++ b/stream/clients/java/kv/pom.xml @@ -0,0 +1,44 @@ + + + + 4.0.0 + + org.apache.bookkeeper + stream-storage-java-client-parent + 4.15.0-SNAPSHOT + + stream-storage-java-kv-client + Apache BookKeeper :: Stream Storage :: Clients :: Java Client :: KV + + + + org.apache.bookkeeper + stream-storage-java-client-base + ${project.parent.version} + + + org.apache.bookkeeper + stream-storage-java-client-base + ${project.parent.version} + test-jar + test + + + + + diff --git a/stream/clients/java/pom.xml b/stream/clients/java/pom.xml new file mode 100644 index 00000000000..44a462220e9 --- /dev/null +++ b/stream/clients/java/pom.xml @@ -0,0 +1,34 @@ + + + + pom + 4.0.0 + + org.apache.bookkeeper + stream-storage-clients-parent + 4.15.0-SNAPSHOT + .. + + stream-storage-java-client-parent + Apache BookKeeper :: Stream Storage :: Clients :: Java Client :: Parent + + base + kv + all + + diff --git a/stream/clients/pom.xml b/stream/clients/pom.xml new file mode 100644 index 00000000000..3001eeb8d90 --- /dev/null +++ b/stream/clients/pom.xml @@ -0,0 +1,32 @@ + + + + pom + 4.0.0 + + org.apache.bookkeeper + stream-storage-parent + 4.15.0-SNAPSHOT + .. + + stream-storage-clients-parent + Apache BookKeeper :: Stream Storage :: Clients :: Parent + + java + + diff --git a/stream/common/pom.xml b/stream/common/pom.xml new file mode 100644 index 00000000000..6f5091ad933 --- /dev/null +++ b/stream/common/pom.xml @@ -0,0 +1,77 @@ + + + + 4.0.0 + + stream-storage-parent + org.apache.bookkeeper + 4.15.0-SNAPSHOT + .. + + org.apache.bookkeeper + stream-storage-common + Apache BookKeeper :: Stream Storage :: Common + + + org.apache.bookkeeper + bookkeeper-common + ${project.parent.version} + + + io.netty + netty-buffer + + + io.grpc + grpc-all + + + io.grpc + grpc-netty-shaded + + + org.bouncycastle + bcpkix-jdk15on + + + io.grpc + grpc-okhttp + + + + + javax.annotation + javax.annotation-api + + true + + + org.apache.bookkeeper + bookkeeper-common + ${project.version} + test-jar + test + + + org.apache.bookkeeper.tests + stream-storage-tests-common + ${project.version} + test + + + diff --git a/stream/distributedlog/common/pom.xml b/stream/distributedlog/common/pom.xml new file mode 100644 index 00000000000..f91055fa99d --- /dev/null +++ b/stream/distributedlog/common/pom.xml @@ -0,0 +1,109 @@ + + + + 4.0.0 + + org.apache.distributedlog + distributedlog + 4.15.0-SNAPSHOT + + distributedlog-common + Apache BookKeeper :: DistributedLog :: Common + + + org.apache.bookkeeper.stats + bookkeeper-stats-api + ${project.parent.version} + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.bookkeeper + bookkeeper-common + ${project.parent.version} + + + org.apache.commons + commons-lang3 + + + com.google.guava + guava + + + commons-lang + commons-lang + + + commons-codec + commons-codec + + + io.netty + netty-buffer + + + net.jpountz.lz4 + lz4 + + + org.jmock + jmock + test + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + maven-compiler-plugin + ${maven-compiler-plugin.version} + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + ${redirectTestOutputToFile} + -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G + always + 1800 + + + + + diff --git a/stream/distributedlog/core/build.gradle b/stream/distributedlog/core/build.gradle index d8f88a366fc..ee9a79fc10a 100644 --- a/stream/distributedlog/core/build.gradle +++ b/stream/distributedlog/core/build.gradle @@ -75,7 +75,6 @@ test.doFirst { } jar { - dependsOn tasks.named("writeClasspath") archiveBaseName = 'distributedlog-core' } diff --git a/stream/distributedlog/core/pom.xml b/stream/distributedlog/core/pom.xml new file mode 100644 index 00000000000..78c6ff62b28 --- /dev/null +++ b/stream/distributedlog/core/pom.xml @@ -0,0 +1,135 @@ + + + + 4.0.0 + + org.apache.distributedlog + distributedlog + 4.15.0-SNAPSHOT + + distributedlog-core + Apache BookKeeper :: DistributedLog :: Core Library + + + org.apache.distributedlog + distributedlog-protocol + ${project.parent.version} + + + org.apache.zookeeper + zookeeper + + + org.apache.thrift + libthrift + + + org.apache.bookkeeper + bookkeeper-server + ${project.parent.version} + + + javax.annotation + javax.annotation-api + + true + + + org.jmock + jmock + test + + + org.apache.bookkeeper + bookkeeper-common + ${project.version} + test-jar + test + + + org.apache.distributedlog + distributedlog-common + ${project.parent.version} + test-jar + test + + + + org.xerial.snappy + snappy-java + test + + + + io.dropwizard.metrics + metrics-core + test + + + + + + maven-compiler-plugin + ${maven-compiler-plugin.version} + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + false + ${redirectTestOutputToFile} + -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G + always + 1800 + + + listener + org.apache.bookkeeper.common.testing.util.TimedOutTestsListener + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + org.apache.maven.plugins + maven-checkstyle-plugin + + ../../buildtools/src/main/resources/distributedlog/suppressions.xml + ../../buildtools/src/main/resources/bookkeeper/checkstyle.xml + **/thrift/**/* + + + + + diff --git a/stream/distributedlog/io/dlfs/pom.xml b/stream/distributedlog/io/dlfs/pom.xml new file mode 100644 index 00000000000..31366ed9fc0 --- /dev/null +++ b/stream/distributedlog/io/dlfs/pom.xml @@ -0,0 +1,91 @@ + + + + 4.0.0 + + distributedlog + org.apache.distributedlog + 4.15.0-SNAPSHOT + ../.. + + org.apache.distributedlog + dlfs + Apache BookKeeper :: DistributedLog :: IO :: FileSystem + http://maven.apache.org + + UTF-8 + ${basedir}/lib + + + + org.apache.distributedlog + distributedlog-core + ${project.parent.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + com.google.protobuf + protobuf-java + + + com.google.guava + guava + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + org.apache.distributedlog + distributedlog-core + ${project.parent.version} + tests + test + + + + org.xerial.snappy + snappy-java + test + + + + io.dropwizard.metrics + metrics-core + test + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + + diff --git a/stream/distributedlog/io/pom.xml b/stream/distributedlog/io/pom.xml new file mode 100644 index 00000000000..38572cb2e0b --- /dev/null +++ b/stream/distributedlog/io/pom.xml @@ -0,0 +1,31 @@ + + + + + org.apache.distributedlog + distributedlog + 4.15.0-SNAPSHOT + + 4.0.0 + distributedlog-io + pom + Apache BookKeeper :: DistributedLog :: IO + + dlfs + + diff --git a/stream/distributedlog/pom.xml b/stream/distributedlog/pom.xml new file mode 100644 index 00000000000..2f77d0fc025 --- /dev/null +++ b/stream/distributedlog/pom.xml @@ -0,0 +1,91 @@ + + + + 4.0.0 + + org.apache.bookkeeper + bookkeeper + 4.15.0-SNAPSHOT + ../.. + + org.apache.distributedlog + distributedlog + pom + Apache BookKeeper :: DistributedLog :: Parent + + Apache DistributedLog provides a high performance replicated log service. + + 2016 + + common + protocol + core + io + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + + -notimestamp + none + + + Core Library + org.apache.distributedlog:org.apache.distributedlog.annotations:org.apache.distributedlog.callback:org.apache.distributedlog.exceptions:org.apache.distributedlog.feature:org.apache.distributedlog.io:org.apache.distributedlog.lock:org.apache.distributedlog.logsegment:org.apache.distributedlog.metadata:org.apache.distributedlog.namespace:org.apache.distributedlog.net:org.apache.distributedlog.stats:org.apache.distributedlog.api.subscription + + + + org.apache.distributedlog.acl:org.apache.distributedlog.admin:org.apache.distributedlog.auditor:org.apache.distributedlog.basic:org.apache.distributedlog.benchmark*:org.apache.distributedlog.bk:org.apache.distributedlog.ownership:org.apache.distributedlog.proxy:org.apache.distributedlog.resolver:org.apache.distributedlog.service.*:org.apache.distributedlog.config:org.apache.distributedlog.function:org.apache.distributedlog.impl*:org.apache.distributedlog.injector:org.apache.distributedlog.kafka:org.apache.distributedlog.limiter:org.apache.distributedlog.mapreduce:org.apache.distributedlog.messaging:org.apache.distributedlog.common.rate:org.apache.distributedlog.readahead:org.apache.distributedlog.selector:org.apache.distributedlog.stats:org.apache.distributedlog.thrift*:org.apache.distributedlog.tools:org.apache.distributedlog.util:org.apache.distributedlog.zk:org.apache.bookkeeper.client:org.apache.bookkeeper.stats + + + + + aggregate + + aggregate + + site + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + ${redirectTestOutputToFile} + -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID + always + 1800 + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${session.executionRootDirectory}/buildtools/src/main/resources/distributedlog/findbugsExclude.xml + + + + + + diff --git a/stream/distributedlog/protocol/pom.xml b/stream/distributedlog/protocol/pom.xml new file mode 100644 index 00000000000..727a8009c21 --- /dev/null +++ b/stream/distributedlog/protocol/pom.xml @@ -0,0 +1,58 @@ + + + + 4.0.0 + + org.apache.distributedlog + distributedlog + 4.15.0-SNAPSHOT + + distributedlog-protocol + Apache BookKeeper :: DistributedLog :: Protocol + + + org.apache.distributedlog + distributedlog-common + ${project.version} + + + io.netty + netty-buffer + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + test-jar + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + + diff --git a/stream/pom.xml b/stream/pom.xml new file mode 100644 index 00000000000..d1b01cec7e0 --- /dev/null +++ b/stream/pom.xml @@ -0,0 +1,92 @@ + + + + 4.0.0 + + org.apache.bookkeeper + bookkeeper + 4.15.0-SNAPSHOT + .. + + pom + org.apache.bookkeeper + stream-storage-parent + Apache BookKeeper :: Stream Storage :: Parent + + + distributedlog + common + tests-common + statelib + api + proto + clients + storage + server + bk-grpc-name-resolver + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + true + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + true + ${redirectTestOutputToFile} + -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID + always + 1800 + + + + + + + + streamTests + + + streamTests + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + + + + + + + + diff --git a/stream/proto/pom.xml b/stream/proto/pom.xml new file mode 100644 index 00000000000..7827096cb49 --- /dev/null +++ b/stream/proto/pom.xml @@ -0,0 +1,115 @@ + + + + 4.0.0 + + org.apache.bookkeeper + stream-storage-parent + 4.15.0-SNAPSHOT + .. + + org.apache.bookkeeper + stream-storage-proto + Apache BookKeeper :: Stream Storage :: Proto + + + + org.apache.bookkeeper + stream-storage-common + ${project.parent.version} + + + org.apache.commons + commons-lang3 + + + com.google.protobuf + protobuf-java + + + javax.annotation + javax.annotation-api + + true + + + org.apache.bookkeeper.tests + stream-storage-tests-common + ${project.version} + test + + + + + + + kr.motd.maven + os-maven-plugin + ${os-maven-plugin.version} + + + + + maven-compiler-plugin + ${maven-compiler-plugin.version} + + ${javac.target} + ${javac.target} + + -Xlint:unchecked + + -Xpkginfo:always + + false + false + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} + true + + + + + compile + compile-custom + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + test-jar + + + + + + + diff --git a/stream/server/build.gradle b/stream/server/build.gradle index d8cba32767f..19faa189683 100644 --- a/stream/server/build.gradle +++ b/stream/server/build.gradle @@ -74,8 +74,12 @@ application { mainClassName = "org.apache.bookkeeper.stream.cluster.StandaloneStarter" } +task writeClasspath { + buildDir.mkdirs() + new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" +} + jar { - dependsOn tasks.named("writeClasspath") archiveBaseName = 'stream-storage-server' } diff --git a/stream/server/pom.xml b/stream/server/pom.xml new file mode 100644 index 00000000000..386b611be91 --- /dev/null +++ b/stream/server/pom.xml @@ -0,0 +1,79 @@ + + + + 4.0.0 + + org.apache.bookkeeper + stream-storage-parent + 4.15.0-SNAPSHOT + + stream-storage-server + Apache BookKeeper :: Stream Storage :: Server + + + + org.apache.bookkeeper + stream-storage-service-impl + ${project.parent.version} + + + org.apache.bookkeeper + stream-storage-java-client + ${project.parent.version} + + + com.beust + jcommander + + + org.apache.bookkeeper.http + vertx-http-server + ${project.parent.version} + runtime + + + + org.xerial.snappy + snappy-java + runtime + + + + io.dropwizard.metrics + metrics-core + runtime + + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + test-jar + + + + + + + diff --git a/stream/statelib/pom.xml b/stream/statelib/pom.xml new file mode 100644 index 00000000000..d2466182dad --- /dev/null +++ b/stream/statelib/pom.xml @@ -0,0 +1,169 @@ + + + + 4.0.0 + + + stream-storage-parent + org.apache.bookkeeper + 4.15.0-SNAPSHOT + .. + + org.apache.bookkeeper + statelib + Apache BookKeeper :: Stream Storage :: State Library + + + org.apache.distributedlog + distributedlog-core + ${project.parent.version} + + + org.apache.bookkeeper + bookkeeper-common + ${project.parent.version} + + + org.apache.bookkeeper + stream-storage-common + ${project.parent.version} + + + org.apache.bookkeeper + stream-storage-api + ${project.parent.version} + + + org.apache.bookkeeper + stream-storage-proto + ${project.parent.version} + + + com.google.guava + guava + + + org.rocksdb + rocksdbjni + + + com.google.protobuf + protobuf-java + + + org.apache.distributedlog + distributedlog-core + ${project.parent.version} + tests + + + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + net.java.dev.javacc + javacc + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-api + + + log4j + log4j + + + io.netty + * + + + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + test-jar + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-api + + + log4j + log4j + + + io.netty + * + + + + + + + org.xerial.snappy + snappy-java + ${snappy.version} + + + + + io.dropwizard.metrics + metrics-core + ${dropwizard.version} + + + + + + + kr.motd.maven + os-maven-plugin + ${os-maven-plugin.version} + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} + + + + + compile + + + + + + + diff --git a/stream/storage/api/pom.xml b/stream/storage/api/pom.xml new file mode 100644 index 00000000000..bc415e3e28b --- /dev/null +++ b/stream/storage/api/pom.xml @@ -0,0 +1,58 @@ + + + + 4.0.0 + + org.apache.bookkeeper + stream-storage-service-parent + 4.15.0-SNAPSHOT + .. + + stream-storage-service-api + Apache BookKeeper :: Stream Storage :: Storage :: Api + + + + org.apache.bookkeeper + bookkeeper-common + ${project.parent.version} + + + org.apache.bookkeeper + stream-storage-proto + ${project.parent.version} + + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + test-jar + + + + + + + diff --git a/stream/storage/impl/pom.xml b/stream/storage/impl/pom.xml new file mode 100644 index 00000000000..7b2ff7d5672 --- /dev/null +++ b/stream/storage/impl/pom.xml @@ -0,0 +1,160 @@ + + + + 4.0.0 + + org.apache.bookkeeper + stream-storage-service-parent + 4.15.0-SNAPSHOT + .. + + stream-storage-service-impl + Apache BookKeeper :: Stream Storage :: Storage :: Impl + + + + org.apache.bookkeeper + stream-storage-service-api + ${project.parent.version} + + + org.apache.bookkeeper + stream-storage-java-client-base + ${project.parent.version} + + + org.apache.bookkeeper + statelib + ${project.parent.version} + + + org.apache.curator + curator-recipes + + + org.apache.distributedlog + distributedlog-core + ${project.parent.version} + tests + test + + + org.apache.bookkeeper + bookkeeper-common + ${project.parent.version} + tests + test + + + org.apache.bookkeeper.tests + stream-storage-tests-common + ${project.version} + + + org.apache.bookkeeper + stream-storage-java-client-base + ${project.parent.version} + test-jar + test + + + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + net.java.dev.javacc + javacc + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-api + + + log4j + log4j + + + io.netty + * + + + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + test-jar + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-api + + + log4j + log4j + + + io.netty + * + + + + + + + org.xerial.snappy + snappy-java + ${snappy.version} + + + + + io.dropwizard.metrics + metrics-core + ${dropwizard.version} + + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + test-jar + + + + + + + diff --git a/stream/storage/pom.xml b/stream/storage/pom.xml new file mode 100644 index 00000000000..22e3eaf395d --- /dev/null +++ b/stream/storage/pom.xml @@ -0,0 +1,33 @@ + + + + pom + 4.0.0 + + org.apache.bookkeeper + stream-storage-parent + 4.15.0-SNAPSHOT + .. + + stream-storage-service-parent + Apache BookKeeper :: Stream Storage :: Storage :: Parent + + api + impl + + diff --git a/stream/tests-common/pom.xml b/stream/tests-common/pom.xml new file mode 100644 index 00000000000..c14fde018ae --- /dev/null +++ b/stream/tests-common/pom.xml @@ -0,0 +1,118 @@ + + + + 4.0.0 + + org.apache.bookkeeper + stream-storage-parent + 4.15.0-SNAPSHOT + .. + + org.apache.bookkeeper.tests + stream-storage-tests-common + Apache BookKeeper :: Stream Storage :: Common Classes for Tests + + + + io.grpc + grpc-all + + + io.grpc + grpc-netty-shaded + + + org.bouncycastle + bcpkix-jdk15on + + + io.grpc + grpc-okhttp + + + + + com.google.protobuf + protobuf-java + + + javax.annotation + javax.annotation-api + + true + + + + + + + kr.motd.maven + os-maven-plugin + ${os-maven-plugin.version} + + + + + maven-compiler-plugin + ${maven-compiler-plugin.version} + + ${javac.target} + ${javac.target} + + -Xlint:unchecked + + -Xpkginfo:always + + false + false + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} + true + + + + + compile + compile-custom + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + test-jar + + + + + + + diff --git a/tests/backward-compat/bc-non-fips/pom.xml b/tests/backward-compat/bc-non-fips/pom.xml new file mode 100644 index 00000000000..eba5b1d7219 --- /dev/null +++ b/tests/backward-compat/bc-non-fips/pom.xml @@ -0,0 +1,79 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + backward-compat + 4.15.0-SNAPSHOT + .. + + + org.apache.bookkeeper.tests.backward-compat + bc-non-fips + jar + Apache BookKeeper :: Tests :: Backward Compatibility :: Test Bouncy Castle Provider load non FIPS version + + 1.68 + + + + + junit + junit + ${junit.version} + + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + + + org.bouncycastle + * + + + test + + + + org.bouncycastle + bcpkix-jdk15on + ${bc-non-fips.version} + + + + org.bouncycastle + bcprov-ext-jdk15on + ${bc-non-fips.version} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/tests/backward-compat/current-server-old-clients/pom.xml b/tests/backward-compat/current-server-old-clients/pom.xml new file mode 100644 index 00000000000..2379977810d --- /dev/null +++ b/tests/backward-compat/current-server-old-clients/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + backward-compat + 4.15.0-SNAPSHOT + .. + + + org.apache.bookkeeper.tests.backward-compat + backward-compat-current-server-old-clients + jar + Apache BookKeeper :: Tests :: Backward Compatibility :: Test old clients working on current server + + diff --git a/tests/backward-compat/hierarchical-ledger-manager/pom.xml b/tests/backward-compat/hierarchical-ledger-manager/pom.xml new file mode 100644 index 00000000000..a75736839c2 --- /dev/null +++ b/tests/backward-compat/hierarchical-ledger-manager/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + backward-compat + 4.15.0-SNAPSHOT + .. + + + org.apache.bookkeeper.tests.backward-compat + hierarchical-ledger-manager + jar + Apache BookKeeper :: Tests :: Backward Compatibility :: Test compat between old version and new version of hierarchical ledger manager + + diff --git a/tests/backward-compat/hostname-bookieid/pom.xml b/tests/backward-compat/hostname-bookieid/pom.xml new file mode 100644 index 00000000000..28d64fce9c7 --- /dev/null +++ b/tests/backward-compat/hostname-bookieid/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + backward-compat + 4.15.0-SNAPSHOT + .. + + + org.apache.bookkeeper.tests.backward-compat + hostname-bookieid + jar + Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between 4.1.0 and current version (with hostname bookie ID) + + diff --git a/tests/backward-compat/old-cookie-new-cluster/pom.xml b/tests/backward-compat/old-cookie-new-cluster/pom.xml new file mode 100644 index 00000000000..c7705c999cd --- /dev/null +++ b/tests/backward-compat/old-cookie-new-cluster/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + backward-compat + 4.15.0-SNAPSHOT + .. + + + org.apache.bookkeeper.tests.backward-compat + old-cookie-new-cluster + jar + Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade 4.1.0 to current in cluster with cookies + + diff --git a/tests/backward-compat/pom.xml b/tests/backward-compat/pom.xml new file mode 100644 index 00000000000..396840bb8f4 --- /dev/null +++ b/tests/backward-compat/pom.xml @@ -0,0 +1,41 @@ + + + + pom + 4.0.0 + + org.apache.bookkeeper.tests + integration-tests-base-groovy + 4.15.0-SNAPSHOT + ../integration-tests-base-groovy + + org.apache.bookkeeper.tests + backward-compat + Apache BookKeeper :: Tests :: Backward Compatibility + + upgrade + upgrade-direct + hierarchical-ledger-manager + hostname-bookieid + recovery-no-password + old-cookie-new-cluster + current-server-old-clients + yahoo-custom-version + bc-non-fips + + diff --git a/tests/backward-compat/recovery-no-password/pom.xml b/tests/backward-compat/recovery-no-password/pom.xml new file mode 100644 index 00000000000..8eb53fc7f07 --- /dev/null +++ b/tests/backward-compat/recovery-no-password/pom.xml @@ -0,0 +1,39 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + backward-compat + 4.15.0-SNAPSHOT + .. + + + org.apache.bookkeeper.tests.backward-compat + recovery-no-password + jar + Apache BookKeeper :: Tests :: Backward Compatibility :: Test recovery does not work when password no in metadata + + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + + + diff --git a/tests/backward-compat/upgrade-direct/pom.xml b/tests/backward-compat/upgrade-direct/pom.xml new file mode 100644 index 00000000000..c03d5686c38 --- /dev/null +++ b/tests/backward-compat/upgrade-direct/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + backward-compat + 4.15.0-SNAPSHOT + .. + + + org.apache.bookkeeper.tests.backward-compat + upgrade-direct + jar + Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between 4.1.0 and current version + + diff --git a/tests/backward-compat/upgrade/pom.xml b/tests/backward-compat/upgrade/pom.xml new file mode 100644 index 00000000000..e8377df4e68 --- /dev/null +++ b/tests/backward-compat/upgrade/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + backward-compat + 4.15.0-SNAPSHOT + .. + + + org.apache.bookkeeper.tests.backward-compat + upgrade + jar + Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between all released versions and current version + + diff --git a/tests/backward-compat/yahoo-custom-version/pom.xml b/tests/backward-compat/yahoo-custom-version/pom.xml new file mode 100644 index 00000000000..6061316a0e7 --- /dev/null +++ b/tests/backward-compat/yahoo-custom-version/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + backward-compat + 4.15.0-SNAPSHOT + .. + + + org.apache.bookkeeper.tests.backward-compat + yahoo-custom-version + jar + Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between yahoo custom version and current + + diff --git a/tests/docker-images/all-released-versions-image/pom.xml b/tests/docker-images/all-released-versions-image/pom.xml new file mode 100644 index 00000000000..7abcbb03f75 --- /dev/null +++ b/tests/docker-images/all-released-versions-image/pom.xml @@ -0,0 +1,71 @@ + + + + + org.apache.bookkeeper.tests + docker-images + 4.15.0-SNAPSHOT + + 4.0.0 + org.apache.bookkeeper.tests + all-released-versions-image + Apache BookKeeper :: Tests :: Docker Images :: All Released Versions + pom + + + docker + + + integrationTests + + + + + + com.spotify + dockerfile-maven-plugin + 1.4.13 + + + default + + build + + + + add-latest-tag + + tag + + + apachebookkeeper/bookkeeper-all-released-versions + latest + + + + + apachebookkeeper/bookkeeper-all-released-versions + ${project.version} + false + + + + + + + diff --git a/tests/docker-images/all-versions-image/pom.xml b/tests/docker-images/all-versions-image/pom.xml new file mode 100644 index 00000000000..1eb92bb6d5c --- /dev/null +++ b/tests/docker-images/all-versions-image/pom.xml @@ -0,0 +1,111 @@ + + + + + org.apache.bookkeeper.tests + docker-images + 4.15.0-SNAPSHOT + + 4.0.0 + org.apache.bookkeeper.tests + all-versions-image + Apache BookKeeper :: Tests :: Docker Images :: All Versions + pom + + + org.apache.bookkeeper.tests + all-released-versions-image + ${project.parent.version} + pom + + + + org.apache.bookkeeper + bookkeeper-dist-server + ${project.parent.version} + bin + tar.gz + provided + + + + + docker + + + integrationTests + + + + + + com.spotify + dockerfile-maven-plugin + ${dockerfile-maven-plugin.version} + + + default + + build + + + + add-latest-tag + + tag + + + apachebookkeeper/bookkeeper-all-versions + latest + + + + + apachebookkeeper/bookkeeper-all-versions + ${project.version} + false + true + + target/bookkeeper-dist-server-${project.version}-bin.tar.gz + + + + + org.apache.maven.plugins + maven-dependency-plugin + ${maven-dependency-plugin.version} + + + copy-tarball + + copy-dependencies + + generate-resources + + ${project.build.directory}/ + bookkeeper-dist-server + true + + + + + + + + + diff --git a/tests/docker-images/current-version-image/pom.xml b/tests/docker-images/current-version-image/pom.xml new file mode 100644 index 00000000000..5e221904e3a --- /dev/null +++ b/tests/docker-images/current-version-image/pom.xml @@ -0,0 +1,147 @@ + + + + + org.apache.bookkeeper.tests + docker-images + 4.15.0-SNAPSHOT + + 4.0.0 + org.apache.bookkeeper.tests + current-version-image + Apache BookKeeper :: Tests :: Docker Images :: Current Version + pom + + + org.apache.bookkeeper + bookkeeper-dist-server + ${project.parent.version} + bin + tar.gz + provided + + + + + docker + + + integrationTests + + + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + build-python-client + generate-resources + + exec + + + ${project.basedir}/target + ${project.basedir}/../../../stream/clients/python/scripts/docker_build.sh + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven-antrun-plugin.version} + + + generate-resources + + run + + + + copy python wheel file + + + copying docker scripts + + + + + + + + + com.spotify + dockerfile-maven-plugin + ${dockerfile-maven-plugin.version} + + + default + + build + + + + add-latest-tag + + tag + + + apachebookkeeper/bookkeeper-current + latest + + + + + apachebookkeeper/bookkeeper-current + ${project.version} + false + true + + ${project.version} + + + + + org.apache.maven.plugins + maven-dependency-plugin + ${maven-dependency-plugin.version} + + + copy-docker-dependencies + + copy-dependencies + + generate-resources + + ${project.build.directory}/ + bookkeeper-dist-server + true + + + + + + + + + diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml new file mode 100644 index 00000000000..a337e91176a --- /dev/null +++ b/tests/docker-images/pom.xml @@ -0,0 +1,34 @@ + + + + pom + 4.0.0 + + org.apache.bookkeeper.tests + tests-parent + 4.15.0-SNAPSHOT + + org.apache.bookkeeper.tests + docker-images + Apache BookKeeper :: Tests :: Docker Images + + all-released-versions-image + all-versions-image + current-version-image + + diff --git a/tests/integration-tests-base-groovy/pom.xml b/tests/integration-tests-base-groovy/pom.xml new file mode 100644 index 00000000000..ccaa5ea4308 --- /dev/null +++ b/tests/integration-tests-base-groovy/pom.xml @@ -0,0 +1,118 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + integration-tests-base + 4.15.0-SNAPSHOT + ../integration-tests-base + + + org.apache.bookkeeper.tests + integration-tests-base-groovy + pom + + Apache BookKeeper :: Tests :: Base module for Arquillian based integration tests using groovy + + + 3.6.0-03 + 3.0.2-02 + + + + + + maven-compiler-plugin + + + ${javac.target} + ${javac.target} + groovy-eclipse-compiler + + + + org.codehaus.groovy + groovy-eclipse-compiler + ${groovy-eclipse-compiler.version} + + + org.codehaus.groovy + groovy-eclipse-batch + ${groovy-eclipse-batch.version} + + + + + org.codehaus.groovy + groovy-eclipse-compiler + ${groovy-eclipse-compiler.version} + true + + + org.codehaus.gmaven + groovy-maven-plugin + 2.0 + + + org.codehaus.groovy + groovy-all + ${groovy.version} + pom + + + + + org.apache.maven.plugins + maven-surefire-plugin + + 2.8.1 + + -Xmx4G -Djava.net.preferIPv4Stack=true + 1 + false + + + System.out + true + + + + + + + + + org.codehaus.groovy + groovy-all + pom + + + + + bintray + + false + + groovy-bintray-plugins + https://dl.bintray.com/groovy/maven + + + diff --git a/tests/integration-tests-base/pom.xml b/tests/integration-tests-base/pom.xml new file mode 100644 index 00000000000..b5a0ce218d7 --- /dev/null +++ b/tests/integration-tests-base/pom.xml @@ -0,0 +1,98 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + tests-parent + 4.15.0-SNAPSHOT + + + org.apache.bookkeeper.tests + integration-tests-base + pom + + Apache BookKeeper :: Tests :: Base module for Arquillian based integration tests + + + + + org.apache.bookkeeper.tests + integration-tests-utils + ${project.version} + + + + org.apache.bookkeeper.tests + integration-tests-topologies + ${project.version} + + + + org.jboss.arquillian.junit + arquillian-junit-standalone + test + + + + junit + junit + test + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + + integrationTests + + + integrationTests + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + + + + + + diff --git a/tests/integration-tests-topologies/pom.xml b/tests/integration-tests-topologies/pom.xml new file mode 100644 index 00000000000..b2797b09635 --- /dev/null +++ b/tests/integration-tests-topologies/pom.xml @@ -0,0 +1,49 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + tests-parent + 4.15.0-SNAPSHOT + + + org.apache.bookkeeper.tests + integration-tests-topologies + jar + + Apache BookKeeper :: Tests :: Common topologies for Docker based integration tests + + + + org.testcontainers + testcontainers + + + junit + junit + compile + + + org.apache.bookkeeper.tests + integration-tests-utils + ${project.version} + + + + diff --git a/tests/integration-tests-utils/pom.xml b/tests/integration-tests-utils/pom.xml new file mode 100644 index 00000000000..6c43bbc1d41 --- /dev/null +++ b/tests/integration-tests-utils/pom.xml @@ -0,0 +1,93 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + tests-parent + 4.15.0-SNAPSHOT + + + org.apache.bookkeeper.tests + integration-tests-utils + jar + + Apache BookKeeper :: Tests :: Utility module for Arquillian based integration tests + + + + org.apache.commons + commons-compress + + + + org.jboss.shrinkwrap.resolver + shrinkwrap-resolver-impl-maven + + + org.jboss.shrinkwrap.resolver + shrinkwrap-resolver-api + + + + org.apache.zookeeper + zookeeper + + + + org.arquillian.cube + arquillian-cube-docker + + + com.github.docker-java + * + + + + + + org.testcontainers + testcontainers + + + + org.codehaus.groovy + groovy-all + pom + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + 2.8.1 + + 1 + false + + + + + + diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java index a29331131fe..55f625c8bcb 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java @@ -48,6 +48,10 @@ public class BookKeeperClusterUtils { Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.4"); + private static final List OLD_CLIENT_VERSIONS_WITH_OLD_BK_BIN_NAME = + Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.3", "4.3-yahoo"); + + private static final Logger LOG = LoggerFactory.getLogger(BookKeeperClusterUtils.class); public static boolean hasVersionLatestMetadataFormat(String version) { @@ -98,7 +102,7 @@ public static boolean metadataFormatIfNeeded(DockerClient docker, String version @Cleanup ZooKeeper zk = BookKeeperClusterUtils.zookeeperClient(docker); if (zk.exists("/ledgers", false) == null) { - String bookkeeper = "/opt/bookkeeper/" + version + "/bin/bookkeeper"; + String bookkeeper = "/opt/bookkeeper/" + version + "/bin/" + computeBinFilenameByVersion(version); runOnAnyBookie(docker, bookkeeper, "shell", "metaformat", "-nonInteractive"); return true; } else { @@ -128,7 +132,7 @@ public static String createDlogNamespaceIfNeeded(DockerClient docker, } public static void formatAllBookies(DockerClient docker, String version) throws Exception { - String bookkeeper = "/opt/bookkeeper/" + version + "/bin/bookkeeper"; + String bookkeeper = "/opt/bookkeeper/" + version + "/bin/" + computeBinFilenameByVersion(version); BookKeeperClusterUtils.runOnAllBookies(docker, bookkeeper, "shell", "bookieformat", "-nonInteractive"); } @@ -256,4 +260,11 @@ public static boolean waitAllBookieUp(DockerClient docker) { .map((b) -> waitBookieUp(docker, b, 10, TimeUnit.SECONDS)) .reduce(true, BookKeeperClusterUtils::allTrue); } + + private static String computeBinFilenameByVersion(String version) { + if (OLD_CLIENT_VERSIONS_WITH_OLD_BK_BIN_NAME.contains(version)) { + return "bookkeeper"; + } + return "bookkeeper_gradle"; + } } diff --git a/tests/integration/cluster/pom.xml b/tests/integration/cluster/pom.xml new file mode 100644 index 00000000000..604aeab3e1f --- /dev/null +++ b/tests/integration/cluster/pom.xml @@ -0,0 +1,79 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + integration + 4.15.0-SNAPSHOT + .. + + + org.apache.bookkeeper.tests.integration + cluster + jar + Apache BookKeeper :: Tests :: Integration :: Cluster test + + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + test + + + + org.apache.bookkeeper + stream-storage-server + ${project.version} + test + + + + org.apache.bookkeeper.tests + integration-tests-topologies + ${project.version} + test + + + + org.apache.bookkeeper + bookkeeper-common + ${project.version} + test-jar + test + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + 0 + ${redirectTestOutputToFile} + + + + + diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml new file mode 100644 index 00000000000..1d7535f59bd --- /dev/null +++ b/tests/integration/pom.xml @@ -0,0 +1,73 @@ + + + + pom + 4.0.0 + + org.apache.bookkeeper.tests + tests-parent + 4.15.0-SNAPSHOT + + org.apache.bookkeeper.tests + integration + Apache BookKeeper :: Tests :: Integration + + smoke + standalone + cluster + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + + integrationTests + + + integrationTests + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + + + + + + diff --git a/tests/integration/smoke/pom.xml b/tests/integration/smoke/pom.xml new file mode 100644 index 00000000000..35cc17511b8 --- /dev/null +++ b/tests/integration/smoke/pom.xml @@ -0,0 +1,75 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + integration + 4.15.0-SNAPSHOT + .. + + + org.apache.bookkeeper.tests.integration + smoke + jar + Apache BookKeeper :: Tests :: Integration :: Smoke test + + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + test + + + + org.apache.bookkeeper.tests + integration-tests-utils + ${project.version} + test + + + + org.apache.bookkeeper.tests + integration-tests-topologies + ${project.version} + test + + + + org.jboss.arquillian.junit + arquillian-junit-standalone + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + 0 + ${redirectTestOutputToFile} + + + + + diff --git a/tests/integration/standalone/pom.xml b/tests/integration/standalone/pom.xml new file mode 100644 index 00000000000..7a9cb5a4272 --- /dev/null +++ b/tests/integration/standalone/pom.xml @@ -0,0 +1,64 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + integration + 4.15.0-SNAPSHOT + .. + + + org.apache.bookkeeper.tests.integration + standalone + jar + Apache BookKeeper :: Tests :: Integration :: Standalone test + + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + test + + + + org.apache.bookkeeper.tests + integration-tests-topologies + ${project.version} + test + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + 0 + ${redirectTestOutputToFile} + + + + + diff --git a/tests/pom.xml b/tests/pom.xml new file mode 100644 index 00000000000..ae4f4b8ab3c --- /dev/null +++ b/tests/pom.xml @@ -0,0 +1,57 @@ + + + + pom + 4.0.0 + + org.apache.bookkeeper + bookkeeper + 4.15.0-SNAPSHOT + + org.apache.bookkeeper.tests + tests-parent + Apache BookKeeper :: Tests + + + 2.5.8 + + + + shaded + docker-images + integration-tests-base + integration-tests-base-groovy + integration-tests-utils + integration-tests-topologies + backward-compat + integration + scripts + + + + + org.apache.maven.plugins + maven-deploy-plugin + ${maven-deploy-plugin.version} + + true + + + + + diff --git a/tests/scripts/pom.xml b/tests/scripts/pom.xml new file mode 100644 index 00000000000..4bbc9e80021 --- /dev/null +++ b/tests/scripts/pom.xml @@ -0,0 +1,75 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + tests-parent + 4.15.0-SNAPSHOT + + + org.apache.bookkeeper.tests + scripts + jar + Apache BookKeeper :: Tests :: Bash Scripts Test + + + + + + + + com.googlecode.maven-download-plugin + download-maven-plugin + + + install-shunit2 + integration-test + + wget + + + https://github.com/kward/shunit2/archive/v2.1.7.zip + true + ${project.basedir}/target/lib + ${skipTests} + + + + + + org.codehaus.mojo + exec-maven-plugin + + + bash-tests + integration-test + + exec + + + ${skipTests} + ${project.basedir}/src/test/bash + ${project.basedir}/src/test/bash/bk_test.sh + + + + + + + diff --git a/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh b/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh index 9a1f7f3917e..c20165f6050 100644 --- a/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh +++ b/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh @@ -27,7 +27,7 @@ # testDefaultVariables() { - source ${BK_BINDIR}/common.sh + source ${BK_BINDIR}/common_gradle.sh assertEquals "BINDIR is not set correctly" "${BK_BINDIR}" "${BINDIR}" assertEquals "BK_HOME is not set correctly" "${BK_HOMEDIR}" "${BK_HOME}" assertEquals "DEFAULT_LOG_CONF is not set correctly" "${BK_CONFDIR}/log4j.properties" "${DEFAULT_LOG_CONF}" @@ -48,7 +48,7 @@ testDefaultVariables() { } testFindModuleJarAt() { - source ${BK_BINDIR}/common.sh + source ${BK_BINDIR}/common_gradle.sh MODULE="test-module" @@ -113,7 +113,7 @@ testFindModuleJar() { echo "" > ${BK_HOME}/conf/bkenv.sh echo "" > ${BK_HOME}/conf/bk_cli_env.sh - source ${BK_BINDIR}/common.sh + source ${BK_BINDIR}/common_gradle.sh MODULE="test-module" MODULE_PATH="testmodule" @@ -159,7 +159,7 @@ testLoadEnvfiles() { echo "CLI_MAX_HEAP_MEMORY=2048M" > ${BK_HOME}/conf/bk_cli_env.sh # load the common_gradle.sh - source ${BK_BINDIR}/common.sh + source ${BK_BINDIR}/common_gradle.sh assertEquals "NETTY_LEAK_DETECTION_LEVEL is not set correctly" "enabled" "${NETTY_LEAK_DETECTION_LEVEL}" assertEquals "BOOKIE_MAX_HEAP_MEMORY is not set correctly" "2048M" "${BOOKIE_MAX_HEAP_MEMORY}" @@ -172,7 +172,7 @@ testLoadEnvfiles() { } testBuildBookieJVMOpts() { - source ${BK_BINDIR}/common.sh + source ${BK_BINDIR}/common_gradle.sh TEST_LOG_DIR=${BK_TMPDIR}/logdir TEST_GC_LOG_FILENAME="test-gc.log" @@ -187,7 +187,7 @@ testBuildBookieJVMOpts() { } testBuildCLIJVMOpts() { - source ${BK_BINDIR}/common.sh + source ${BK_BINDIR}/common_gradle.sh TEST_LOG_DIR=${BK_TMPDIR}/logdir TEST_GC_LOG_FILENAME="test-gc.log" @@ -202,7 +202,7 @@ testBuildCLIJVMOpts() { } testBuildNettyOpts() { - source ${BK_BINDIR}/common.sh + source ${BK_BINDIR}/common_gradle.sh ACTUAL_NETTY_OPTS=$(build_netty_opts) EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled \ @@ -213,7 +213,7 @@ testBuildNettyOpts() { } testBuildBookieOpts() { - source ${BK_BINDIR}/common.sh + source ${BK_BINDIR}/common_gradle.sh ACTUAL_OPTS=$(build_bookie_opts) EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true" diff --git a/tests/shaded/bookkeeper-server-shaded-test/pom.xml b/tests/shaded/bookkeeper-server-shaded-test/pom.xml new file mode 100644 index 00000000000..cc548bc2c6f --- /dev/null +++ b/tests/shaded/bookkeeper-server-shaded-test/pom.xml @@ -0,0 +1,62 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests.shaded + shaded-tests-parent + 4.15.0-SNAPSHOT + .. + + bookkeeper-server-shaded-test + Apache BookKeeper :: Tests :: bookkeeper-server-shaded test + + + org.apache.bookkeeper + bookkeeper-server-shaded + ${project.version} + test + + + + com.google.protobuf + protobuf-java + + + com.google.guava + guava + + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + maven-compiler-plugin + ${maven-compiler-plugin.version} + + + + diff --git a/tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml b/tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml new file mode 100644 index 00000000000..2a8cb5490c3 --- /dev/null +++ b/tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml @@ -0,0 +1,78 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests.shaded + shaded-tests-parent + 4.15.0-SNAPSHOT + .. + + bookkeeper-server-tests-shaded-test + Apache BookKeeper :: Tests :: bookkeeper-server-tests-shaded test + + + org.apache.bookkeeper + bookkeeper-server-shaded + ${project.version} + test + + + + com.google.protobuf + protobuf-java + + + com.google.guava + guava + + + + + org.apache.bookkeeper + bookkeeper-server-tests-shaded + ${project.version} + test + + + + com.google.protobuf + protobuf-java + + + com.google.guava + guava + + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + + diff --git a/tests/shaded/distributedlog-core-shaded-test/pom.xml b/tests/shaded/distributedlog-core-shaded-test/pom.xml new file mode 100644 index 00000000000..7b2206c4ccf --- /dev/null +++ b/tests/shaded/distributedlog-core-shaded-test/pom.xml @@ -0,0 +1,82 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests.shaded + shaded-tests-parent + 4.15.0-SNAPSHOT + .. + + distributedlog-core-shaded-test + Apache BookKeeper :: Tests :: distributedlog-core-shaded test + + + org.apache.distributedlog + distributedlog-core-shaded + ${project.version} + test + + + + org.apache.bookkeeper + bookkeeper-server + + + org.apache.bookkeeper.http + bookkeeper-http + + + org.apache.bookkeeper + circe-checksum + + + org.apache.distributedlog + distributedlog-core + + + org.apache.distributedlog + distributedlog-common + + + org.apache.distributedlog + distributedlog-protocol + + + org.apache.zookeeper + zookeeper + + + org.apache.thrift + libthrift + + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + + diff --git a/tests/shaded/pom.xml b/tests/shaded/pom.xml new file mode 100644 index 00000000000..f86a357115b --- /dev/null +++ b/tests/shaded/pom.xml @@ -0,0 +1,35 @@ + + + + pom + 4.0.0 + + org.apache.bookkeeper.tests + tests-parent + 4.15.0-SNAPSHOT + .. + + org.apache.bookkeeper.tests.shaded + shaded-tests-parent + Apache BookKeeper :: Tests :: Test Shaded Jars + + bookkeeper-server-shaded-test + bookkeeper-server-tests-shaded-test + distributedlog-core-shaded-test + + diff --git a/tools/all/build.gradle b/tools/all/build.gradle index 670f2c41a72..7593b6a2c6e 100644 --- a/tools/all/build.gradle +++ b/tools/all/build.gradle @@ -36,6 +36,5 @@ dependencies { } jar { - dependsOn tasks.named("writeClasspath") archiveBaseName = 'bookkeeper-tools' } diff --git a/tools/all/pom.xml b/tools/all/pom.xml new file mode 100644 index 00000000000..ac15e6bb12a --- /dev/null +++ b/tools/all/pom.xml @@ -0,0 +1,89 @@ + + + + 4.0.0 + + bookkeeper-tools-parent + org.apache.bookkeeper + 4.15.0-SNAPSHOT + + bookkeeper-tools + Apache BookKeeper :: Tools + + + org.apache.bookkeeper + bookkeeper-tools-ledger + ${project.version} + + + org.apache.bookkeeper + stream-storage-cli + ${project.version} + + + + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven-antrun-plugin.version} + + + append-ledger-commands + generate-resources + + run + + + + + + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven-antrun-plugin.version} + + + append-stream-commands + generate-resources + + run + + + + + + + + + + + + + + + diff --git a/tools/framework/pom.xml b/tools/framework/pom.xml new file mode 100644 index 00000000000..9f5e86b1ef6 --- /dev/null +++ b/tools/framework/pom.xml @@ -0,0 +1,44 @@ + + + + 4.0.0 + + bookkeeper-tools-parent + org.apache.bookkeeper + 4.15.0-SNAPSHOT + + bookkeeper-tools-framework + Apache BookKeeper :: Tools :: Framework + + + com.beust + jcommander + + + org.apache.bookkeeper + bookkeeper-common + ${project.version} + + + org.apache.bookkeeper + buildtools + ${project.parent.version} + test + + + diff --git a/tools/ledger/pom.xml b/tools/ledger/pom.xml new file mode 100644 index 00000000000..8becea1f281 --- /dev/null +++ b/tools/ledger/pom.xml @@ -0,0 +1,67 @@ + + + + 4.0.0 + + bookkeeper-tools-parent + org.apache.bookkeeper + 4.15.0-SNAPSHOT + + bookkeeper-tools-ledger + Apache BookKeeper :: Tools :: Ledger + + + org.apache.bookkeeper + bookkeeper-tools-framework + ${project.version} + + + org.apache.bookkeeper + bookkeeper-server + ${project.parent.version} + + + org.apache.logging.log4j + log4j-1.2-api + runtime + + + org.apache.logging.log4j + log4j-core + runtime + + + org.apache.logging.log4j + log4j-slf4j-impl + runtime + + + org.apache.bookkeeper + buildtools + ${project.parent.version} + test + + + org.apache.bookkeeper + bookkeeper-server + test-jar + ${project.parent.version} + test + + + diff --git a/tools/perf/build.gradle b/tools/perf/build.gradle index 80df1221ae9..1223e2000d0 100644 --- a/tools/perf/build.gradle +++ b/tools/perf/build.gradle @@ -67,6 +67,5 @@ dependencies { } jar { - dependsOn tasks.named("writeClasspath") archiveBaseName = 'bookkeeper-tools-perf' } diff --git a/tools/perf/pom.xml b/tools/perf/pom.xml new file mode 100644 index 00000000000..a0c39e39687 --- /dev/null +++ b/tools/perf/pom.xml @@ -0,0 +1,52 @@ + + + + 4.0.0 + + org.apache.bookkeeper + bookkeeper-tools-parent + 4.15.0-SNAPSHOT + + bookkeeper-perf + Apache BookKeeper :: Tools :: Perf + + + + org.apache.bookkeeper + bookkeeper-tools-framework + ${project.version} + + + org.apache.distributedlog + distributedlog-core + ${project.version} + + + org.apache.bookkeeper + stream-storage-java-client + ${project.version} + + + org.apache.bookkeeper.stats + prometheus-metrics-provider + ${project.version} + + + org.hdrhistogram + HdrHistogram + + + + diff --git a/tools/pom.xml b/tools/pom.xml new file mode 100644 index 00000000000..50010001879 --- /dev/null +++ b/tools/pom.xml @@ -0,0 +1,35 @@ + + + + 4.0.0 + + bookkeeper + org.apache.bookkeeper + 4.15.0-SNAPSHOT + + bookkeeper-tools-parent + Apache BookKeeper :: Tools :: Parent + pom + + framework + ledger + stream + perf + all + + diff --git a/tools/stream/pom.xml b/tools/stream/pom.xml new file mode 100644 index 00000000000..ba9eea73590 --- /dev/null +++ b/tools/stream/pom.xml @@ -0,0 +1,52 @@ + + + + 4.0.0 + + org.apache.bookkeeper + bookkeeper-tools-parent + 4.15.0-SNAPSHOT + + stream-storage-cli + Apache BookKeeper :: Tools :: Stream + + + + org.apache.bookkeeper + stream-storage-java-client + ${project.version} + + + org.apache.bookkeeper + stream-storage-service-impl + ${project.version} + + + org.apache.bookkeeper + bookkeeper-tools-framework + ${project.version} + + + org.apache.bookkeeper + bookkeeper-server + ${project.version} + + + + From 235fd3a781791a6808b97a8dd9f32b07c9698e09 Mon Sep 17 00:00:00 2001 From: lushiji Date: Fri, 14 Jan 2022 15:18:37 +0800 Subject: [PATCH 02/14] rxjava: add maven dependency (cherry picked from commit ac73541ce79953141a08c60642cd39c7984ade1e) --- bookkeeper-common/pom.xml | 5 +++++ bookkeeper-server/pom.xml | 5 +++++ pom.xml | 9 ++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/bookkeeper-common/pom.xml b/bookkeeper-common/pom.xml index e7709f4597a..6a2806b9824 100644 --- a/bookkeeper-common/pom.xml +++ b/bookkeeper-common/pom.xml @@ -69,6 +69,11 @@ error_prone_annotations provided + + + io.reactivex.rxjava3 + rxjava + org.apache.commons commons-lang3 diff --git a/bookkeeper-server/pom.xml b/bookkeeper-server/pom.xml index 469aab161d9..3fdf47c52c7 100644 --- a/bookkeeper-server/pom.xml +++ b/bookkeeper-server/pom.xml @@ -130,6 +130,11 @@ org.apache.httpcomponents httpclient + + + io.reactivex.rxjava3 + rxjava + org.apache.bookkeeper diff --git a/pom.xml b/pom.xml index ac7658821b0..609c8c60214 100644 --- a/pom.xml +++ b/pom.xml @@ -198,6 +198,7 @@ 3.1.8 1 4.0.0 + 3.0.1 @@ -757,6 +758,13 @@ jmh-generator-annprocess ${jmh.version} + + + + io.reactivex.rxjava3 + rxjava + ${rxjava.version} + @@ -783,7 +791,6 @@ commons-configuration commons-configuration - junit From 408c4690ff3d177a225944372d24bab0a84b13f7 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 21 Mar 2022 20:04:30 -0700 Subject: [PATCH 03/14] Bring guava to the same version as gradle --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 609c8c60214..6ff775dd656 100644 --- a/pom.xml +++ b/pom.xml @@ -131,7 +131,7 @@ 3.0.2 2.4.0 1.42.1 - 30.0-jre + 31.0.1-jre 1.1.1 2.10.0 1.3 From dfc54541042b35a6b77a9024fe458fa45cea7c3a Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 21 Mar 2022 20:05:04 -0700 Subject: [PATCH 04/14] ignore deprecation warnings in tests --- .../apache/bookkeeper/bookie/datainteg/CookieValidationTest.java | 1 + .../bookkeeper/bookie/datainteg/DataIntegrityCheckTest.java | 1 + .../org/apache/bookkeeper/bookie/datainteg/EntryCopierTest.java | 1 + .../bookkeeper/util/collections/ConcurrentLongHashSetTest.java | 1 + .../util/collections/ConcurrentLongLongHashMapTest.java | 1 + .../util/collections/ConcurrentLongLongPairHashMapTest.java | 1 + 6 files changed, 6 insertions(+) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/CookieValidationTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/CookieValidationTest.java index ea8cf81a911..a424d00bb63 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/CookieValidationTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/CookieValidationTest.java @@ -60,6 +60,7 @@ /** * Test the DataIntegrityCookieValidation implementation of CookieValidation. */ +@SuppressWarnings("deprecation") public class CookieValidationTest { private static Logger log = LoggerFactory.getLogger(CookieValidationTest.class); final TmpDirs tmpDirs = new TmpDirs(); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/DataIntegrityCheckTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/DataIntegrityCheckTest.java index eb58e335467..d5e240d57fa 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/DataIntegrityCheckTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/DataIntegrityCheckTest.java @@ -84,6 +84,7 @@ /** * Test of DataIntegrityCheckImpl. */ +@SuppressWarnings("deprecation") public class DataIntegrityCheckTest { private static final byte[] PASSWD = new byte[0]; diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/EntryCopierTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/EntryCopierTest.java index 0d7d3d8a451..f4b851d5240 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/EntryCopierTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/EntryCopierTest.java @@ -69,6 +69,7 @@ /** * Tests for EntryCopierImpl. */ +@SuppressWarnings("deprecation") public class EntryCopierTest { private static final Logger log = LoggerFactory.getLogger(EntryCopierTest.class); private static final BookieId bookie1 = BookieId.parse("bookie1:3181"); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/collections/ConcurrentLongHashSetTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/collections/ConcurrentLongHashSetTest.java index a1f587cb882..402e1a3390f 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/collections/ConcurrentLongHashSetTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/collections/ConcurrentLongHashSetTest.java @@ -41,6 +41,7 @@ /** * Test the ConcurrentLongHashSet class. */ +@SuppressWarnings("deprecation") public class ConcurrentLongHashSetTest { @Test diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/collections/ConcurrentLongLongHashMapTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/collections/ConcurrentLongLongHashMapTest.java index 055d98dbf54..1d4b4ed3aae 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/collections/ConcurrentLongLongHashMapTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/collections/ConcurrentLongLongHashMapTest.java @@ -44,6 +44,7 @@ /** * Test the ConcurrentLongLongHashMap class. */ +@SuppressWarnings("deprecation") public class ConcurrentLongLongHashMapTest { @Test diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/collections/ConcurrentLongLongPairHashMapTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/collections/ConcurrentLongLongPairHashMapTest.java index fc63aed5dd3..bdfe39c15a5 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/collections/ConcurrentLongLongPairHashMapTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/collections/ConcurrentLongLongPairHashMapTest.java @@ -43,6 +43,7 @@ /** * Test the concurrent long-long pair hashmap class. */ +@SuppressWarnings("deprecation") public class ConcurrentLongLongPairHashMapTest { @Test From ba5a857183fec6f3cfe925c06273344cb1e26e33 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 21 Mar 2022 20:18:28 -0700 Subject: [PATCH 05/14] mockito-inline, as in gradle + suppress warnings --- pom.xml | 7 ++++++- tools/ledger/pom.xml | 6 ++++++ .../cli/commands/autorecovery/AutoRecoveryCommandTest.java | 1 + .../cli/commands/bookies/DecommissionCommandTest.java | 1 + .../tools/cli/commands/bookies/RecoverCommandTest.java | 1 + .../cli/commands/client/LedgerMetaDataCommandTest.java | 1 + .../tools/cli/commands/cookie/AdminCommandTest.java | 1 + .../bookkeeper/tools/cli/helpers/CommandTestBase.java | 1 + .../bookkeeper/tools/cli/helpers/MockCommandSupport.java | 1 + 9 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6ff775dd656..aa1cb6f62f9 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,7 @@ 1.18.20 2.17.1 1.3.0 - 3.0.0 + 3.12.4 4.1.72.Final 2.0.46.Final 9.1.3 @@ -688,6 +688,11 @@ mockito-core ${mockito.version} + + org.mockito + mockito-inline + ${mockito.version} + org.powermock powermock-api-mockito2 diff --git a/tools/ledger/pom.xml b/tools/ledger/pom.xml index 8becea1f281..b24227a65b2 100644 --- a/tools/ledger/pom.xml +++ b/tools/ledger/pom.xml @@ -63,5 +63,11 @@ ${project.parent.version} test + + org.mockito + mockito-inline + ${mockito.version} + test + diff --git a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/autorecovery/AutoRecoveryCommandTest.java b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/autorecovery/AutoRecoveryCommandTest.java index a0af7c0f521..dd0d3d9a3e7 100644 --- a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/autorecovery/AutoRecoveryCommandTest.java +++ b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/autorecovery/AutoRecoveryCommandTest.java @@ -35,6 +35,7 @@ /** * Unit test for {@link ToggleCommand}. */ +@SuppressWarnings("unchecked") public class AutoRecoveryCommandTest extends BookieCommandTestBase { private LedgerManagerFactory ledgerManagerFactory; diff --git a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookies/DecommissionCommandTest.java b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookies/DecommissionCommandTest.java index 73136b56e43..829198ba23b 100644 --- a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookies/DecommissionCommandTest.java +++ b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookies/DecommissionCommandTest.java @@ -42,6 +42,7 @@ /** * Unit test for {@link DecommissionCommand}. */ +@SuppressWarnings("unchecked") public class DecommissionCommandTest extends BookieCommandTestBase { private Versioned cookieVersioned; diff --git a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookies/RecoverCommandTest.java b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookies/RecoverCommandTest.java index 7dadb5f2e44..7ebad849b2a 100644 --- a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookies/RecoverCommandTest.java +++ b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookies/RecoverCommandTest.java @@ -57,6 +57,7 @@ /** * Unit test for {@link RecoverCommand}. */ +@SuppressWarnings("unchecked") public class RecoverCommandTest extends BookieCommandTestBase { private static final BookieId bookieSocketAddress = BookieId.parse("127.0.0.1:8000"); diff --git a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/client/LedgerMetaDataCommandTest.java b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/client/LedgerMetaDataCommandTest.java index 3d38caa88a1..da6e03f7de4 100644 --- a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/client/LedgerMetaDataCommandTest.java +++ b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/client/LedgerMetaDataCommandTest.java @@ -43,6 +43,7 @@ /** * Unit test for {@link LedgerMetaDataCommand}. */ +@SuppressWarnings("unchecked") public class LedgerMetaDataCommandTest extends BookieCommandTestBase { private LedgerManager ledgerManager; diff --git a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/cookie/AdminCommandTest.java b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/cookie/AdminCommandTest.java index 65469fdc922..5eb40ad21d1 100644 --- a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/cookie/AdminCommandTest.java +++ b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/cookie/AdminCommandTest.java @@ -50,6 +50,7 @@ /** * Unit test for {@link AdminCommand}. */ +@SuppressWarnings("unchecked") public class AdminCommandTest extends BookieCommandTestBase { private static final BookieId bookieSocketAddress = BookieId.parse("localhost:9000"); diff --git a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/helpers/CommandTestBase.java b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/helpers/CommandTestBase.java index 687755a95dd..8efe85aa4e0 100644 --- a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/helpers/CommandTestBase.java +++ b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/helpers/CommandTestBase.java @@ -44,6 +44,7 @@ * A test base providing an environment for run a command. */ @Slf4j +@SuppressWarnings("unchecked") public class CommandTestBase extends MockCommandSupport { @Rule diff --git a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/helpers/MockCommandSupport.java b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/helpers/MockCommandSupport.java index 47f5b984cd3..8366f1f0a6c 100644 --- a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/helpers/MockCommandSupport.java +++ b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/helpers/MockCommandSupport.java @@ -35,6 +35,7 @@ * A test base providing utility methods to mock environment to run commands test. */ @Slf4j +@SuppressWarnings("unchecked") public abstract class MockCommandSupport { private Map> miscMockedConstructions = new HashMap<>(); From 08efaac0e955fe9255ad3e49fe21489f28bd9215 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 21 Mar 2022 20:23:55 -0700 Subject: [PATCH 06/14] suppressed warning --- .../org/apache/bookkeeper/bookie/GarbageCollectorThreadTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/GarbageCollectorThreadTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/GarbageCollectorThreadTest.java index d15db12463d..f070143cb4b 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/GarbageCollectorThreadTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/GarbageCollectorThreadTest.java @@ -45,6 +45,7 @@ /** * Unit test for {@link GarbageCollectorThread}. */ +@SuppressWarnings("deprecation") public class GarbageCollectorThreadTest { private static final Logger LOG = LoggerFactory.getLogger(GarbageCollectorThreadTest.class); @InjectMocks From 25e615b89a8ebf18aaf0c2615630b543295d67c2 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 22 Mar 2022 11:21:31 -0700 Subject: [PATCH 07/14] Exclude site3/ from RAT check --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index aa1cb6f62f9..e7108f8895b 100644 --- a/pom.xml +++ b/pom.xml @@ -1014,6 +1014,7 @@ site/** site2/** + site3/** **/org/apache/distributedlog/thrift/* From 515bc7b70f199b0e8f4d18c242a20cbe321b963f Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 22 Mar 2022 11:54:57 -0700 Subject: [PATCH 08/14] CI to use (mostly) maven --- .github/workflows/bookie-tests.yml | 15 ++++- .github/workflows/client-tests.yml | 4 +- .../workflows/compatibility-check-java11.yml | 7 +-- .../workflows/compatibility-check-java8.yml | 7 +-- .github/workflows/gradle-build.yml | 55 +++++++++++++++++++ .github/workflows/integration-tests.yml | 18 +++--- .github/workflows/pr-validation.yml | 7 ++- .github/workflows/remaining-tests.yml | 5 +- .github/workflows/replication-tests.yml | 4 +- .github/workflows/stream-tests.yml | 22 ++------ .github/workflows/tls-tests.yml | 4 +- 11 files changed, 99 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/gradle-build.yml diff --git a/.github/workflows/bookie-tests.yml b/.github/workflows/bookie-tests.yml index 767e9a3c084..04214182749 100644 --- a/.github/workflows/bookie-tests.yml +++ b/.github/workflows/bookie-tests.yml @@ -31,7 +31,7 @@ on: workflow_dispatch: env: - GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 jobs: test: @@ -51,8 +51,17 @@ jobs: distribution: 'temurin' java-version: 11 - - name: Run bookie test - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" ${GRADLE_ARGS} + - name: Maven build bookkeeper-server + run: mvn -B -nsu -am -pl bookkeeper-server clean install -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + + - name: Run EntryLogTests + run: mvn -B -nsu -pl bookkeeper-server test -Dtest="org.apache.bookkeeper.bookie.TestEntryLog" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + + - name: Run InterleavedLedgerStorageTest + run: mvn -B -nsu -pl bookkeeper-server test -Dtest="org.apache.bookkeeper.bookie.TestInterleavedLederStorage" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + + - name: Run bookie tests + run: mvn -B -nsu -pl bookkeeper-server test -Dtest="org.apache.bookkeeper.bookie.*Test" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: print JVM thread dumps when cancelled if: cancelled() diff --git a/.github/workflows/client-tests.yml b/.github/workflows/client-tests.yml index 0a868fe2b75..4089e98e3de 100644 --- a/.github/workflows/client-tests.yml +++ b/.github/workflows/client-tests.yml @@ -31,7 +31,7 @@ on: workflow_dispatch: env: - GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 jobs: test: @@ -51,7 +51,7 @@ jobs: distribution: 'temurin' java-version: 11 - name: Run client tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" ${GRADLE_ARGS} + run: mvn -B -am -nsu -pl bookkeeper-server clean install test -Dtest="org.apache.bookkeeper.client.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: print JVM thread dumps when cancelled if: cancelled() diff --git a/.github/workflows/compatibility-check-java11.yml b/.github/workflows/compatibility-check-java11.yml index 101f24d23ce..3ce6a888967 100644 --- a/.github/workflows/compatibility-check-java11.yml +++ b/.github/workflows/compatibility-check-java11.yml @@ -31,7 +31,7 @@ on: workflow_dispatch: env: - GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 jobs: check: @@ -50,9 +50,8 @@ jobs: with: distribution: 'temurin' java-version: 11 - - name: Build with gradle - run: | - ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" ${GRADLE_ARGS} + - name: Build with Maven + run: mvn clean package -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: print JVM thread dumps when cancelled if: cancelled() run: ./dev/ci-tool print_thread_dumps diff --git a/.github/workflows/compatibility-check-java8.yml b/.github/workflows/compatibility-check-java8.yml index 1045ad0cf03..3011039e43d 100644 --- a/.github/workflows/compatibility-check-java8.yml +++ b/.github/workflows/compatibility-check-java8.yml @@ -31,7 +31,7 @@ on: workflow_dispatch: env: - GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 jobs: check: @@ -49,9 +49,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Build with gradle - run: | - ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" ${GRADLE_ARGS} + - name: Build with Maven + run: mvn clean package -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: print JVM thread dumps when cancelled if: cancelled() diff --git a/.github/workflows/gradle-build.yml b/.github/workflows/gradle-build.yml new file mode 100644 index 00000000000..7c9bc7071ef --- /dev/null +++ b/.github/workflows/gradle-build.yml @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Build with gradle on JDK 11 + +on: + push: + pull_request: + branches: + - master + - branch-* + paths-ignore: + - 'site/**' + - 'site3/**' + workflow_dispatch: + +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + +jobs: + test: + + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: 11 + + - name: Build everything with gradle to validate the build works + run: ./gradlew build -x signDistTar -x test \ No newline at end of file diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 6d704b29592..8fb3d5c517c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -31,7 +31,7 @@ on: workflow_dispatch: env: - GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 jobs: test: @@ -51,14 +51,14 @@ jobs: distribution: 'temurin' java-version: 11 - - name: Build tar - run: ./gradlew stream:server:build -x test ${GRADLE_ARGS} - - name: run cluster integration test - run: ./gradlew :tests:integration:cluster:test ${GRADLE_ARGS} || (tail -n +1 tests/integration/cluster/build/reports/tests/test/classes/* && tail -n +1 tests/integration/cluster/build/container-logs/**/* && exit 1) - - name: run smoke test - run: ./gradlew tests:integration:smoke:test ${GRADLE_ARGS} - - name: run standalone test - run: ./gradlew tests:integration:standalone:test ${GRADLE_ARGS} + - name: Build with Maven + run: mvn -B -nsu clean install -Pdocker -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + + - name: Run metadata driver tests + run: mvn -B -nsu -f metadata-drivers/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + + - name: Run all integration tests + run: mvn -B -nsu -f tests/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: print JVM thread dumps when cancelled if: cancelled() diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index f00c33587d0..440edc38c22 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -31,9 +31,9 @@ on: workflow_dispatch: env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true - jobs: check: @@ -52,10 +52,11 @@ jobs: distribution: 'temurin' java-version: 11 - name: Validate pull request - run: ./gradlew build -x signDistTar -x test ${GRADLE_ARGS} + run: mvn clean -B -nsu apache-rat:check checkstyle:check package -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: Check license files - run: dev/check-all-licenses-gradle + run: dev/check-all-licenses + # keeping on gradle, `mvn site javadoc:javadoc`, to figure out later - name: Generate Javadoc run: ./gradlew generateApiJavadoc diff --git a/.github/workflows/remaining-tests.yml b/.github/workflows/remaining-tests.yml index 0bc1d144c3e..61d5dead792 100644 --- a/.github/workflows/remaining-tests.yml +++ b/.github/workflows/remaining-tests.yml @@ -31,8 +31,7 @@ on: workflow_dispatch: env: - GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true - + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 jobs: test: @@ -52,7 +51,7 @@ jobs: distribution: 'temurin' java-version: 11 - name: Run remaining tests - run: ./gradlew bookkeeper-server:test -PexcludeTests="**/org/apache/bookkeeper/bookie/*, **/org/apache/bookkeeper/client/*, **/org/apache/bookkeeper/replication/*, **/org/apache/bookkeeper/tls/*" ${GRADLE_ARGS} + run: mvn -B -nsu -am -pl bookkeeper-server clean install test -Dtest="!org.apache.bookkeeper.client.**,!org.apache.bookkeeper.bookie.**,!org.apache.bookkeeper.replication.**,!org.apache.bookkeeper.tls.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: print JVM thread dumps when cancelled if: cancelled() diff --git a/.github/workflows/replication-tests.yml b/.github/workflows/replication-tests.yml index 954f61730bf..6845d8f1379 100644 --- a/.github/workflows/replication-tests.yml +++ b/.github/workflows/replication-tests.yml @@ -31,7 +31,7 @@ on: workflow_dispatch: env: - GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 jobs: test: @@ -51,7 +51,7 @@ jobs: distribution: 'temurin' java-version: 11 - name: Run replication tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" ${GRADLE_ARGS} + run: mvn -B -nsu -am -pl bookkeeper-server clean install test -Dtest="org.apache.bookkeeper.replication.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: print JVM thread dumps when cancelled if: cancelled() diff --git a/.github/workflows/stream-tests.yml b/.github/workflows/stream-tests.yml index 50923fd6f8d..311417844f0 100644 --- a/.github/workflows/stream-tests.yml +++ b/.github/workflows/stream-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 jobs: test: @@ -49,22 +49,10 @@ jobs: with: distribution: 'temurin' java-version: 11 - - name: Run stream:distributedlog:core tests - run: ./gradlew stream:distributedlog:core:test ${GRADLE_ARGS} - - name: Run stream:distributedlog:common tests - run: ./gradlew stream:distributedlog:common:test ${GRADLE_ARGS} - - name: Run stream:distributedlog:protocol tests - run: ./gradlew stream:distributedlog:protocol:test ${GRADLE_ARGS} - - name: Run stream:proto tests - run: ./gradlew stream:proto:test ${GRADLE_ARGS} - - name: Run stream:serve tests - run: ./gradlew stream:server:test ${GRADLE_ARGS} - - name: Run stream:statelib tests - run: ./gradlew stream:statelib:test ${GRADLE_ARGS} - - name: Run stream:storage:api:test tests - run: ./gradlew stream:storage:api:test ${GRADLE_ARGS} - - name: Run stream:storage:impl tests - run: ./gradlew stream:storage:impl:test ${GRADLE_ARGS} + - name: Build with Maven + run: mvn -B -nsu clean install -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + - name: Run StreamStorage tests + run: mvn -B -nsu -f stream/pom.xml verify -DstreamTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: print JVM thread dumps when cancelled if: cancelled() diff --git a/.github/workflows/tls-tests.yml b/.github/workflows/tls-tests.yml index 4673f55d3ec..1b456ca4513 100644 --- a/.github/workflows/tls-tests.yml +++ b/.github/workflows/tls-tests.yml @@ -31,7 +31,7 @@ on: workflow_dispatch: env: - GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 jobs: test: @@ -51,7 +51,7 @@ jobs: distribution: 'temurin' java-version: 11 - name: Run tls tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" ${GRADLE_ARGS} + run: mvn -B -am -nsu -pl bookkeeper-server clean install test -Dtest="org.apache.bookkeeper.tls.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: print JVM thread dumps when cancelled if: cancelled() From e669fab29c43217cc6a2c76d3afe53e42fc26aee Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 22 Mar 2022 12:06:08 -0700 Subject: [PATCH 09/14] OWASP check with maven --- .github/workflows/owasp-dep-check.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/owasp-dep-check.yml b/.github/workflows/owasp-dep-check.yml index 7edce7a7e69..83718f4c44d 100644 --- a/.github/workflows/owasp-dep-check.yml +++ b/.github/workflows/owasp-dep-check.yml @@ -30,6 +30,8 @@ on: - 'site3/**' workflow_dispatch: +env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 jobs: check: @@ -43,13 +45,24 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm + - name: Detect changed pom files + id: changes + uses: apache/pulsar-test-infra/paths-filter@master + with: + filters: | + poms: + - 'pom.xml' + - '**/pom.xml' + - name: Set up JDK 1.8 uses: actions/setup-java@v1 + if: ${{ steps.changes.outputs.poms == 'true' }} with: java-version: 1.8 - - name: run "clean build dependencyCheckAggregate" to trigger dependency check - run: ./gradlew clean build -x signDistTar -x test dependencyCheckAggregate + - name: run "clean install verify" to trigger dependency check + if: ${{ steps.changes.outputs.poms == 'true' }} + run: mvn -q -B -ntp clean install verify -PskipDocker,owasp-dependency-check -DskipTests - name: Upload report uses: actions/upload-artifact@v2 @@ -57,4 +70,4 @@ jobs: continue-on-error: true with: name: dependency report - path: build/reports/dependency-check-report.html + path: target/dependency-check-report.html From f5e56ea32760d178194c17d48ad3aada21d63d43 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 22 Mar 2022 14:51:05 -0700 Subject: [PATCH 10/14] Up'd versions to match gradle, corrected license files: looks like gradle build didn't force versions consistently --- .../src/main/resources/LICENSE-all.bin.txt | 6 +- .../src/main/resources/LICENSE-bkctl.bin.txt | 6 +- .../src/main/resources/LICENSE-server.bin.txt | 6 +- pom.xml | 71 ++++++++++++++++--- 4 files changed, 71 insertions(+), 18 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 27a4f3c857d..50acf576fc3 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -205,9 +205,9 @@ The following bundled 3rd party jars are distributed under the Apache Software License, Version 2. -- lib/com.fasterxml.jackson.core-jackson-annotations-2.11.0.jar [1] +- lib/com.fasterxml.jackson.core-jackson-annotations-2.11.3.jar [1] - lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] -- lib/com.fasterxml.jackson.core-jackson-databind-2.11.0.jar [3] +- lib/com.fasterxml.jackson.core-jackson-databind-2.11.3.jar [3] - lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] @@ -296,7 +296,7 @@ Apache Software License, Version 2. - lib/org.apache.httpcomponents-httpcore-4.4.13.jar [40] - lib/org.apache.thrift-libthrift-0.14.2.jar [41] - lib/com.google.android-annotations-4.1.1.4.jar [42] -- lib/com.google.http-client-google-http-client-1.34.0.jar [43] +- lib/com.google.http-client-google-http-client-1.38.0.jar [43] - lib/com.google.http-client-google-http-client-jackson2-1.38.0.jar [43] - lib/com.google.auto.value-auto-value-annotations-1.7.4.jar [44] - lib/com.google.j2objc-j2objc-annotations-1.3.jar [45] diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index 4ed5c49cc0b..1afdc14e9d8 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -205,9 +205,9 @@ The following bundled 3rd party jars are distributed under the Apache Software License, Version 2. -- lib/com.fasterxml.jackson.core-jackson-annotations-2.11.0.jar [1] +- lib/com.fasterxml.jackson.core-jackson-annotations-2.11.3.jar [1] - lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] -- lib/com.fasterxml.jackson.core-jackson-databind-2.11.0.jar [3] +- lib/com.fasterxml.jackson.core-jackson-databind-2.11.3.jar [3] - lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] @@ -276,7 +276,7 @@ Apache Software License, Version 2. - lib/org.apache.thrift-libthrift-0.14.2.jar [40] - lib/com.google.android-annotations-4.1.1.4.jar [41] - lib/com.google.auto.value-auto-value-annotations-1.7.4.jar [42] -- lib/com.google.http-client-google-http-client-1.34.0.jar [43] +- lib/com.google.http-client-google-http-client-1.38.0.jar [43] - lib/com.google.http-client-google-http-client-jackson2-1.38.0.jar [43] - lib/com.google.j2objc-j2objc-annotations-1.3.jar [44] - lib/com.google.re2j-re2j-1.5.jar [45] diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index aeb9fe2eed7..9353c230783 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -205,9 +205,9 @@ The following bundled 3rd party jars are distributed under the Apache Software License, Version 2. -- lib/com.fasterxml.jackson.core-jackson-annotations-2.11.0.jar [1] +- lib/com.fasterxml.jackson.core-jackson-annotations-2.11.3.jar [1] - lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] -- lib/com.fasterxml.jackson.core-jackson-databind-2.11.0.jar [3] +- lib/com.fasterxml.jackson.core-jackson-databind-2.11.3.jar [3] - lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] @@ -296,7 +296,7 @@ Apache Software License, Version 2. - lib/org.apache.httpcomponents-httpcore-4.4.13.jar [40] - lib/org.apache.thrift-libthrift-0.14.2.jar [41] - lib/com.google.android-annotations-4.1.1.4.jar [42] -- lib/com.google.http-client-google-http-client-1.34.0.jar [43] +- lib/com.google.http-client-google-http-client-1.38.0.jar [43] - lib/com.google.http-client-google-http-client-jackson2-1.38.0.jar [43] - lib/com.google.auto.value-auto-value-annotations-1.7.4.jar [44] - lib/com.google.j2objc-j2objc-annotations-1.3.jar [45] diff --git a/pom.xml b/pom.xml index e7108f8895b..f4119b84039 100644 --- a/pom.xml +++ b/pom.xml @@ -129,14 +129,14 @@ 0.5.11 2.7.0 3.0.2 - 2.4.0 + 2.9.0 1.42.1 31.0.1-jre 1.1.1 2.10.0 1.3 2.1.10 - 2.11.0 + 2.11.3 1.78 9.4.43.v20210629 1.19 @@ -148,21 +148,21 @@ 2.17.1 1.3.0 3.12.4 - 4.1.72.Final - 2.0.46.Final + 4.1.74.Final + 2.0.48.Final 9.1.3 2.0.2 0.8.1 0.8.3 4.5.13 - 3.14.0 - 3.14.0 + 3.17.2 + 3.17.2 ${grpc.version} 0.9.11 - 6.22.1.1 + 6.27.3 3.0.1 1.7.32 - 1.19 + 1.30 3.1.8 1.3.2 1.15.1 @@ -191,6 +191,7 @@ 3.2.0 2.2.1 3.0.0-M5 + 6.1.6 3.5.2 1.4.1.Final 0.6.1 @@ -1092,7 +1093,59 @@ - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + src/owasp-dependency-check-suppressions.xml + + 7 + false + false + false + false + false + false + false + false + false + false + false + false + + + + + aggregate + + + + + + + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + + + + + + dev From cc5aefff748cb7b5f6245921c88136167785538d Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 22 Mar 2022 14:58:05 -0700 Subject: [PATCH 11/14] Removed current-version-image to match https://github.com/apache/bookkeeper/pull/3027 --- .../current-version-image/pom.xml | 147 ------------------ tests/docker-images/pom.xml | 1 - 2 files changed, 148 deletions(-) delete mode 100644 tests/docker-images/current-version-image/pom.xml diff --git a/tests/docker-images/current-version-image/pom.xml b/tests/docker-images/current-version-image/pom.xml deleted file mode 100644 index 5e221904e3a..00000000000 --- a/tests/docker-images/current-version-image/pom.xml +++ /dev/null @@ -1,147 +0,0 @@ - - - - - org.apache.bookkeeper.tests - docker-images - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.tests - current-version-image - Apache BookKeeper :: Tests :: Docker Images :: Current Version - pom - - - org.apache.bookkeeper - bookkeeper-dist-server - ${project.parent.version} - bin - tar.gz - provided - - - - - docker - - - integrationTests - - - - - - - org.codehaus.mojo - exec-maven-plugin - ${exec-maven-plugin.version} - - - build-python-client - generate-resources - - exec - - - ${project.basedir}/target - ${project.basedir}/../../../stream/clients/python/scripts/docker_build.sh - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - ${maven-antrun-plugin.version} - - - generate-resources - - run - - - - copy python wheel file - - - copying docker scripts - - - - - - - - - com.spotify - dockerfile-maven-plugin - ${dockerfile-maven-plugin.version} - - - default - - build - - - - add-latest-tag - - tag - - - apachebookkeeper/bookkeeper-current - latest - - - - - apachebookkeeper/bookkeeper-current - ${project.version} - false - true - - ${project.version} - - - - - org.apache.maven.plugins - maven-dependency-plugin - ${maven-dependency-plugin.version} - - - copy-docker-dependencies - - copy-dependencies - - generate-resources - - ${project.build.directory}/ - bookkeeper-dist-server - true - - - - - - - - - diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index a337e91176a..924d8322f25 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -29,6 +29,5 @@ all-released-versions-image all-versions-image - current-version-image From a9ff0908a6cbe21e8818a1c3343f1a91b79e3839 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 22 Mar 2022 16:39:59 -0700 Subject: [PATCH 12/14] Shading patetrn to match gradle --- shaded/distributedlog-core-shaded/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shaded/distributedlog-core-shaded/pom.xml b/shaded/distributedlog-core-shaded/pom.xml index c1cf1fef955..2871b08a6f4 100644 --- a/shaded/distributedlog-core-shaded/pom.xml +++ b/shaded/distributedlog-core-shaded/pom.xml @@ -201,7 +201,7 @@ org.apache.distributedlog - org.apache.distributedlog + dlshade.org.apache.distributedlog From d602226979dda6153ea25a27f88d9518d87ed0c4 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 22 Mar 2022 17:48:16 -0700 Subject: [PATCH 13/14] Fixed/suppressed CVEs --- .github/workflows/owasp-dep-check.yml | 4 +- .../src/main/resources/LICENSE-all.bin.txt | 6 +-- .../src/main/resources/LICENSE-bkctl.bin.txt | 6 +-- .../src/main/resources/LICENSE-server.bin.txt | 6 +-- dependencies.gradle | 4 +- pom.xml | 36 ++++++---------- src/owasp-dependency-check-suppressions.xml | 42 +++++++++++++++++++ 7 files changed, 68 insertions(+), 36 deletions(-) diff --git a/.github/workflows/owasp-dep-check.yml b/.github/workflows/owasp-dep-check.yml index 83718f4c44d..695d97c6caf 100644 --- a/.github/workflows/owasp-dep-check.yml +++ b/.github/workflows/owasp-dep-check.yml @@ -62,7 +62,9 @@ jobs: - name: run "clean install verify" to trigger dependency check if: ${{ steps.changes.outputs.poms == 'true' }} - run: mvn -q -B -ntp clean install verify -PskipDocker,owasp-dependency-check -DskipTests + # excluding dlfs because it includes hadoop lib with + # CVEs that we cannot patch up anyways + run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs' - name: Upload report uses: actions/upload-artifact@v2 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 50acf576fc3..1a815b633f2 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -205,9 +205,9 @@ The following bundled 3rd party jars are distributed under the Apache Software License, Version 2. -- lib/com.fasterxml.jackson.core-jackson-annotations-2.11.3.jar [1] -- lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] -- lib/com.fasterxml.jackson.core-jackson-databind-2.11.3.jar [3] +- lib/com.fasterxml.jackson.core-jackson-annotations-2.13.2.jar [1] +- lib/com.fasterxml.jackson.core-jackson-core-2.13.2.jar [2] +- lib/com.fasterxml.jackson.core-jackson-databind-2.13.2.jar [3] - lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index 1afdc14e9d8..ba06c3b16aa 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -205,9 +205,9 @@ The following bundled 3rd party jars are distributed under the Apache Software License, Version 2. -- lib/com.fasterxml.jackson.core-jackson-annotations-2.11.3.jar [1] -- lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] -- lib/com.fasterxml.jackson.core-jackson-databind-2.11.3.jar [3] +- lib/com.fasterxml.jackson.core-jackson-annotations-2.13.2.jar [1] +- lib/com.fasterxml.jackson.core-jackson-core-2.13.2.jar [2] +- lib/com.fasterxml.jackson.core-jackson-databind-2.13.2.jar [3] - lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 9353c230783..34f8c788004 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -205,9 +205,9 @@ The following bundled 3rd party jars are distributed under the Apache Software License, Version 2. -- lib/com.fasterxml.jackson.core-jackson-annotations-2.11.3.jar [1] -- lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] -- lib/com.fasterxml.jackson.core-jackson-databind-2.11.3.jar [3] +- lib/com.fasterxml.jackson.core-jackson-annotations-2.13.2.jar [1] +- lib/com.fasterxml.jackson.core-jackson-core-2.13.2.jar [2] +- lib/com.fasterxml.jackson.core-jackson-databind-2.13.2.jar [3] - lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] diff --git a/dependencies.gradle b/dependencies.gradle index ee90fa22485..8da660af028 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -29,7 +29,7 @@ depVersions = [ commonsCli: "1.2", commonsCodec: "1.6", commonsCollections4: "4.1", - commonsCompress: "1.19", + commonsCompress: "1.21", commonsConfiguration: "1.10", commonsIO: "2.7", commonsLang2: "2.6", @@ -49,7 +49,7 @@ depVersions = [ hamcrest: "1.3", hdrhistogram: "2.1.10", httpclient: "4.5.13", - jackson: "2.11.0", + jackson: "2.13.2", javaxServlet: "4.0.0", javaAnnotations:"1.3.2", jcommander: "1.78", diff --git a/pom.xml b/pom.xml index f4119b84039..44710dd1bd4 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ 4.1 1.6 1.10 - 1.19 + 1.21 2.6 3.6 2.7 @@ -136,11 +136,12 @@ 2.10.0 1.3 2.1.10 - 2.11.3 + 2.13.2 1.78 9.4.43.v20210629 1.19 2.8.2 + 1.14.3 3.2.7 4.12 0.14.2 @@ -342,35 +343,17 @@ - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-annotations + com.fasterxml.jackson + jackson-bom ${jackson.version} + pom + import javax.servlet javax.servlet-api ${servlet-api.version} - - com.fasterxml.jackson.module - jackson-module-paranamer - ${jackson.version} - - - com.fasterxml.jackson.module - jackson-module-scala_2.11 - ${jackson.version} - @@ -752,6 +735,11 @@ testcontainers ${testcontainers.version} + + org.jsoup + jsoup + ${jsoup.version} + diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index c65b18b1b2e..3628a297d6d 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -66,6 +66,48 @@ 6dac6efe035a2be9ba299fbf31be5f903401869f CVE-2020-15113 + + + + ^pkg:maven/org\.apache\.bookkeeper\.http/http\-server@.*$ + cpe:/a:apache:http_server + + + + ^pkg:maven/org\.apache\.bookkeeper\.http/vertx\-http\-server@.*$ + CVE-2009-1890 + + + CVE-2021-43045 affects only .NET distro, see https://github.com/apache/avro/pull/1357 + org\.apache\.avro:.* + CVE-2021-43045 + + + + ^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$ + cpe:/a:netty:netty + + + + + ^pkg:maven/org\.arquillian\.cube/arquillian\-cube\-docker@.*$ + cpe:/a:docker:docker + + + + ^pkg:maven/org\.arquillian\.cube/arquillian\-cube\-docker@.*$ + cpe:/a:redhat:docker + From 21bdeb6190d99b6fe7cbd95724fd36ea7fe9e069 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Wed, 23 Mar 2022 13:42:20 -0700 Subject: [PATCH 14/14] Attempt to fix failing tests in CompactionByEntriesWithMetadataCacheTest --- .../apache/bookkeeper/bookie/CompactionTest.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java index 2b35d7f0b53..47726b0b0c5 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java @@ -1530,6 +1530,7 @@ public void testSuspendGarbageCollection() throws Exception { conf.setGcWaitTime(500); conf.setMinorCompactionInterval(1); conf.setMajorCompactionInterval(2); + conf.setMajorCompactionMaxTimeMillis(5000); runFunctionWithLedgerManagerFactory(conf, lmf -> { try (LedgerManager lm = lmf.newLedgerManager()) { testSuspendGarbageCollection(conf, lm); @@ -1579,8 +1580,9 @@ public void checkpointComplete(Checkpoint checkPoint, boolean compact) int majorCompactions = stats.getCounter("storage.gc." + MAJOR_COMPACTION_COUNT).get().intValue(); int minorCompactions = stats.getCounter("storage.gc." + MINOR_COMPACTION_COUNT).get().intValue(); - Thread.sleep(conf.getMajorCompactionInterval() * 1000 - + conf.getGcWaitTime()); + Thread.sleep(3 * (conf.getMajorCompactionInterval() * 1000 + + conf.getGcWaitTime() + + conf.getMajorCompactionMaxTimeMillis())); assertTrue( "Major compaction should have happened", stats.getCounter("storage.gc." + MAJOR_COMPACTION_COUNT).get() > majorCompactions); @@ -1764,11 +1766,16 @@ public void testCompactionFailureShouldNotResultInDuplicatedData() throws Except restartBookies(c -> { // now enable normal compaction c.setMajorCompactionThreshold(0.5f); + c.setMajorCompactionMaxTimeMillis(5000); return c; }); + getGCThread().enableForceGC(); + getGCThread().triggerGC().get(); + Thread.sleep(confByIndex(0).getMajorCompactionInterval() * 1000 - + confByIndex(0).getGcWaitTime()); + + confByIndex(0).getGcWaitTime() + + confByIndex(0).getMajorCompactionMaxTimeMillis()); // compaction worker should compact [0-4].log for (File ledgerDirectory : bookieLedgerDirs()) { assertFalse("Entry log file ([0,1,2].log should have been compacted in ledgerDirectory: "