BOOKKEEPER-108: add configuration support for BK (Sijie via ivank)

git-svn-id: https://svn.apache.org/repos/asf/zookeeper/bookkeeper/trunk@1203568 13f79535-47bb-0310-9956-ffa450edef68

Author: ivankelly

CHANGES.txt

@@ -74,6 +74,8 @@ BUGFIXES:
 
   BOOKKEEPER-91: Bookkeeper and hedwig clients should not use log4j directly (ivank via fpj)
 
+  BOOKKEEPER-108: add configuration support for BK (Sijie via ivank)
+
  hedwig-server/
 
   BOOKKEEPER-43: NullPointException when releasing topic (Sijie Guo via breed)

bookkeeper-server/bin/bookkeeper

@@ -23,10 +23,7 @@
 BINDIR=`dirname "$0"`
 BK_HOME=`cd $BINDIR/..;pwd`
 
-DEFAULT_ZOOKEEPER=localhost:2181
-DEFAULT_PORT=3181
-DEFAULT_TXN_DIR=/tmp/bk-txn
-DEFAULT_DATA_DIR=/tmp/bk-data
+DEFAULT_CONF=$BK_HOME/conf/bk_server.conf
 
 source $BK_HOME/conf/bkenv.sh
 
@@ -55,11 +52,8 @@ where command is one of:
 or command is the full name of a class with a defined main() method.
 
 Environment variables:
-   BOOKIE_ZOOKEEPER       Zookeeper ensemble (default: $DEFAULT_ZOOKEEPER)
-   BOOKIE_PORT            Port to listen on (default: $DEFAULT_PORT)
-   BOOKIE_TXN_LOGDIR      Directory for transaction logs (default: $DEFAULT_TXN_DIR)
-   BOOKIE_DATA_DIR        Directory for data (default: $DEFAULT_DATA_DIR)
    BOOKIE_LOG_CONF        Log4j configuration file
+   BOOKIE_CONF            Configuration file (default: conf/bk_server.conf)
    BOOKIE_EXTRA_OPTS      Extra options to be passed to the jvm
 
 These variable can also be set in conf/bkenv.sh
@@ -101,20 +95,8 @@ fi
 COMMAND=$1
 shift
 
-if [ "$BOOKIE_ZOOKEEPER" == "" ]; then
-    BOOKIE_ZOOKEEPER=$DEFAULT_ZOOKEEPER
-fi
-
-if [ "$BOOKIE_PORT" == "" ]; then
-    BOOKIE_PORT=$DEFAULT_PORT
-fi
-
-if [ "$BOOKIE_TXN_LOGDIR" == "" ]; then
-    BOOKIE_TXN_LOGDIR=$DEFAULT_TXN_DIR
-fi
-
-if [ "$BOOKIE_DATA_DIR" == "" ]; then
-    BOOKIE_DATA_DIR=$DEFAULT_DATA_DIR
+if [ "$BOOKIE_CONF" == "" ]; then
+    BOOKIE_CONF=$DEFAULT_CONF
 fi
 
 BOOKIE_CLASSPATH="$BOOKIE_JAR:$BOOKIE_CLASSPATH"
@@ -127,11 +109,11 @@ OPTS="-cp $BOOKIE_CLASSPATH $OPTS $BOOKIE_EXTRA_OPTS"
 OPTS="$OPTS $BOOKIE_EXTRA_OPTS"
 
 if [ $COMMAND == "bookie" ]; then
-    exec java $OPTS org.apache.bookkeeper.proto.BookieServer $BOOKIE_PORT $BOOKIE_ZOOKEEPER $BOOKIE_TXN_LOGDIR $BOOKIE_DATA_DIR $@
+    exec java $OPTS org.apache.bookkeeper.proto.BookieServer --conf $BOOKIE_CONF $@
 elif [ $COMMAND == "localbookie" ]; then
     NUMBER=$1
     shift
-    exec java $OPTS org.apache.bookkeeper.util.LocalBookKeeper $NUMBER $@
+    exec java $OPTS org.apache.bookkeeper.util.LocalBookKeeper $NUMBER $BOOKIE_CONF $@
 elif [ $COMMAND == "help" ]; then
     bookkeeper_help;
 else

bookkeeper-server/conf/bk_server.conf

@@ -0,0 +1,124 @@
+#!/bin/sh
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+## Bookie settings
+
+# Port that bookie server listen on
+bookie_port=3181
+
+# Directory Bookkeeper outputs its write ahead log
+journal_dir=/tmp/bk-txn
+
+# Directory Bookkeeper outputs ledger snapshots
+# could define multi directories to store snapshots, separated by ','
+# For example:
+# ledger_dirs=/tmp/bk1-data,/tmp/bk2-data
+# 
+# Ideally ledger dirs and journal dir are each in a differet device,
+# which reduce the contention between random i/o and sequential write.
+# It is possible to run with a single disk, but performance will be significantly lower.
+ledger_dirs=/tmp/bk-data
+
+# Max file size of entry logger, in bytes
+# A new entry log file will be created when the old one reaches the file size limitation
+# logSizeLimit=2147483648
+
+# Max file size of journal file, in mega bytes
+# A new journal file will be created when the old one reaches the file size limitation
+# journal_max_size_mb=2048
+
+# Max number of old journal file to kept
+# Keep a number of old journal files would help data recovery in specia case
+# journal_max_backups=5
+
+# How long the interval to trigger next garbage collection, in milliseconds
+# Since garbage collection is running in background, too frequent gc
+# will heart performance. It is better to give a higher number of gc
+# interval if there is enough disk capacity.
+# gcWaitTime=1000
+
+# How long the interval to flush ledger index pages to disk, in milliseconds
+# Flushing index files will introduce much random disk I/O.
+# If separating journal dir and ledger dirs each on different devices,
+# flushing would not affect performance. But if putting journal dir
+# and ledger dirs on same device, performance degrade significantly
+# on too frequent flushing. You can consider increment flush interval
+# to get better performance, but you need to pay more time on bookie
+# server restart after failure.
+# flush_interval=100
+
+# Interval to watch whether bookie is dead or not, in milliseconds
+# bookie_death_watch_interval=1000
+
+## zookeeper client settings
+
+# A list of one of more servers on which zookeeper is running.
+# The server list can be comma separated values, for example:
+# zkServers=zk1:2181,zk2:2181,zk3:2181
+zkServers=localhost:2181
+# ZooKeeper client session timeout in milliseconds
+# Bookie server will exit if it received SESSION_EXPIRED because it
+# was partitioned off from ZooKeeper for more than the session timeout
+# JVM garbage collection, disk I/O will cause SESSION_EXPIRED.
+# Increment this value could help avoiding this issue
+zkTimeout=10000
+
+## NIO Server settings
+
+# This settings is used to enabled/disabled Nagle's algorithm, which is a means of
+# improving the efficiency of TCP/IP networks by reducing the number of packets
+# that need to be sent over the network.
+# If you are sending many small messages, such that more than one can fit in
+# a single IP packet, setting server.tcpnodelay to false to enable Nagle algorithm
+# can provide better performance.
+# Default value is true.
+# server.tcpnodelay=true
+
+## ledger cache settings
+
+# Max number of ledger index files could be opened in bookie server
+# If number of ledger index files reaches this limitation, bookie
+# server started to swap some ledgers from memory to disk.
+# Too frequent swap will affect performance. You can tune this number
+# to gain performance according your requirements.
+# openFileLimit=900
+
+# Size of a index page in ledger cache, in bytes
+# A larger index page can improve performance writing page to disk,
+# which is efficent when you have small number of ledgers and these
+# ledgers have similar number of entries.
+# If you have large number of ledgers and each ledger has fewer entries,
+# smaller index page would improve memory usage.
+# pageSize=8192
+
+# How many index pages provided in ledger cache
+# If number of index pages reaches this limitation, bookie server
+# starts to swap some ledgers from memory to disk. You can increment
+# this value when you found swap became more frequent. But make sure
+# pageLimit*pageSize should not more than JVM max memory limitation,
+# otherwise you would got OutOfMemoryException.
+# In general, incrementing pageLimit, using smaller index page would
+# gain bettern performance in lager number of ledgers with fewer entries case
+# If pageLimit is -1, bookie server will use 1/3 of JVM memory to compute
+# the limitation of number of index pages.
+# pageLimit=-1
+

bookkeeper-server/conf/bkenv.sh

@@ -22,14 +22,5 @@
 
 # default settings for starting bookkeeper
 
-# Zookeeper ensemble for bookkeeper to use
-#BOOKIE_ZOOKEEPER=
-
-# Port for bookie to listen on
-#BOOKIE_PORT=
-
-# Directory Bookkeeper outputs its write ahead log
-#BOOKIE_TXN_LOGDIR=
-
-# Directory Bookkeeper outputs ledger snapshots
-#BOOKIE_DATA_DIR=
+# Configuration file of settings used in bookie server
+# BOOKIE_CONF=

bookkeeper-server/pom.xml

@@ -50,6 +50,16 @@
       <version>3.2.4.Final</version>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>commons-configuration</groupId>
+      <artifactId>commons-configuration</artifactId>
+      <version>1.6</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+      <version>1.2</version>
+    </dependency>
     <!--
 	Annoying dependency we need to include because
 	zookeeper uses log4j and so we transatively do, but

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java

@@ -44,6 +44,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.bookkeeper.bookie.BookieException;
+import org.apache.bookkeeper.conf.ServerConfiguration;
 import org.apache.bookkeeper.proto.BookieServer;
 import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.WriteCallback;
 import org.slf4j.Logger;
@@ -66,14 +67,18 @@ public class Bookie extends Thread {
     static Logger LOG = LoggerFactory.getLogger(Bookie.class);
     final static long MB = 1024 * 1024L;
     // max journal file size
-    final static long MAX_JOURNAL_SIZE = Long.getLong("journal_max_size_mb", 2 * 1024) * MB;
+    final long maxJournalSize;
     // number journal files kept before marked journal
-    final static int MAX_BACKUP_JOURNALS = Integer.getInteger("journal_max_backups", 5);
+    final int maxBackupJournals;
 
     final File journalDirectory;
 
     final File ledgerDirectories[];
 
+    final ServerConfiguration conf;
+
+    final SyncThread syncThread;
+
     /**
      * Current directory layout version. Increment this 
      * when you make a change to the format of any of the files in 
@@ -82,7 +87,6 @@ public class Bookie extends Thread {
     static final int CURRENT_DIRECTORY_LAYOUT_VERSION = 1;
     static final String VERSION_FILENAME = "VERSION";
 
-
     // ZK registration path for this bookie
     static final String BOOKIE_REGISTRATION_PATH = "/ledgers/available/";
 
@@ -149,9 +153,13 @@ class SyncThread extends Thread {
         // flag to ensure sync thread will not be interrupted during flush
         final AtomicBoolean flushing = new AtomicBoolean(false);
         // make flush interval as a parameter
-        final int flushInterval = Integer.getInteger("flush_interval", 100);
-        public SyncThread() {
+        final int flushInterval;
+        public SyncThread(ServerConfiguration conf) {
             super("SyncThread");
+            flushInterval = conf.getFlushInterval();
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("Flush Interval : " + flushInterval);
+            }
         }
         @Override
         public void run() {
@@ -204,8 +212,8 @@ public boolean accept(long journalId) {
                 });
 
                 // keep MAX_BACKUP_JOURNALS journal files before marked journal
-                if (logs.size() >= MAX_BACKUP_JOURNALS) {
-                    int maxIdx = logs.size() - MAX_BACKUP_JOURNALS;
+                if (logs.size() >= maxBackupJournals) {
+                    int maxIdx = logs.size() - maxBackupJournals;
                     for (int i=0; i<maxIdx; i++) {
                         long id = logs.get(i);
                         // make sure the journal id is smaller than marked journal id
@@ -233,18 +241,24 @@ void shutdown() throws InterruptedException {
             this.join();
         }
     }
-    SyncThread syncThread = new SyncThread();
 
-    public Bookie(int port, String zkServers, File journalDirectory, File ledgerDirectories[]) throws IOException {
+    public Bookie(ServerConfiguration conf) throws IOException {
+        this.journalDirectory = conf.getJournalDir();
+        this.ledgerDirectories = conf.getLedgerDirs();
+        this.conf = conf;
+
         checkDirectoryLayoutVersion(journalDirectory);
         for (File dir : ledgerDirectories) {
             checkDirectoryLayoutVersion(dir);
         }
 
-        this.journalDirectory = journalDirectory;
-        this.ledgerDirectories = ledgerDirectories;
-        entryLogger = new EntryLogger(ledgerDirectories, this);
-        ledgerCache = new LedgerCache(ledgerDirectories);
+        this.maxJournalSize = conf.getMaxJournalSize() * MB;
+        this.maxBackupJournals = conf.getMaxBackupJournals();
+
+        syncThread = new SyncThread(conf);
+        entryLogger = new EntryLogger(conf, this);
+        ledgerCache = new LedgerCache(conf);
+    
         lastLogMark.readLog();
         if (LOG.isDebugEnabled()) {
             LOG.debug("Last Log Mark : " + lastLogMark);
@@ -318,7 +332,7 @@ public boolean accept(long journalId) {
                 }
             }
         }
-        instantiateZookeeperClient(port, zkServers);
+        instantiateZookeeperClient(conf.getBookiePort(), conf.getZkServers());
         setDaemon(true);
         LOG.debug("I'm starting a bookie with journal directory " + journalDirectory.getName());
         start();
@@ -372,7 +386,7 @@ private void instantiateZookeeperClient(int port, String zkServers) throws IOExc
             isZkExpired = false;
             return;
         }
-        int zkTimeout = Integer.getInteger("zkTimeout", 10000);
+        int zkTimeout = conf.getZkTimeout();
         // Create the ZooKeeper client instance
         zk = newZookeeper(zkServers, zkTimeout);
         // Create the ZK ephemeral node for this Bookie.
@@ -771,7 +785,7 @@ public void run() {
                             toFlush.clear();
 
                             // check whether journal file is over file limit
-                            if (bc.position() > MAX_JOURNAL_SIZE) {
+                            if (bc.position() > maxJournalSize) {
                                 logFile.close();
                                 logFile = null;
                                 continue;
@@ -971,7 +985,7 @@ synchronized public void waitZero() throws InterruptedException {
      */
     public static void main(String[] args) throws IOException,
         InterruptedException, BookieException {
-        Bookie b = new Bookie(5000, null, new File("/tmp"), new File[] { new File("/tmp") });
+        Bookie b = new Bookie(new ServerConfiguration());
         CounterCallback cb = new CounterCallback();
         long start = System.currentTimeMillis();
         for (int i = 0; i < 100000; i++) {