Allow to use IO uring instead of epoll

bookkeeper-common/pom.xml

@@ -59,6 +59,16 @@
       <groupId>org.jctools</groupId>
       <artifactId>jctools-core</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.netty.incubator</groupId>
+      <artifactId>netty-incubator-transport-native-io_uring</artifactId>
+      <classifier>linux-x86_64</classifier>
+    </dependency>
+    <dependency>
+      <groupId>io.netty.incubator</groupId>
+      <artifactId>netty-incubator-transport-native-io_uring</artifactId>
+      <classifier>linux-aarch_64</classifier>
+    </dependency>
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>

bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt

@@ -238,6 +238,9 @@ Apache Software License, Version 2.
 - lib/io.netty-netty-transport-4.1.81.Final.jar [11]
 - lib/io.netty-netty-transport-classes-epoll-4.1.81.Final.jar [11]
 - lib/io.netty-netty-transport-native-epoll-4.1.81.Final-linux-x86_64.jar [11]
+- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-x86_64.jar [11]
+- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-aarch_64.jar [11]
+- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.15.Final.jar [11]
 - lib/io.netty-netty-transport-native-unix-common-4.1.81.Final.jar [11]
 - lib/io.prometheus-simpleclient-0.15.0.jar [12]
 - lib/io.prometheus-simpleclient_common-0.15.0.jar [12]

bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt

@@ -236,6 +236,9 @@ Apache Software License, Version 2.
 - lib/io.netty-netty-transport-4.1.81.Final.jar [11]
 - lib/io.netty-netty-transport-classes-epoll-4.1.81.Final.jar [11]
 - lib/io.netty-netty-transport-native-epoll-4.1.81.Final-linux-x86_64.jar [11]
+- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-x86_64.jar [11]
+- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-aarch_64.jar [11]
+- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.15.Final.jar [11]
 - lib/io.netty-netty-transport-native-unix-common-4.1.81.Final.jar [11]
 - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [16]
 - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [16]

bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt

@@ -238,6 +238,9 @@ Apache Software License, Version 2.
 - lib/io.netty-netty-transport-4.1.81.Final.jar [11]
 - lib/io.netty-netty-transport-classes-epoll-4.1.81.Final.jar [11]
 - lib/io.netty-netty-transport-native-epoll-4.1.81.Final-linux-x86_64.jar [11]
+- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-x86_64.jar [11]
+- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-aarch_64.jar [11]
+- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.15.Final.jar [11]
 - lib/io.netty-netty-transport-native-unix-common-4.1.81.Final.jar [11]
 - lib/io.prometheus-simpleclient-0.15.0.jar [12]
 - lib/io.prometheus-simpleclient_common-0.15.0.jar [12]

bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt

@@ -44,6 +44,8 @@ LongAdder), which was released with the following comments:
 - lib/io.netty-netty-transport-4.1.81.Final.jar
 - lib/io.netty-netty-transport-classes-epoll-4.1.81.Final.jar
 - lib/io.netty-netty-transport-native-epoll-4.1.81.Final-linux-x86_64.jar
+- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-x86_64.jar
+- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-aarch_64.jar
 - lib/io.netty-netty-transport-native-unix-common-4.1.81.Final.jar
 
 

bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt

@@ -24,6 +24,8 @@ The Apache Software Foundation (http://www.apache.org/).
 - lib/io.netty-netty-transport-4.1.81.Final.jar
 - lib/io.netty-netty-transport-classes-epoll-4.1.81.Final.jar
 - lib/io.netty-netty-transport-native-epoll-4.1.81.Final-linux-x86_64.jar
+- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-x86_64.jar
+- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-aarch_64.jar
 - lib/io.netty-netty-transport-native-unix-common-4.1.81.Final.jar
 
 

bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt

@@ -26,6 +26,8 @@ The Apache Software Foundation (http://www.apache.org/).
 - lib/io.netty-netty-transport-4.1.81.Final.jar
 - lib/io.netty-netty-transport-classes-epoll-4.1.81.Final.jar
 - lib/io.netty-netty-transport-native-epoll-4.1.81.Final-linux-x86_64.jar
+- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-x86_64.jar
+- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-aarch_64.jar
 - lib/io.netty-netty-transport-native-unix-common-4.1.81.Final.jar
 
 

bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java

@@ -52,6 +52,8 @@
 import io.netty.handler.codec.LengthFieldPrepender;
 import io.netty.handler.flush.FlushConsolidationHandler;
 import io.netty.handler.ssl.SslHandler;
+import io.netty.incubator.channel.uring.IOUringEventLoopGroup;
+import io.netty.incubator.channel.uring.IOUringServerSocketChannel;
 import io.netty.util.concurrent.DefaultThreadFactory;
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -316,7 +318,9 @@ private void listenOn(InetSocketAddress address, BookieSocketAddress bookieAddre
             bootstrap.option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(
                     conf.getServerWriteBufferLowWaterMark(), conf.getServerWriteBufferHighWaterMark()));
 
-            if (eventLoopGroup instanceof EpollEventLoopGroup) {
+            if (eventLoopGroup instanceof IOUringEventLoopGroup){
+                bootstrap.channel(IOUringServerSocketChannel.class);
+            } else if (eventLoopGroup instanceof EpollEventLoopGroup) {
                 bootstrap.channel(EpollServerSocketChannel.class);
             } else {
                 bootstrap.channel(NioServerSocketChannel.class);
@@ -386,6 +390,8 @@ protected void initChannel(SocketChannel ch) throws Exception {
 
             if (jvmEventLoopGroup instanceof DefaultEventLoopGroup) {
                 jvmBootstrap.channel(LocalServerChannel.class);
+            } else if (jvmEventLoopGroup instanceof IOUringEventLoopGroup) {
+                jvmBootstrap.channel(IOUringServerSocketChannel.class);
             } else if (jvmEventLoopGroup instanceof EpollEventLoopGroup) {
                 jvmBootstrap.channel(EpollServerSocketChannel.class);
             } else {

bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java

@@ -57,6 +57,9 @@
 import io.netty.handler.codec.TooLongFrameException;
 import io.netty.handler.flush.FlushConsolidationHandler;
 import io.netty.handler.ssl.SslHandler;
+import io.netty.incubator.channel.uring.IOUringChannelOption;
+import io.netty.incubator.channel.uring.IOUringEventLoopGroup;
+import io.netty.incubator.channel.uring.IOUringSocketChannel;
 import io.netty.util.Recycler;
 import io.netty.util.Recycler.Handle;
 import io.netty.util.concurrent.Future;
@@ -543,7 +546,14 @@ protected ChannelFuture connect() {
         // Set up the ClientBootStrap so we can create a new Channel connection to the bookie.
         Bootstrap bootstrap = new Bootstrap();
         bootstrap.group(eventLoopGroup);
-        if (eventLoopGroup instanceof EpollEventLoopGroup) {
+        if (eventLoopGroup instanceof IOUringEventLoopGroup) {
+            bootstrap.channel(IOUringSocketChannel.class);
+            try {
+                bootstrap.option(IOUringChannelOption.TCP_USER_TIMEOUT, conf.getTcpUserTimeoutMillis());
+            } catch (NoSuchElementException e) {
+                // Property not set, so keeping default value.
+            }
+        } else if (eventLoopGroup instanceof EpollEventLoopGroup) {
             bootstrap.channel(EpollSocketChannel.class);
             try {
                 // For Epoll channels, configure the TCP user timeout.

bookkeeper-server/src/main/java/org/apache/bookkeeper/util/EventLoopUtil.java

@@ -21,13 +21,16 @@
 import io.netty.channel.SelectStrategy;
 import io.netty.channel.epoll.EpollEventLoopGroup;
 import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.incubator.channel.uring.IOUring;
+import io.netty.incubator.channel.uring.IOUringEventLoopGroup;
 import java.util.concurrent.ThreadFactory;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.common.util.affinity.CpuAffinity;
 import org.apache.bookkeeper.conf.ClientConfiguration;
 import org.apache.bookkeeper.conf.ServerConfiguration;
 import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.StringUtils;
 
 
 /**
@@ -36,6 +39,9 @@
 @Slf4j
 @UtilityClass
 public class EventLoopUtil {
+
+    private static final String ENABLE_IO_URING = "enable.io_uring";
+
     public static EventLoopGroup getClientEventLoopGroup(ClientConfiguration conf, ThreadFactory threadFactory) {
         return getEventLoopGroup(threadFactory, conf.getNumIOThreads(), conf.isBusyWaitEnabled());
     }
@@ -54,33 +60,43 @@ private static EventLoopGroup getEventLoopGroup(ThreadFactory threadFactory,
             return new NioEventLoopGroup(numThreads, threadFactory);
         }
 
-        try {
-            if (!enableBusyWait) {
-                // Regular Epoll based event loop
-                return new EpollEventLoopGroup(numThreads, threadFactory);
-            }
+        String enableIoUring = System.getProperty(ENABLE_IO_URING);
 
-            // With low latency setting, put the Netty event loop on busy-wait loop to reduce cost of
-            // context switches
-            EpollEventLoopGroup eventLoopGroup = new EpollEventLoopGroup(numThreads, threadFactory,
-                    () -> (selectSupplier, hasTasks) -> SelectStrategy.BUSY_WAIT);
+        // By default, io_uring will not be enabled, even if available. The environment variable will be used:
+        // enable.io_uring=1
+        if (StringUtils.equalsAnyIgnoreCase(enableIoUring, "1", "true")) {
+            // Throw exception if IOUring cannot be used
+            IOUring.ensureAvailability();
+            return new IOUringEventLoopGroup(numThreads, threadFactory);
+        } else {
+            try {
+                if (!enableBusyWait) {
+                    // Regular Epoll based event loop
+                    return new EpollEventLoopGroup(numThreads, threadFactory);
+                }
 
-            // Enable CPU affinity on IO threads
-            for (int i = 0; i < numThreads; i++) {
-                eventLoopGroup.next().submit(() -> {
-                    try {
-                        CpuAffinity.acquireCore();
-                    } catch (Throwable t) {
-                        log.warn("Failed to acquire CPU core for thread {} err {} {}",
-                                Thread.currentThread().getName(), t.getMessage(), t);
-                    }
-                });
-            }
+                // With low latency setting, put the Netty event loop on busy-wait loop to reduce cost of
+                // context switches
+                EpollEventLoopGroup eventLoopGroup = new EpollEventLoopGroup(numThreads, threadFactory,
+                        () -> (selectSupplier, hasTasks) -> SelectStrategy.BUSY_WAIT);
 
-            return eventLoopGroup;
-        } catch (ExceptionInInitializerError | NoClassDefFoundError | UnsatisfiedLinkError e) {
-            log.warn("Could not use Netty Epoll event loop: {}", e.getMessage());
-            return new NioEventLoopGroup(numThreads, threadFactory);
+                // Enable CPU affinity on IO threads
+                for (int i = 0; i < numThreads; i++) {
+                    eventLoopGroup.next().submit(() -> {
+                        try {
+                            CpuAffinity.acquireCore();
+                        } catch (Throwable t) {
+                            log.warn("Failed to acquire CPU core for thread {} err {} {}",
+                                    Thread.currentThread().getName(), t.getMessage(), t);
+                        }
+                    });
+                }
+
+                return eventLoopGroup;
+            } catch (ExceptionInInitializerError | NoClassDefFoundError | UnsatisfiedLinkError e) {
+                log.warn("Could not use Netty Epoll event loop: {}", e.getMessage());
+                return new NioEventLoopGroup(numThreads, threadFactory);
+            }
         }
     }
 }

pom.xml

@@ -155,6 +155,7 @@
     <mockito.version>3.12.4</mockito.version>
     <netty.version>4.1.81.Final</netty.version>
     <netty-boringssl.version>2.0.54.Final</netty-boringssl.version>
+    <netty-iouring.version>0.0.15.Final</netty-iouring.version>
     <ostrich.version>9.1.3</ostrich.version>
     <powermock.version>2.0.9</powermock.version>
     <prometheus.version>0.15.0</prometheus.version>
@@ -467,6 +468,23 @@
         <artifactId>netty-tcnative-boringssl-static</artifactId>
         <version>${netty-boringssl.version}</version>
       </dependency>
+      <dependency>
+        <groupId>io.netty.incubator</groupId>
+        <artifactId>netty-incubator-transport-native-io_uring</artifactId>
+        <version>${netty-iouring.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.netty.incubator</groupId>
+        <artifactId>netty-incubator-transport-native-io_uring</artifactId>
+        <version>${netty-iouring.version}</version>
+        <classifier>linux-x86_64</classifier>
+      </dependency>
+      <dependency>
+        <groupId>io.netty.incubator</groupId>
+        <artifactId>netty-incubator-transport-native-io_uring</artifactId>
+        <version>${netty-iouring.version}</version>
+        <classifier>linux-aarch_64</classifier>
+      </dependency>
 
       <!-- grpc dependencies -->
       <dependency>