From d19805a7bca624e27d2b8d6e1d80c9438a61a026 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Mon, 18 Jul 2022 15:48:40 -0700 Subject: [PATCH] HBASE-27195 Clean up netty worker/thread pool configuration (#4619) The configuration settings "hbase.netty.worker.count" and "hbase.netty.eventloop.rpcserver.thread.count" appear to duplicate each other. Also, formalizes another setting found in NettyEventLoopGroupConfig, "hbase.netty.nativetransport". Also, native epoll is not limited to amd64. aarch64 supports it too. Signed-off-by: Duo Zhang Conflicts: hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java --- .../org/apache/hadoop/hbase/util/JVM.java | 19 ++++++++--- .../hadoop/hbase/ipc/NettyRpcServer.java | 32 +++++-------------- .../hbase/util/NettyEventLoopGroupConfig.java | 24 +++++++++++--- .../src/test/resources/hbase-site.xml | 5 --- 4 files changed, 42 insertions(+), 38 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/JVM.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/JVM.java index 161dbddf4d8..2104511b4f2 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/JVM.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/JVM.java @@ -42,12 +42,15 @@ public class JVM { private static final boolean ibmvendor = System.getProperty("java.vendor") != null && System.getProperty("java.vendor").contains("IBM"); - private static final boolean windows = - System.getProperty("os.name") != null && System.getProperty("os.name").startsWith("Windows"); - private static final boolean linux = - System.getProperty("os.name") != null && System.getProperty("os.name").startsWith("Linux"); + // At least on my systems os.name reports as "linux", not "Linux". Prefer case insensitive tests. + private static final boolean windows = System.getProperty("os.name") != null + && System.getProperty("os.name").toLowerCase().contains("windows"); + private static final boolean linux = System.getProperty("os.name") != null + && System.getProperty("os.name").toLowerCase().contains("linux"); private static final boolean amd64 = System.getProperty("os.arch") != null && System.getProperty("os.arch").contains("amd64"); + private static final boolean aarch64 = + System.getProperty("os.arch") != null && System.getProperty("os.arch").contains("aarch64"); private static final String JVMVersion = System.getProperty("java.version"); @@ -99,6 +102,14 @@ public class JVM { return amd64; } + /** + * Check if the arch is aarch64; + * @return whether this is aarch64 or not. + */ + public static boolean isAarch64() { + return aarch64; + } + /** * Check if the finish() method of GZIPOutputStream is broken * @return whether GZIPOutputStream.finish() is broken. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java index 2b7c0ffb302..a08d095d3fe 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java @@ -53,10 +53,7 @@ import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; import org.apache.hbase.thirdparty.io.netty.channel.ServerChannel; import org.apache.hbase.thirdparty.io.netty.channel.group.ChannelGroup; import org.apache.hbase.thirdparty.io.netty.channel.group.DefaultChannelGroup; -import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; -import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioServerSocketChannel; import org.apache.hbase.thirdparty.io.netty.handler.codec.FixedLengthFrameDecoder; -import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory; import org.apache.hbase.thirdparty.io.netty.util.concurrent.GlobalEventExecutor; /** @@ -67,14 +64,6 @@ import org.apache.hbase.thirdparty.io.netty.util.concurrent.GlobalEventExecutor; public class NettyRpcServer extends RpcServer { public static final Logger LOG = LoggerFactory.getLogger(NettyRpcServer.class); - /** - * Name of property to change netty rpc server eventloop thread count. Default is 0. Tests may set - * this down from unlimited. - */ - public static final String HBASE_NETTY_EVENTLOOP_RPCSERVER_THREADCOUNT_KEY = - "hbase.netty.eventloop.rpcserver.thread.count"; - private static final int EVENTLOOP_THREADCOUNT_DEFAULT = 0; - /** * Name of property to change the byte buf allocator for the netty channels. Default is no value, * which causes us to use PooledByteBufAllocator. Valid settings here are "pooled", "unpooled", @@ -103,21 +92,16 @@ public class NettyRpcServer extends RpcServer { super(server, name, services, bindAddress, conf, scheduler, reservoirEnabled); this.bindAddress = bindAddress; this.channelAllocator = getChannelAllocator(conf); - EventLoopGroup eventLoopGroup; - Class channelClass; + // Get the event loop group configuration from the server class if available. + NettyEventLoopGroupConfig config = null; if (server instanceof HRegionServer) { - NettyEventLoopGroupConfig config = ((HRegionServer) server).getEventLoopGroupConfig(); - eventLoopGroup = config.group(); - channelClass = config.serverChannelClass(); - } else { - int threadCount = server == null - ? EVENTLOOP_THREADCOUNT_DEFAULT - : server.getConfiguration().getInt(HBASE_NETTY_EVENTLOOP_RPCSERVER_THREADCOUNT_KEY, - EVENTLOOP_THREADCOUNT_DEFAULT); - eventLoopGroup = new NioEventLoopGroup(threadCount, - new DefaultThreadFactory("NettyRpcServer", true, Thread.MAX_PRIORITY)); - channelClass = NioServerSocketChannel.class; + config = ((HRegionServer) server).getEventLoopGroupConfig(); } + if (config == null) { + config = new NettyEventLoopGroupConfig(conf, "NettyRpcServer"); + } + EventLoopGroup eventLoopGroup = config.group(); + Class channelClass = config.serverChannelClass(); ServerBootstrap bootstrap = new ServerBootstrap().group(eventLoopGroup).channel(channelClass) .childOption(ChannelOption.TCP_NODELAY, tcpNoDelay) .childOption(ChannelOption.SO_KEEPALIVE, tcpKeepAlive) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java index 4ff401633ab..b487cf40697 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java @@ -37,6 +37,13 @@ import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory */ @InterfaceAudience.Private public class NettyEventLoopGroupConfig { + + public static final String NETTY_WORKER_COUNT_KEY = "hbase.netty.worker.count"; + public static final int DEFAULT_NETTY_WORKER_COUNT = 0; + + public static final String NETTY_NATIVETRANSPORT_KEY = "hbase.netty.nativetransport"; + public static final boolean DEFAULT_NETTY_NATIVETRANSPORT = true; + private final EventLoopGroup group; private final Class serverChannelClass; @@ -45,14 +52,21 @@ public class NettyEventLoopGroupConfig { private static boolean useEpoll(Configuration conf) { // Config to enable native transport. - boolean epollEnabled = conf.getBoolean("hbase.netty.nativetransport", true); - // Use the faster native epoll transport mechanism on linux if enabled - return epollEnabled && JVM.isLinux() && JVM.isAmd64(); + final boolean epollEnabled = + conf.getBoolean(NETTY_NATIVETRANSPORT_KEY, DEFAULT_NETTY_NATIVETRANSPORT); + // Use the faster native epoll transport mechanism on linux if enabled and the + // hardware architecture is either amd64 or aarch64. Netty is known to have native + // epoll support for these combinations. + return epollEnabled && JVM.isLinux() && (JVM.isAmd64() || JVM.isAarch64()); } public NettyEventLoopGroupConfig(Configuration conf, String threadPoolName) { - boolean useEpoll = useEpoll(conf); - int workerCount = conf.getInt("hbase.netty.worker.count", 0); + final boolean useEpoll = useEpoll(conf); + final int workerCount = conf.getInt(NETTY_WORKER_COUNT_KEY, + // For backwards compatibility we also need to consider + // "hbase.netty.eventloop.rpcserver.thread.count" + // if it is defined in site configuration instead. + conf.getInt("hbase.netty.eventloop.rpcserver.thread.count", DEFAULT_NETTY_WORKER_COUNT)); ThreadFactory eventLoopThreadFactory = new DefaultThreadFactory(threadPoolName, true, Thread.MAX_PRIORITY); if (useEpoll) { diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml index 5e64bfcd907..0b6f1d59a0e 100644 --- a/hbase-server/src/test/resources/hbase-site.xml +++ b/hbase-server/src/test/resources/hbase-site.xml @@ -277,9 +277,4 @@ 3 Default is unbounded - - hbase.netty.eventloop.rpcserver.thread.count - 3 - Default is unbounded -