Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-23956 Use less resources running tests (#1266) 

Add being able to configure netty thread counts. Enable socket reuse
(should not have any impact).

hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
 Rename the threads we create in here so they are NOT named same was
 threads created by Hadoop RPC.

hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
 Allow configuring eventloopgroup thread count (so can override for
 tests)

hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java
 Enable socket resuse.

hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java
 Enable socket resuse and config for how many threads to use.

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
 Thread name edit; drop the redundant 'Thread' suffix.

hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
 Make closeable and shutdown executor when called.

hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
 Call close on HFileReplicator

hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
 HDFS creates lots of threads. Use less of it so less threads overall.

hbase-server/src/test/resources/hbase-site.xml
hbase-server/src/test/resources/hdfs-site.xml
 Constrain resources when running in test context.

hbase-server/src/test/resources/log4j.properties
 Enable debug on netty to see netty configs in our log

pom.xml
 Add system properties when we launch JVMs to constrain thread counts in
 tests

 Signed-off-by: Duo Zhang <zhangduo@apache.org>
This commit is contained in:
Michael Stack 2020-03-11 10:25:11 -07:00 committed by stack
parent d019fd2eed
commit aaf9a148a0
35 changed files with 1764 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
hbase-backup/src/test/resources/hbase-site.xml
View File

@ -21,6 +21,22 @@
*/
-->
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
<value>100</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
</description>
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
<value>1000</value>
<value>100</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as hbase:meta scanner and log roller.
</description>
</property>
<property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
@ -29,4 +45,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>3</value>
<description>Default is 30</description>
</property>
<property>
<name>hbase.regionserver.metahandler.count</name>
<value>3</value>
<description>Default is 20</description>
</property>
<property>
<name>hbase.netty.worker.count</name>
<value>3</value>
<description>Default is 0</description>
</property>
<property>
<name>hbase.hconnection.threads.max</name>
<value>6</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.htable.threads.max</name>
<value>3</value>
<description>Default is MAX_INTEGER</description>
</property>
<property>
<name>hbase.region.replica.replication.threads.max</name>
<value>7</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.rest.threads.max</name>
<value>5</value>
<description>Default is 100</description>
</property>
<property>
<name>hbase.replication.bulkload.copy.maxthreads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.loadincremental.threads.max</name>
<value>1</value>
<description>Default is # of CPUs</description>
</property>
<property>
<name>hbase.hstore.flusher.count</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.oldwals.cleaner.thread.size</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.master.procedure.threads</name>
<value>5</value>
<description>Default is at least 16</description>
</property>
<property>
<name>hbase.procedure.remote.dispatcher.threadpool.size</name>
<value>3</value>
<description>Default is 128</description>
</property>
<property>
<name>hbase.regionserver.executor.closeregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openpriorityregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.storescanner.parallel.seek.threads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hfile.compaction.discharger.thread.count</name>
<value>1</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.regionserver.executor.refresh.peer.threads</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.hregion.open.and.init.threads.max</name>
<value>3</value>
<description>Default is 16 or # of Regions</description>
</property>
<property>
<name>hbase.master.handler.count</name>
<value>7</value>
<description>Default is 25</description>
</property>
<property>
<name>hbase.replication.source.maxthreads</name>
<value></value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hconnection.meta.lookup.threads.max</name>
<value>5</value>
<description>Default is 128</description>
</property>
</configuration>

2
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
View File

@ -233,7 +233,7 @@ class BlockingRpcConnection extends RpcConnection implements Runnable {
this.connectionHeaderWithLength = baos.getBuffer();
UserGroupInformation ticket = remoteId.ticket.getUGI();
this.threadName = "IPC Client (" + this.rpcClient.socketFactory.hashCode() + ") connection to "
this.threadName = "BRPC Connection (" + this.rpcClient.socketFactory.hashCode() + ") to "
+ remoteId.getAddress().toString()
+ ((ticket == null) ? " from an unknown user" : (" from " + ticket.getUserName()));

40
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java
View File

@ -1,40 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.ipc;
import org.apache.hbase.thirdparty.io.netty.channel.Channel;
import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.util.Pair;
/**
* The default netty event loop config
*/
@InterfaceAudience.Private
class DefaultNettyEventLoopConfig {
public static final Pair<EventLoopGroup, Class<? extends Channel>> GROUP_AND_CHANNEL_CLASS = Pair
.<EventLoopGroup, Class<? extends Channel>> newPair(
new NioEventLoopGroup(0,
new DefaultThreadFactory("Default-IPC-NioEventLoopGroup", true, Thread.MAX_PRIORITY)),
NioSocketChannel.class);
}

13
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java
View File

@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -54,12 +54,15 @@ public class NettyRpcClient extends AbstractRpcClient<NettyRpcConnection> {
public NettyRpcClient(Configuration configuration, String clusterId, SocketAddress localAddress,
MetricsConnection metrics) {
super(configuration, clusterId, localAddress, metrics);
Pair<EventLoopGroup, Class<? extends Channel>> groupAndChannelClass = NettyRpcClientConfigHelper
.getEventLoopConfig(conf);
Pair<EventLoopGroup, Class<? extends Channel>> groupAndChannelClass =
NettyRpcClientConfigHelper.getEventLoopConfig(conf);
if (groupAndChannelClass == null) {
// Use our own EventLoopGroup.
this.group = new NioEventLoopGroup(0,
new DefaultThreadFactory("IPC-NioEventLoopGroup", true, Thread.MAX_PRIORITY));
int threadCount = conf.getInt(
NettyRpcClientConfigHelper.HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY, 0);
this.group = new NioEventLoopGroup(threadCount,
new DefaultThreadFactory("RPCClient(own)-NioEventLoopGroup", true,
Thread.NORM_PRIORITY));
this.channelClass = NioSocketChannel.class;
this.shutdownGroupWhenClose = true;
} else {

36
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java
View File

@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -17,18 +17,18 @@
*/
package org.apache.hadoop.hbase.ipc;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
import org.apache.hbase.thirdparty.io.netty.channel.Channel;
import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
import org.apache.hbase.thirdparty.io.netty.channel.Channel;
import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
/**
* Helper class for passing config to {@link NettyRpcClient}.
@ -39,15 +39,27 @@ import org.apache.hadoop.hbase.util.Pair;
* @since 2.0.0
*/
@InterfaceAudience.Public
public class NettyRpcClientConfigHelper {
public final class NettyRpcClientConfigHelper {
public static final String EVENT_LOOP_CONFIG = "hbase.rpc.client.event-loop.config";
/**
* Name of property to change netty rpc client eventloop thread count. Default is 0.
* Tests may set this down from unlimited.
*/
public static final String HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY =
"hbase.netty.eventloop.rpcclient.thread.count";
private static final String CONFIG_NAME = "global-event-loop";
private static final Map<String, Pair<EventLoopGroup, Class<? extends Channel>>>
EVENT_LOOP_CONFIG_MAP = new HashMap<>();
/**
* Shutdown constructor.
*/
private NettyRpcClientConfigHelper() {}
/**
* Set the EventLoopGroup and channel class for {@code AsyncRpcClient}.
*/
@ -71,12 +83,14 @@ public class NettyRpcClientConfigHelper {
static Pair<EventLoopGroup, Class<? extends Channel>> getEventLoopConfig(Configuration conf) {
String name = conf.get(EVENT_LOOP_CONFIG);
if (name == null) {
return DefaultNettyEventLoopConfig.GROUP_AND_CHANNEL_CLASS;
int threadCount = conf.getInt(HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY, 0);
return new Pair<>(new NioEventLoopGroup(threadCount,
new DefaultThreadFactory("RPCClient-NioEventLoopGroup", true,
Thread.NORM_PRIORITY)), NioSocketChannel.class);
}
if (StringUtils.isBlank(name)) {
return null;
}
return EVENT_LOOP_CONFIG_MAP.get(name);
}
}

131
hbase-client/src/test/resources/hbase-site.xml
View File

@ -21,6 +21,22 @@
*/
-->
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
<value>100</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
</description>
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
<value>1000</value>
<value>100</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as hbase:meta scanner and log roller.
</description>
</property>
<property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
@ -29,4 +45,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>3</value>
<description>Default is 30</description>
</property>
<property>
<name>hbase.regionserver.metahandler.count</name>
<value>3</value>
<description>Default is 20</description>
</property>
<property>
<name>hbase.netty.worker.count</name>
<value>3</value>
<description>Default is 0</description>
</property>
<property>
<name>hbase.hconnection.threads.max</name>
<value>6</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.htable.threads.max</name>
<value>3</value>
<description>Default is MAX_INTEGER</description>
</property>
<property>
<name>hbase.region.replica.replication.threads.max</name>
<value>7</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.rest.threads.max</name>
<value>5</value>
<description>Default is 100</description>
</property>
<property>
<name>hbase.replication.bulkload.copy.maxthreads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.loadincremental.threads.max</name>
<value>1</value>
<description>Default is # of CPUs</description>
</property>
<property>
<name>hbase.hstore.flusher.count</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.oldwals.cleaner.thread.size</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.master.procedure.threads</name>
<value>5</value>
<description>Default is at least 16</description>
</property>
<property>
<name>hbase.procedure.remote.dispatcher.threadpool.size</name>
<value>3</value>
<description>Default is 128</description>
</property>
<property>
<name>hbase.regionserver.executor.closeregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openpriorityregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.storescanner.parallel.seek.threads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hfile.compaction.discharger.thread.count</name>
<value>1</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.regionserver.executor.refresh.peer.threads</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.hregion.open.and.init.threads.max</name>
<value>3</value>
<description>Default is 16 or # of Regions</description>
</property>
<property>
<name>hbase.master.handler.count</name>
<value>7</value>
<description>Default is 25</description>
</property>
<property>
<name>hbase.replication.source.maxthreads</name>
<value></value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hconnection.meta.lookup.threads.max</name>
<value>5</value>
<description>Default is 128</description>
</property>
</configuration>

56
hbase-client/src/test/resources/hdfs-site.xml Normal file
View File

@ -0,0 +1,56 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<!-- hadoop-2.0.5+'s HDFS-4305 by default enforces a min blocks size
of 1024*1024. Many unit tests that use the hlog use smaller
blocks. Setting this config to 0 to have tests pass -->
<property>
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
<property>
<name>dfs.datanode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.service.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<!--
Constraining this config makes tests fail.
<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>16</value>
<description>Default is 4096. If constrain this
too much, tests do not complete.</description>
</property>
-->
</configuration>

131
hbase-endpoint/src/test/resources/hbase-site.xml
View File

@ -21,6 +21,22 @@
*/
-->
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
<value>100</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
</description>
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
<value>1000</value>
<value>100</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as hbase:meta scanner and log roller.
</description>
</property>
<property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
@ -29,4 +45,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>3</value>
<description>Default is 30</description>
</property>
<property>
<name>hbase.regionserver.metahandler.count</name>
<value>3</value>
<description>Default is 20</description>
</property>
<property>
<name>hbase.netty.worker.count</name>
<value>3</value>
<description>Default is 0</description>
</property>
<property>
<name>hbase.hconnection.threads.max</name>
<value>6</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.htable.threads.max</name>
<value>3</value>
<description>Default is MAX_INTEGER</description>
</property>
<property>
<name>hbase.region.replica.replication.threads.max</name>
<value>7</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.rest.threads.max</name>
<value>5</value>
<description>Default is 100</description>
</property>
<property>
<name>hbase.replication.bulkload.copy.maxthreads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.loadincremental.threads.max</name>
<value>1</value>
<description>Default is # of CPUs</description>
</property>
<property>
<name>hbase.hstore.flusher.count</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.oldwals.cleaner.thread.size</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.master.procedure.threads</name>
<value>5</value>
<description>Default is at least 16</description>
</property>
<property>
<name>hbase.procedure.remote.dispatcher.threadpool.size</name>
<value>3</value>
<description>Default is 128</description>
</property>
<property>
<name>hbase.regionserver.executor.closeregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openpriorityregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.storescanner.parallel.seek.threads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hfile.compaction.discharger.thread.count</name>
<value>1</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.regionserver.executor.refresh.peer.threads</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.hregion.open.and.init.threads.max</name>
<value>3</value>
<description>Default is 16 or # of Regions</description>
</property>
<property>
<name>hbase.master.handler.count</name>
<value>7</value>
<description>Default is 25</description>
</property>
<property>
<name>hbase.replication.source.maxthreads</name>
<value></value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hconnection.meta.lookup.threads.max</name>
<value>5</value>
<description>Default is 128</description>
</property>
</configuration>

56
hbase-endpoint/src/test/resources/hdfs-site.xml Normal file
View File

@ -0,0 +1,56 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<!-- hadoop-2.0.5+'s HDFS-4305 by default enforces a min blocks size
of 1024*1024. Many unit tests that use the hlog use smaller
blocks. Setting this config to 0 to have tests pass -->
<property>
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
<property>
<name>dfs.datanode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.service.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<!--
Constraining this config makes tests fail.
<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>16</value>
<description>Default is 4096. If constrain this
too much, tests do not complete.</description>
</property>
-->
</configuration>

1
hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java
View File

@ -229,6 +229,7 @@ public class HttpProxyExample {
channelGroup = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE);
serverChannel = new ServerBootstrap().group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class).childOption(ChannelOption.TCP_NODELAY, true)
.childOption(ChannelOption.SO_REUSEADDR, true)
.childHandler(new ChannelInitializer<Channel>() {
@Override

131
hbase-examples/src/test/resources/hbase-site.xml
View File

@ -21,8 +21,139 @@
*/
-->
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
<value>100</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
</description>
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
<value>1000</value>
<value>100</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as hbase:meta scanner and log roller.
</description>
</property>
<property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>3</value>
<description>Default is 30</description>
</property>
<property>
<name>hbase.regionserver.metahandler.count</name>
<value>3</value>
<description>Default is 20</description>
</property>
<property>
<name>hbase.netty.worker.count</name>
<value>3</value>
<description>Default is 0</description>
</property>
<property>
<name>hbase.hconnection.threads.max</name>
<value>6</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.htable.threads.max</name>
<value>3</value>
<description>Default is MAX_INTEGER</description>
</property>
<property>
<name>hbase.region.replica.replication.threads.max</name>
<value>7</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.rest.threads.max</name>
<value>5</value>
<description>Default is 100</description>
</property>
<property>
<name>hbase.replication.bulkload.copy.maxthreads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.loadincremental.threads.max</name>
<value>1</value>
<description>Default is # of CPUs</description>
</property>
<property>
<name>hbase.hstore.flusher.count</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.oldwals.cleaner.thread.size</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.master.procedure.threads</name>
<value>5</value>
<description>Default is at least 16</description>
</property>
<property>
<name>hbase.procedure.remote.dispatcher.threadpool.size</name>
<value>3</value>
<description>Default is 128</description>
</property>
<property>
<name>hbase.regionserver.executor.closeregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openpriorityregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.storescanner.parallel.seek.threads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hfile.compaction.discharger.thread.count</name>
<value>1</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.regionserver.executor.refresh.peer.threads</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.hregion.open.and.init.threads.max</name>
<value>3</value>
<description>Default is 16 or # of Regions</description>
</property>
<property>
<name>hbase.master.handler.count</name>
<value>7</value>
<description>Default is 25</description>
</property>
<property>
<name>hbase.replication.source.maxthreads</name>
<value></value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hconnection.meta.lookup.threads.max</name>
<value>5</value>
<description>Default is 128</description>
</property>
</configuration>

56
hbase-examples/src/test/resources/hdfs-site.xml Normal file
View File

@ -0,0 +1,56 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<!-- hadoop-2.0.5+'s HDFS-4305 by default enforces a min blocks size
of 1024*1024. Many unit tests that use the hlog use smaller
blocks. Setting this config to 0 to have tests pass -->
<property>
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
<property>
<name>dfs.datanode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.service.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<!--
Constraining this config makes tests fail.
<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>16</value>
<description>Default is 4096. If constrain this
too much, tests do not complete.</description>
</property>
-->
</configuration>

131
hbase-mapreduce/src/test/resources/hbase-site.xml
View File

@ -21,6 +21,22 @@
*/
-->
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
<value>100</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
</description>
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
<value>1000</value>
<value>100</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as hbase:meta scanner and log roller.
</description>
</property>
<property>
<name>hbase.regionserver.msginterval</name>
<value>1000</value>
@ -158,4 +174,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>3</value>
<description>Default is 30</description>
</property>
<property>
<name>hbase.regionserver.metahandler.count</name>
<value>3</value>
<description>Default is 20</description>
</property>
<property>
<name>hbase.netty.worker.count</name>
<value>3</value>
<description>Default is 0</description>
</property>
<property>
<name>hbase.hconnection.threads.max</name>
<value>6</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.htable.threads.max</name>
<value>3</value>
<description>Default is MAX_INTEGER</description>
</property>
<property>
<name>hbase.region.replica.replication.threads.max</name>
<value>7</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.rest.threads.max</name>
<value>5</value>
<description>Default is 100</description>
</property>
<property>
<name>hbase.replication.bulkload.copy.maxthreads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.loadincremental.threads.max</name>
<value>1</value>
<description>Default is # of CPUs</description>
</property>
<property>
<name>hbase.hstore.flusher.count</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.oldwals.cleaner.thread.size</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.master.procedure.threads</name>
<value>5</value>
<description>Default is at least 16</description>
</property>
<property>
<name>hbase.procedure.remote.dispatcher.threadpool.size</name>
<value>3</value>
<description>Default is 128</description>
</property>
<property>
<name>hbase.regionserver.executor.closeregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openpriorityregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.storescanner.parallel.seek.threads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hfile.compaction.discharger.thread.count</name>
<value>1</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.regionserver.executor.refresh.peer.threads</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.hregion.open.and.init.threads.max</name>
<value>3</value>
<description>Default is 16 or # of Regions</description>
</property>
<property>
<name>hbase.master.handler.count</name>
<value>7</value>
<description>Default is 25</description>
</property>
<property>
<name>hbase.replication.source.maxthreads</name>
<value></value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hconnection.meta.lookup.threads.max</name>
<value>5</value>
<description>Default is 128</description>
</property>
</configuration>

26
hbase-mapreduce/src/test/resources/hdfs-site.xml
View File

@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
</configuration>
<property>
<name>dfs.datanode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.service.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<!--
Constraining this config makes tests fail.
<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>16</value>
<description>Default is 4096. If constrain this
too much, tests do not complete.</description>
</property>
-->
</configuration>

131
hbase-procedure/src/test/resources/hbase-site.xml
View File

@ -21,6 +21,22 @@
*/
-->
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
<value>100</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
</description>
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
<value>1000</value>
<value>100</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as hbase:meta scanner and log roller.
</description>
</property>
<property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
@ -45,4 +61,119 @@
WARNING: Doing so may expose you to additional risk of data loss!
</description>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>3</value>
<description>Default is 30</description>
</property>
<property>
<name>hbase.regionserver.metahandler.count</name>
<value>3</value>
<description>Default is 20</description>
</property>
<property>
<name>hbase.netty.worker.count</name>
<value>3</value>
<description>Default is 0</description>
</property>
<property>
<name>hbase.hconnection.threads.max</name>
<value>6</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.htable.threads.max</name>
<value>3</value>
<description>Default is MAX_INTEGER</description>
</property>
<property>
<name>hbase.region.replica.replication.threads.max</name>
<value>7</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.rest.threads.max</name>
<value>5</value>
<description>Default is 100</description>
</property>
<property>
<name>hbase.replication.bulkload.copy.maxthreads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.loadincremental.threads.max</name>
<value>1</value>
<description>Default is # of CPUs</description>
</property>
<property>
<name>hbase.hstore.flusher.count</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.oldwals.cleaner.thread.size</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.master.procedure.threads</name>
<value>5</value>
<description>Default is at least 16</description>
</property>
<property>
<name>hbase.procedure.remote.dispatcher.threadpool.size</name>
<value>3</value>
<description>Default is 128</description>
</property>
<property>
<name>hbase.regionserver.executor.closeregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openpriorityregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.storescanner.parallel.seek.threads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hfile.compaction.discharger.thread.count</name>
<value>1</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.regionserver.executor.refresh.peer.threads</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.hregion.open.and.init.threads.max</name>
<value>3</value>
<description>Default is 16 or # of Regions</description>
</property>
<property>
<name>hbase.master.handler.count</name>
<value>7</value>
<description>Default is 25</description>
</property>
<property>
<name>hbase.replication.source.maxthreads</name>
<value></value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hconnection.meta.lookup.threads.max</name>
<value>5</value>
<description>Default is 128</description>
</property>
</configuration>

56
hbase-procedure/src/test/resources/hdfs-site.xml Normal file
View File

@ -0,0 +1,56 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<!-- hadoop-2.0.5+'s HDFS-4305 by default enforces a min blocks size
of 1024*1024. Many unit tests that use the hlog use smaller
blocks. Setting this config to 0 to have tests pass -->
<property>
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
<property>
<name>dfs.datanode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.service.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<!--
Constraining this config makes tests fail.
<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>16</value>
<description>Default is 4096. If constrain this
too much, tests do not complete.</description>
</property>
-->
</configuration>

26
hbase-rest/src/test/resources/hdfs-site.xml
View File

@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
</configuration>
<property>
<name>dfs.datanode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.service.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<!--
Constraining this config makes tests fail.
<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>16</value>
<description>Default is 4096. If constrain this
too much, tests do not complete.</description>
</property>
-->
</configuration>

56
hbase-rsgroup/src/test/resources/hdfs-site.xml Normal file
View File

@ -0,0 +1,56 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<!-- hadoop-2.0.5+'s HDFS-4305 by default enforces a min blocks size
of 1024*1024. Many unit tests that use the hlog use smaller
blocks. Setting this config to 0 to have tests pass -->
<property>
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
<property>
<name>dfs.datanode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.service.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<!--
Constraining this config makes tests fail.
<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>16</value>
<description>Default is 4096. If constrain this
too much, tests do not complete.</description>
</property>
-->
</configuration>

65
hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java
View File

@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -17,6 +17,28 @@
*/
package org.apache.hadoop.hbase.ipc;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.InetSocketAddress;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.security.HBasePolicyProvider;
import org.apache.hadoop.hbase.util.NettyEventLoopGroupConfig;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.protobuf.BlockingService;
import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor;
import org.apache.hbase.thirdparty.com.google.protobuf.Message;
import org.apache.hbase.thirdparty.io.netty.bootstrap.ServerBootstrap;
import org.apache.hbase.thirdparty.io.netty.channel.Channel;
import org.apache.hbase.thirdparty.io.netty.channel.ChannelInitializer;
@ -32,39 +54,22 @@ import org.apache.hbase.thirdparty.io.netty.handler.codec.FixedLengthFrameDecode
import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
import org.apache.hbase.thirdparty.io.netty.util.concurrent.GlobalEventExecutor;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.InetSocketAddress;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.Server;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.security.HBasePolicyProvider;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.protobuf.BlockingService;
import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor;
import org.apache.hbase.thirdparty.com.google.protobuf.Message;
import org.apache.hadoop.hbase.util.NettyEventLoopGroupConfig;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
/**
* An RPC server with Netty4 implementation.
* @since 2.0.0
*/
@InterfaceAudience.LimitedPrivate({HBaseInterfaceAudience.CONFIG})
public class NettyRpcServer extends RpcServer {
public static final Logger LOG = LoggerFactory.getLogger(NettyRpcServer.class);
/**
* Name of property to change netty rpc server eventloop thread count. Default is 0.
* Tests may set this down from unlimited.
*/
public static final String HBASE_NETTY_EVENTLOOP_RPCSERVER_THREADCOUNT_KEY =
"hbase.netty.eventloop.rpcserver.thread.count";
private static final int EVENTLOOP_THREADCOUNT_DEFAULT = 0;
private final InetSocketAddress bindAddress;
private final CountDownLatch closed = new CountDownLatch(1);
@ -84,13 +89,17 @@ public class NettyRpcServer extends RpcServer {
eventLoopGroup = config.group();
channelClass = config.serverChannelClass();
} else {
eventLoopGroup = new NioEventLoopGroup(0,
new DefaultThreadFactory("NettyRpcServer", true, Thread.MAX_PRIORITY));
int threadCount = server == null? EVENTLOOP_THREADCOUNT_DEFAULT:
server.getConfiguration().getInt(HBASE_NETTY_EVENTLOOP_RPCSERVER_THREADCOUNT_KEY,
EVENTLOOP_THREADCOUNT_DEFAULT);
eventLoopGroup = new NioEventLoopGroup(threadCount,
new DefaultThreadFactory("NettyRpcServer", true, Thread.MAX_PRIORITY));
channelClass = NioServerSocketChannel.class;
}
ServerBootstrap bootstrap = new ServerBootstrap().group(eventLoopGroup).channel(channelClass)
.childOption(ChannelOption.TCP_NODELAY, tcpNoDelay)
.childOption(ChannelOption.SO_KEEPALIVE, tcpKeepAlive)
.childOption(ChannelOption.SO_REUSEADDR, true)
.childHandler(new ChannelInitializer<Channel>() {
@Override

2
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -1691,7 +1691,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
if (!stores.isEmpty()) {
// initialize the thread pool for closing stores in parallel.
ThreadPoolExecutor storeCloserThreadPool =
getStoreOpenAndCloseThreadPool("StoreCloserThread-" +
getStoreOpenAndCloseThreadPool("StoreCloser-" +
getRegionInfo().getRegionNameAsString());
CompletionService<Pair<byte[], Collection<HStoreFile>>> completionService =
new ExecutorCompletionService<>(storeCloserThreadPool);

4
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
View File

@ -569,7 +569,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
}
// initialize the thread pool for opening store files in parallel..
ThreadPoolExecutor storeFileOpenerThreadPool =
this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-"
this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpener-"
+ this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName());
CompletionService<HStoreFile> completionService =
new ExecutorCompletionService<>(storeFileOpenerThreadPool);
@ -969,7 +969,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
if (!result.isEmpty()) {
// initialize the thread pool for closing store files in parallel.
ThreadPoolExecutor storeFileCloserThreadPool = this.region
.getStoreFileOpenAndCloseThreadPool("StoreFileCloserThread-"
.getStoreFileOpenAndCloseThreadPool("StoreFileCloser-"
+ this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName());
// close each store file in parallel

6
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
View File

@ -228,8 +228,10 @@ public class AsyncFSWAL extends AbstractFSWAL<AsyncWriter> {
}
} else {
ThreadPoolExecutor threadPool =
new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(),
new ThreadFactoryBuilder().setNameFormat("AsyncFSWAL-%d").setDaemon(true).build());
new ThreadPoolExecutor(1, 1, 0L,
TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(),
new ThreadFactoryBuilder().setNameFormat("AsyncFSWAL-%d-" + rootDir.toString()).
setDaemon(true).build());
hasConsumerTask = () -> threadPool.getQueue().peek() == consumer;
this.consumeExecutor = threadPool;
}

27
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
View File

@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
@ -55,16 +56,16 @@ import org.apache.hadoop.hbase.util.Threads;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* It is used for replicating HFile entries. It will first copy parallely all the hfiles to a local
* staging directory and then it will use ({@link BulkLoadHFiles} to prepare a collection of
* {@link LoadQueueItem} which will finally be loaded(replicated) into the table of this cluster.
* Call {@link #close()} when done.
*/
@InterfaceAudience.Private
public class HFileReplicator {
public class HFileReplicator implements Closeable {
/** Maximum number of threads to allow in pool to copy hfiles during replication */
public static final String REPLICATION_BULKLOAD_COPY_MAXTHREADS_KEY =
"hbase.replication.bulkload.copy.maxthreads";
@ -113,7 +114,8 @@ public class HFileReplicator {
REPLICATION_BULKLOAD_COPY_MAXTHREADS_DEFAULT);
this.exec = Threads.getBoundedCachedThreadPool(maxCopyThreads, 60, TimeUnit.SECONDS,
new ThreadFactoryBuilder().setDaemon(true)
.setNameFormat("HFileReplicationCallable-%1$d").build());
.setNameFormat("HFileReplicationCopier-%1$d-" + this.sourceBaseNamespaceDirPath).
build());
this.copiesPerThread =
conf.getInt(REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_KEY,
REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_DEFAULT);
@ -121,6 +123,13 @@ public class HFileReplicator {
sinkFs = FileSystem.get(conf);
}
@Override
public void close() throws IOException {
if (this.exec != null) {
this.exec.shutdown();
}
}
public Void replicate() throws IOException {
// Copy all the hfiles to the local file system
Map<String, Path> tableStagingDirsMap = copyHFilesToStagingDir();
@ -130,7 +139,6 @@ public class HFileReplicator {
for (Entry<String, Path> tableStagingDir : tableStagingDirsMap.entrySet()) {
String tableNameString = tableStagingDir.getKey();
Path stagingDir = tableStagingDir.getValue();
TableName tableName = TableName.valueOf(tableNameString);
// Prepare collection of queue of hfiles to be loaded(replicated)
@ -139,8 +147,7 @@ public class HFileReplicator {
false);
if (queue.isEmpty()) {
LOG.warn("Replication process did not find any files to replicate in directory "
+ stagingDir.toUri());
LOG.warn("Did not find any files to replicate in directory {}", stagingDir.toUri());
return null;
}
fsDelegationToken.acquireDelegationToken(sinkFs);
@ -162,13 +169,11 @@ public class HFileReplicator {
loader.setClusterIds(sourceClusterIds);
for (int count = 0; !queue.isEmpty(); count++) {
if (count != 0) {
LOG.warn("Error occurred while replicating HFiles, retry attempt " + count + " with " +
queue.size() + " files still remaining to replicate.");
LOG.warn("Error replicating HFiles; retry={} with {} remaining.", count, queue.size());
}
if (maxRetries != 0 && count >= maxRetries) {
throw new IOException(
"Retry attempted " + count + " times without completing, bailing out.");
throw new IOException("Retry attempted " + count + " times without completing, bailing.");
}
// Try bulk load

35
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
View File

@ -57,7 +57,6 @@ import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor;
@ -151,7 +150,7 @@ public class ReplicationSink {
if (this.conf.get(HConstants.CLIENT_ZOOKEEPER_QUORUM) != null) {
this.conf.unset(HConstants.CLIENT_ZOOKEEPER_QUORUM);
}
}
}
/**
* Replicate this array of entries directly into the local cluster using the native client. Only
@ -166,7 +165,9 @@ public class ReplicationSink {
public void replicateEntries(List<WALEntry> entries, final CellScanner cells,
String replicationClusterId, String sourceBaseNamespaceDirPath,
String sourceHFileArchiveDirPath) throws IOException {
if (entries.isEmpty()) return;
if (entries.isEmpty()) {
return;
}
// Very simple optimization where we batch sequences of rows going
// to the same table.
try {
@ -259,18 +260,13 @@ public class ReplicationSink {
bulkLoadsPerClusters.entrySet()) {
Map<String, List<Pair<byte[], List<String>>>> bulkLoadHFileMap = entry.getValue();
if (bulkLoadHFileMap != null && !bulkLoadHFileMap.isEmpty()) {
if(LOG.isDebugEnabled()) {
LOG.debug("Started replicating bulk loaded data from cluster ids: {}.",
entry.getKey().toString());
}
HFileReplicator hFileReplicator =
new HFileReplicator(this.provider.getConf(this.conf, replicationClusterId),
LOG.debug("Replicating {} bulk loaded data", entry.getKey().toString());
Configuration providerConf = this.provider.getConf(this.conf, replicationClusterId);
try (HFileReplicator hFileReplicator = new HFileReplicator(providerConf,
sourceBaseNamespaceDirPath, sourceHFileArchiveDirPath, bulkLoadHFileMap, conf,
getConnection(), entry.getKey());
hFileReplicator.replicate();
if(LOG.isDebugEnabled()) {
LOG.debug("Finished replicating bulk loaded data from cluster id: {}",
entry.getKey().toString());
getConnection(), entry.getKey())) {
hFileReplicator.replicate();
LOG.debug("Finished replicating {} bulk loaded data", entry.getKey().toString());
}
}
}
@ -352,8 +348,6 @@ public class ReplicationSink {
}
/**
* @param previousCell
* @param cell
* @return True if we have crossed over onto a new row or type
*/
private boolean isNewRowOrType(final Cell previousCell, final Cell cell) {
@ -368,13 +362,10 @@ public class ReplicationSink {
/**
* Simple helper to a map from key to (a list of) values
* TODO: Make a general utility method
* @param map
* @param key1
* @param key2
* @param value
* @return the list of values corresponding to key1 and key2
*/
private <K1, K2, V> List<V> addToHashMultiMap(Map<K1, Map<K2,List<V>>> map, K1 key1, K2 key2, V value) {
private <K1, K2, V> List<V> addToHashMultiMap(Map<K1, Map<K2,List<V>>> map, K1 key1,
K2 key2, V value) {
Map<K2,List<V>> innerMap = map.get(key1);
if (innerMap == null) {
innerMap = new HashMap<>();
@ -450,7 +441,7 @@ public class ReplicationSink {
/**
* Get a string representation of this sink's metrics
* @return string with the total replicated edits count and the date
* of the last edit that was applied
* of the last edit that was applied
*/
public String getStats() {
long total = this.totalReplicatedEdits.get();

6
hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
View File

@ -110,7 +110,7 @@ public abstract class ModifyRegionUtils {
if (newRegions == null) return null;
int regionNumber = newRegions.length;
ThreadPoolExecutor exec = getRegionOpenAndInitThreadPool(conf,
"RegionOpenAndInitThread-" + tableDescriptor.getTableName(), regionNumber);
"RegionOpenAndInit-" + tableDescriptor.getTableName(), regionNumber);
try {
return createRegions(exec, conf, rootDir, tableDescriptor, newRegions, task);
} finally {
@ -230,8 +230,8 @@ public abstract class ModifyRegionUtils {
final String threadNamePrefix, int regionNumber) {
int maxThreads = Math.min(regionNumber, conf.getInt(
"hbase.hregion.open.and.init.threads.max", 16));
ThreadPoolExecutor regionOpenAndInitThreadPool = Threads
.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
ThreadPoolExecutor regionOpenAndInitThreadPool = Threads.
getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
Threads.newDaemonThreadFactory(threadNamePrefix));
return regionOpenAndInitThreadPool;
}

3
hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java
View File

@ -27,9 +27,7 @@ import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioServerSocketChannel;
import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
import java.util.concurrent.ThreadFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.yetus.audience.InterfaceAudience;
@ -38,7 +36,6 @@ import org.apache.yetus.audience.InterfaceAudience;
*/
@InterfaceAudience.Private
public class NettyEventLoopGroupConfig {
private final EventLoopGroup group;
private final Class<? extends ServerChannel> serverChannelClass;

4
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
View File

@ -79,8 +79,8 @@ public class TestReplicationBase {
protected static Configuration CONF1 = UTIL1.getConfiguration();
protected static Configuration CONF2 = UTIL2.getConfiguration();
protected static final int NUM_SLAVES1 = 2;
protected static final int NUM_SLAVES2 = 4;
protected static final int NUM_SLAVES1 = 1;
protected static final int NUM_SLAVES2 = 1;
protected static final int NB_ROWS_IN_BATCH = 100;
protected static final int NB_ROWS_IN_BIG_BATCH =
NB_ROWS_IN_BATCH * 10;

126
hbase-server/src/test/resources/hbase-site.xml
View File

@ -23,7 +23,7 @@
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
<value>1000</value>
<value>100</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
@ -35,7 +35,7 @@
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
<value>1000</value>
<value>100</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as hbase:meta scanner and log roller.
</description>
@ -48,11 +48,13 @@
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>5</value>
<value>3</value>
<description>Default is 30</description>
</property>
<property>
<name>hbase.regionserver.metahandler.count</name>
<value>6</value>
<value>3</value>
<description>Default is 20</description>
</property>
<property>
<name>hbase.ipc.server.read.threadpool.size</name>
@ -83,7 +85,6 @@
<name>hbase.ipc.client.fallback-to-simple-auth-allowed</name>
<value>true</value>
</property>
<property>
<name>hbase.regionserver.info.port</name>
<value>-1</value>
@ -158,4 +159,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.netty.worker.count</name>
<value>3</value>
<description>Default is 0</description>
</property>
<property>
<name>hbase.hconnection.threads.max</name>
<value>6</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.htable.threads.max</name>
<value>3</value>
<description>Default is MAX_INTEGER</description>
</property>
<property>
<name>hbase.region.replica.replication.threads.max</name>
<value>7</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.rest.threads.max</name>
<value>5</value>
<description>Default is 100</description>
</property>
<property>
<name>hbase.replication.bulkload.copy.maxthreads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.loadincremental.threads.max</name>
<value>1</value>
<description>Default is # of CPUs</description>
</property>
<property>
<name>hbase.hstore.flusher.count</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.oldwals.cleaner.thread.size</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.master.procedure.threads</name>
<value>5</value>
<description>Default is at least 16</description>
</property>
<property>
<name>hbase.procedure.remote.dispatcher.threadpool.size</name>
<value>3</value>
<description>Default is 128</description>
</property>
<property>
<name>hbase.regionserver.executor.closeregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openpriorityregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.storescanner.parallel.seek.threads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hfile.compaction.discharger.thread.count</name>
<value>1</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.regionserver.executor.refresh.peer.threads</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.hregion.open.and.init.threads.max</name>
<value>3</value>
<description>Default is 16 or # of Regions</description>
</property>
<property>
<name>hbase.master.handler.count</name>
<value>7</value>
<description>Default is 25</description>
</property>
<property>
<name>hbase.replication.source.maxthreads</name>
<value></value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hconnection.meta.lookup.threads.max</name>
<value>5</value>
<description>Default is 128</description>
</property>
<property>
<name>hbase.netty.eventloop.rpclient.thread.count</name>
<value>3</value>
<description>Default is unbounded</description>
</property>
<property>
<name>hbase.netty.eventloop.rpcserver.thread.count</name>
<value>3</value>
<description>Default is unbounded</description>
</property>
</configuration>

26
hbase-server/src/test/resources/hdfs-site.xml
View File

@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
</configuration>
<property>
<name>dfs.datanode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.service.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<!--
Constraining this config makes tests fail.
<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>16</value>
<description>Default is 4096. If constrain this
too much, tests do not complete.</description>
</property>
-->
</configuration>

1
hbase-server/src/test/resources/log4j.properties
View File

@ -66,6 +66,7 @@ log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN
log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN
log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN
log4j.logger.org.apache.hadoop.metrics2.util.MBeans=WARN
log4j.logger.io.netty.channel=DEBUG
# Enable this to get detailed connection error/retry logging.
# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE
log4j.logger.org.apache.directory=WARN

131
hbase-shell/src/test/resources/hbase-site.xml
View File

@ -21,6 +21,22 @@
*/
-->
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
<value>100</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
</description>
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
<value>1000</value>
<value>100</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as hbase:meta scanner and log roller.
</description>
</property>
<property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
@ -29,4 +45,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>3</value>
<description>Default is 30</description>
</property>
<property>
<name>hbase.regionserver.metahandler.count</name>
<value>3</value>
<description>Default is 20</description>
</property>
<property>
<name>hbase.netty.worker.count</name>
<value>3</value>
<description>Default is 0</description>
</property>
<property>
<name>hbase.hconnection.threads.max</name>
<value>6</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.htable.threads.max</name>
<value>3</value>
<description>Default is MAX_INTEGER</description>
</property>
<property>
<name>hbase.region.replica.replication.threads.max</name>
<value>7</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.rest.threads.max</name>
<value>5</value>
<description>Default is 100</description>
</property>
<property>
<name>hbase.replication.bulkload.copy.maxthreads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.loadincremental.threads.max</name>
<value>1</value>
<description>Default is # of CPUs</description>
</property>
<property>
<name>hbase.hstore.flusher.count</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.oldwals.cleaner.thread.size</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.master.procedure.threads</name>
<value>5</value>
<description>Default is at least 16</description>
</property>
<property>
<name>hbase.procedure.remote.dispatcher.threadpool.size</name>
<value>3</value>
<description>Default is 128</description>
</property>
<property>
<name>hbase.regionserver.executor.closeregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openpriorityregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.storescanner.parallel.seek.threads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hfile.compaction.discharger.thread.count</name>
<value>1</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.regionserver.executor.refresh.peer.threads</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.hregion.open.and.init.threads.max</name>
<value>3</value>
<description>Default is 16 or # of Regions</description>
</property>
<property>
<name>hbase.master.handler.count</name>
<value>7</value>
<description>Default is 25</description>
</property>
<property>
<name>hbase.replication.source.maxthreads</name>
<value></value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hconnection.meta.lookup.threads.max</name>
<value>5</value>
<description>Default is 128</description>
</property>
</configuration>

56
hbase-shell/src/test/resources/hdfs-site.xml Normal file
View File

@ -0,0 +1,56 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<!-- hadoop-2.0.5+'s HDFS-4305 by default enforces a min blocks size
of 1024*1024. Many unit tests that use the hlog use smaller
blocks. Setting this config to 0 to have tests pass -->
<property>
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
<property>
<name>dfs.datanode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.service.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<!--
Constraining this config makes tests fail.
<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>16</value>
<description>Default is 4096. If constrain this
too much, tests do not complete.</description>
</property>
-->
</configuration>

131
hbase-thrift/src/test/resources/hbase-site.xml
View File

@ -21,6 +21,22 @@
*/
-->
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
<value>100</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
</description>
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
<value>1000</value>
<value>100</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as hbase:meta scanner and log roller.
</description>
</property>
<property>
<name>hbase.regionserver.msginterval</name>
<value>1000</value>
@ -154,4 +170,119 @@
Enable replay sanity checks on procedure tests.
</description>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>3</value>
<description>Default is 30</description>
</property>
<property>
<name>hbase.regionserver.metahandler.count</name>
<value>3</value>
<description>Default is 20</description>
</property>
<property>
<name>hbase.netty.worker.count</name>
<value>3</value>
<description>Default is 0</description>
</property>
<property>
<name>hbase.hconnection.threads.max</name>
<value>6</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.htable.threads.max</name>
<value>3</value>
<description>Default is MAX_INTEGER</description>
</property>
<property>
<name>hbase.region.replica.replication.threads.max</name>
<value>7</value>
<description>Default is 256</description>
</property>
<property>
<name>hbase.rest.threads.max</name>
<value>5</value>
<description>Default is 100</description>
</property>
<property>
<name>hbase.replication.bulkload.copy.maxthreads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.loadincremental.threads.max</name>
<value>1</value>
<description>Default is # of CPUs</description>
</property>
<property>
<name>hbase.hstore.flusher.count</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.oldwals.cleaner.thread.size</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.master.procedure.threads</name>
<value>5</value>
<description>Default is at least 16</description>
</property>
<property>
<name>hbase.procedure.remote.dispatcher.threadpool.size</name>
<value>3</value>
<description>Default is 128</description>
</property>
<property>
<name>hbase.regionserver.executor.closeregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.regionserver.executor.openpriorityregion.threads</name>
<value>1</value>
<description>Default is 3</description>
</property>
<property>
<name>hbase.storescanner.parallel.seek.threads</name>
<value>3</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hfile.compaction.discharger.thread.count</name>
<value>1</value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.regionserver.executor.refresh.peer.threads</name>
<value>1</value>
<description>Default is 2</description>
</property>
<property>
<name>hbase.hregion.open.and.init.threads.max</name>
<value>3</value>
<description>Default is 16 or # of Regions</description>
</property>
<property>
<name>hbase.master.handler.count</name>
<value>7</value>
<description>Default is 25</description>
</property>
<property>
<name>hbase.replication.source.maxthreads</name>
<value></value>
<description>Default is 10</description>
</property>
<property>
<name>hbase.hconnection.meta.lookup.threads.max</name>
<value>5</value>
<description>Default is 128</description>
</property>
</configuration>

56
hbase-thrift/src/test/resources/hdfs-site.xml Normal file
View File

@ -0,0 +1,56 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<!-- hadoop-2.0.5+'s HDFS-4305 by default enforces a min blocks size
of 1024*1024. Many unit tests that use the hlog use smaller
blocks. Setting this config to 0 to have tests pass -->
<property>
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
<property>
<name>dfs.datanode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<property>
<name>dfs.namenode.service.handler.count</name>
<value>5</value>
<description>Default is 10</description>
</property>
<!--
Constraining this config makes tests fail.
<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>16</value>
<description>Default is 4096. If constrain this
too much, tests do not complete.</description>
</property>
-->
</configuration>

14
pom.xml
View File

@ -746,8 +746,17 @@
<testFailureIgnore>${surefire.testFailureIgnore}</testFailureIgnore>
<forkedProcessTimeoutInSeconds>${surefire.timeout}</forkedProcessTimeoutInSeconds>
<redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>
<!-- These are incontext system properties set on mvn. For forked
jvms, see hbase-surefire.argLine... So, we might be dup'ing
configs here and over on what we pass to the forked jvm.
-->
<systemPropertyVariables>
<test.build.classes>${test.build.classes}</test.build.classes>
<!--And for eventloops that have no explicit configuration, netty sets
nioeventloopgroup thread count to CPU count * 2. Thats too much
for mini clusters/tests.
-->
<io.netty.eventLoopThreads>3</io.netty.eventLoopThreads>
</systemPropertyVariables>
<excludes>
<!-- users can add -D option to skip particular test classes
@ -1596,11 +1605,16 @@
<surefire.Xmx>2800m</surefire.Xmx>
<surefire.cygwinXmx>2800m</surefire.cygwinXmx>
<!--Mark our test runs with '-Dhbase.build.id' so we can identify a surefire test as ours in a process listing
And for netty eventloops that have no explicit configuration, netty sets
nioeventloopgroup thread count to CPU count * 2. Thats too much for mini
clusters/tests.
-->
<hbase-surefire.argLine>-enableassertions -Dhbase.build.id=${build.id} -Xmx${surefire.Xmx}
-Djava.security.egd=file:/dev/./urandom -Djava.net.preferIPv4Stack=true
-Djava.awt.headless=true -Djdk.net.URLClassPath.disableClassPathURLCheck=true
-Dorg.apache.hbase.thirdparty.io.netty.leakDetection.level=advanced
-Dio.netty.eventLoopThreads=3
</hbase-surefire.argLine>
<hbase-surefire.cygwin-argLine>-enableassertions -Xmx${surefire.cygwinXmx}
-Djava.security.egd=file:/dev/./urandom -Djava.net.preferIPv4Stack=true