Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-13272. DataNodeHttpServer to have configurable HttpServer2 threads. Contributed by Erik Krogen

This commit is contained in:
Chris Douglas 2018-05-02 21:23:57 -07:00
parent 67468651b1
commit b2acaa52d2
2 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
View File

@ -43,6 +43,7 @@ import io.netty.handler.stream.ChunkedWriteHandler;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSConfigKeys;
@ -91,6 +92,11 @@ public class DatanodeHttpServer implements Closeable {
private InetSocketAddress httpsAddress; private InetSocketAddress httpsAddress;
static final Log LOG = LogFactory.getLog(DatanodeHttpServer.class); static final Log LOG = LogFactory.getLog(DatanodeHttpServer.class);
@InterfaceAudience.Private
public static final String DATANODE_HTTP_MAX_THREADS_KEY =
"dfs.datanode.http.max-threads";
private static final int DATANODE_HTTP_MAX_THREADS_DEFAULT = 10;
public DatanodeHttpServer(final Configuration conf, public DatanodeHttpServer(final Configuration conf,
final DataNode datanode, final DataNode datanode,
final ServerSocketChannel externalHttpChannel) final ServerSocketChannel externalHttpChannel)
@ -99,7 +105,9 @@ public class DatanodeHttpServer implements Closeable {
this.conf = conf; this.conf = conf;
Configuration confForInfoServer = new Configuration(conf); Configuration confForInfoServer = new Configuration(conf);
confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10); confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS,
conf.getInt(DATANODE_HTTP_MAX_THREADS_KEY,
DATANODE_HTTP_MAX_THREADS_DEFAULT));
int proxyPort = int proxyPort =
confForInfoServer.getInt(DFS_DATANODE_HTTP_INTERNAL_PROXY_PORT, 0); confForInfoServer.getInt(DFS_DATANODE_HTTP_INTERNAL_PROXY_PORT, 0);
HttpServer2.Builder builder = new HttpServer2.Builder() HttpServer2.Builder builder = new HttpServer2.Builder()

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
View File

@ -104,6 +104,7 @@ import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
import org.apache.hadoop.hdfs.server.datanode.web.DatanodeHttpServer;
import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNode;
@ -855,6 +856,9 @@ public class MiniDFSCluster implements AutoCloseable {
conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
StaticMapping.class, DNSToSwitchMapping.class); StaticMapping.class, DNSToSwitchMapping.class);
} }
// Set to the minimum number of threads possible to avoid starting
// unnecessary threads in unit tests
conf.setInt(DatanodeHttpServer.DATANODE_HTTP_MAX_THREADS_KEY, 2);
// In an HA cluster, in order for the StandbyNode to perform checkpoints, // In an HA cluster, in order for the StandbyNode to perform checkpoints,
// it needs to know the HTTP port of the Active. So, if ephemeral ports // it needs to know the HTTP port of the Active. So, if ephemeral ports