From e99ed054447def8c709b270c66ef57213d3f6e5e Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Wed, 5 Sep 2018 09:52:35 -0700 Subject: [PATCH] HADOOP-15696. KMS performance regression due to too many open file descriptors after Jetty migration. Contributed by Wei-Chiu Chuang. (cherry picked from commit e780556ae9229fe7a90817eb4e5449d7eed35dd8) Conflicts: hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java (cherry picked from commit d1ea7df43d6a5623b139acbbc9992831d5c7e484) --- .../org/apache/hadoop/http/HttpServer2.java | 26 ++++++++++--------- .../apache/hadoop/http/TestHttpServer.java | 14 ++++++++++ .../src/main/resources/kms-default.xml | 7 +++++ .../src/main/resources/httpfs-default.xml | 7 +++++ 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index e83f7f8bcb9..b4484e51a50 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -138,6 +138,10 @@ public final class HttpServer2 implements FilterContainer { "hadoop.http.selector.count"; // -1 to use default behavior of setting count based on CPU core count public static final int HTTP_SELECTOR_COUNT_DEFAULT = -1; + // idle timeout in milliseconds + public static final String HTTP_IDLE_TIMEOUT_MS_KEY = + "hadoop.http.idle_timeout.ms"; + public static final int HTTP_IDLE_TIMEOUT_MS_DEFAULT = 10000; public static final String HTTP_TEMP_DIR_KEY = "hadoop.http.temp.dir"; static final String FILTER_INITIALIZER_PROPERTY @@ -436,6 +440,8 @@ public final class HttpServer2 implements FilterContainer { int responseHeaderSize = conf.getInt( HTTP_MAX_RESPONSE_HEADER_SIZE_KEY, HTTP_MAX_RESPONSE_HEADER_SIZE_DEFAULT); + int idleTimeout = conf.getInt(HTTP_IDLE_TIMEOUT_MS_KEY, + HTTP_IDLE_TIMEOUT_MS_DEFAULT); HttpConfiguration httpConfig = new HttpConfiguration(); httpConfig.setRequestHeaderSize(requestHeaderSize); @@ -461,6 +467,7 @@ public final class HttpServer2 implements FilterContainer { connector.setHost(ep.getHost()); connector.setPort(ep.getPort() == -1 ? 0 : ep.getPort()); connector.setAcceptQueueSize(backlogSize); + connector.setIdleTimeout(idleTimeout); server.addListener(connector); } server.loadListeners(); @@ -474,7 +481,13 @@ public final class HttpServer2 implements FilterContainer { conf.getInt(HTTP_SELECTOR_COUNT_KEY, HTTP_SELECTOR_COUNT_DEFAULT)); ConnectionFactory connFactory = new HttpConnectionFactory(httpConfig); conn.addConnectionFactory(connFactory); - configureChannelConnector(conn); + if(Shell.WINDOWS) { + // result of setting the SO_REUSEADDR flag is different on Windows + // http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx + // without this 2 NN's can start on the same machine and listen on + // the same port with indeterminate routing of incoming requests to them + conn.setReuseAddress(false); + } return conn; } @@ -653,17 +666,6 @@ public final class HttpServer2 implements FilterContainer { Collections. emptyMap(), new String[] { "/*" }); } - private static void configureChannelConnector(ServerConnector c) { - c.setIdleTimeout(10000); - if(Shell.WINDOWS) { - // result of setting the SO_REUSEADDR flag is different on Windows - // http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx - // without this 2 NN's can start on the same machine and listen on - // the same port with indeterminate routing of incoming requests to them - c.setReuseAddress(false); - } - } - /** Get an array of FilterConfiguration specified in the conf */ private static FilterInitializer[] getFilterInitializers(Configuration conf) { if (conf == null) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java index ba4be2760fc..fc424e90b01 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java @@ -55,6 +55,7 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.MediaType; import java.io.IOException; import java.io.PrintWriter; +import java.lang.reflect.Field; import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; @@ -701,4 +702,17 @@ public class TestHttpServer extends HttpServerFunctionalTest { ServerConnector listener = (ServerConnector)listeners.get(0); assertEquals(backlogSize, listener.getAcceptQueueSize()); } + + @Test + public void testIdleTimeout() throws Exception { + final int idleTimeout = 1000; + Configuration conf = new Configuration(); + conf.setInt(HttpServer2.HTTP_IDLE_TIMEOUT_MS_KEY, idleTimeout); + HttpServer2 srv = createServer("test", conf); + Field f = HttpServer2.class.getDeclaredField("listeners"); + f.setAccessible(true); + List listeners = (List) f.get(srv); + ServerConnector listener = (ServerConnector)listeners.get(0); + assertEquals(idleTimeout, listener.getIdleTimeout()); + } } diff --git a/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml b/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml index 0c785a228ce..eda91ff4a12 100644 --- a/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml +++ b/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml @@ -101,6 +101,13 @@ + + hadoop.http.idle_timeout.ms + 1000 + + KMS Server connection timeout in milliseconds. + + diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml index e4204564c25..3e9064f4472 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml @@ -54,6 +54,13 @@ + + hadoop.http.idle_timeout.ms + 1000 + + Httpfs Server connection timeout in milliseconds. + + hadoop.http.max.threads