From df13f6d485754467c0fce12d4fff5ead4e1fa3e8 Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Tue, 19 Nov 2019 12:14:00 -0800 Subject: [PATCH] HADOOP-16703. Backport HADOOP-16152 to branch-3.1. Contributed by Siyao Meng. Signed-off-by: Wei-Chiu Chuang --- .../hadoop-client-minicluster/pom.xml | 18 +++++++- .../hadoop-client-runtime/pom.xml | 2 + .../apache/hadoop/http/HttpRequestLog.java | 12 ++--- .../org/apache/hadoop/http/HttpServer2.java | 44 +++++++------------ .../hadoop/http/TestHttpRequestLog.java | 5 ++- .../apache/hadoop/test/TestJettyHelper.java | 3 +- .../datanode/web/DatanodeHttpServer.java | 3 +- hadoop-project/pom.xml | 2 +- 8 files changed, 49 insertions(+), 40 deletions(-) diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 63c780d8225..fcfccf209eb 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -762,7 +762,6 @@ ehcache-core.xsd - @@ -774,7 +773,15 @@ update* - + + + org.eclipse.jetty:jetty-client + + */** + + @@ -903,6 +910,13 @@ **/pom.xml + + javax/websocket/ + ${shaded.dependency.prefix}.javax.websocket. + + **/pom.xml + + jersey/ ${shaded.dependency.prefix}.jersey. diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml index b0987d04939..823da6bc295 100644 --- a/hadoop-client-modules/hadoop-client-runtime/pom.xml +++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml @@ -157,6 +157,8 @@ com.google.code.findbugs:jsr305 + org.eclipse.jetty:jetty-client + org.eclipse.jetty:jetty-http diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java index 05573a8de95..b2f18538b6c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java @@ -24,7 +24,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogConfigurationException; import org.apache.commons.logging.LogFactory; import org.apache.log4j.Appender; -import org.eclipse.jetty.server.NCSARequestLog; +import org.eclipse.jetty.server.AsyncRequestLogWriter; +import org.eclipse.jetty.server.CustomRequestLog; import org.eclipse.jetty.server.RequestLog; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,10 +86,11 @@ public class HttpRequestLog { if (appender instanceof HttpRequestLogAppender) { HttpRequestLogAppender requestLogAppender = (HttpRequestLogAppender)appender; - NCSARequestLog requestLog = new NCSARequestLog(); - requestLog.setFilename(requestLogAppender.getFilename()); - requestLog.setRetainDays(requestLogAppender.getRetainDays()); - return requestLog; + AsyncRequestLogWriter logWriter = new AsyncRequestLogWriter(); + logWriter.setFilename(requestLogAppender.getFilename()); + logWriter.setRetainDays(requestLogAppender.getRetainDays()); + return new CustomRequestLog(logWriter, + CustomRequestLog.EXTENDED_NCSA_FORMAT); } else { LOG.warn("Jetty request log for {} was of the wrong class", loggerName); return null; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 02bd383eb91..dbdef9dcf00 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -80,12 +80,10 @@ import org.eclipse.jetty.server.RequestLog; import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.SessionManager; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.server.handler.HandlerCollection; import org.eclipse.jetty.server.handler.RequestLogHandler; -import org.eclipse.jetty.server.session.AbstractSessionManager; import org.eclipse.jetty.server.session.SessionHandler; import org.eclipse.jetty.servlet.DefaultServlet; import org.eclipse.jetty.servlet.FilterHolder; @@ -498,7 +496,8 @@ public final class HttpServer2 implements FilterContainer { httpConfig.addCustomizer(new SecureRequestCustomizer()); ServerConnector conn = createHttpChannelConnector(server, httpConfig); - SslContextFactory sslContextFactory = new SslContextFactory(); + SslContextFactory.Server sslContextFactory = + new SslContextFactory.Server(); sslContextFactory.setNeedClientAuth(needsClientAuth); sslContextFactory.setKeyManagerPassword(keyPassword); if (keyStore != null) { @@ -566,12 +565,9 @@ public final class HttpServer2 implements FilterContainer { threadPool.setMaxThreads(maxThreads); } - SessionManager sm = webAppContext.getSessionHandler().getSessionManager(); - if (sm instanceof AbstractSessionManager) { - AbstractSessionManager asm = (AbstractSessionManager)sm; - asm.setHttpOnly(true); - asm.getSessionCookieConfig().setSecure(true); - } + SessionHandler handler = webAppContext.getSessionHandler(); + handler.setHttpOnly(true); + handler.getSessionCookieConfig().setSecure(true); ContextHandlerCollection contexts = new ContextHandlerCollection(); RequestLog requestLog = HttpRequestLog.getRequestLog(name); @@ -713,12 +709,8 @@ public final class HttpServer2 implements FilterContainer { } logContext.setDisplayName("logs"); SessionHandler handler = new SessionHandler(); - SessionManager sm = handler.getSessionManager(); - if (sm instanceof AbstractSessionManager) { - AbstractSessionManager asm = (AbstractSessionManager) sm; - asm.setHttpOnly(true); - asm.getSessionCookieConfig().setSecure(true); - } + handler.setHttpOnly(true); + handler.getSessionCookieConfig().setSecure(true); logContext.setSessionHandler(handler); setContextAttributes(logContext, conf); addNoCacheFilter(logContext); @@ -735,12 +727,8 @@ public final class HttpServer2 implements FilterContainer { params.put("org.eclipse.jetty.servlet.Default.dirAllowed", "false"); params.put("org.eclipse.jetty.servlet.Default.gzip", "true"); SessionHandler handler = new SessionHandler(); - SessionManager sm = handler.getSessionManager(); - if (sm instanceof AbstractSessionManager) { - AbstractSessionManager asm = (AbstractSessionManager) sm; - asm.setHttpOnly(true); - asm.getSessionCookieConfig().setSecure(true); - } + handler.setHttpOnly(true); + handler.getSessionCookieConfig().setSecure(true); staticContext.setSessionHandler(handler); setContextAttributes(staticContext, conf); defaultContexts.put(staticContext, true); @@ -1208,7 +1196,7 @@ public final class HttpServer2 implements FilterContainer { * @return */ private static BindException constructBindException(ServerConnector listener, - BindException ex) { + IOException ex) { BindException be = new BindException("Port in use: " + listener.getHost() + ":" + listener.getPort()); if (ex != null) { @@ -1230,7 +1218,7 @@ public final class HttpServer2 implements FilterContainer { try { bindListener(listener); break; - } catch (BindException ex) { + } catch (IOException ex) { if (port == 0 || !findPort) { throw constructBindException(listener, ex); } @@ -1250,13 +1238,13 @@ public final class HttpServer2 implements FilterContainer { */ private void bindForPortRange(ServerConnector listener, int startPort) throws Exception { - BindException bindException = null; + IOException ioException = null; try { bindListener(listener); return; - } catch (BindException ex) { + } catch (IOException ex) { // Ignore exception. - bindException = ex; + ioException = ex; } for(Integer port : portRanges) { if (port == startPort) { @@ -1269,10 +1257,10 @@ public final class HttpServer2 implements FilterContainer { return; } catch (BindException ex) { // Ignore exception. Move to next port. - bindException = ex; + ioException = ex; } } - throw constructBindException(listener, bindException); + throw constructBindException(listener, ioException); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java index 212807f78ef..d0123e32039 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java @@ -18,7 +18,7 @@ package org.apache.hadoop.http; import org.apache.log4j.Logger; -import org.eclipse.jetty.server.NCSARequestLog; +import org.eclipse.jetty.server.CustomRequestLog; import org.eclipse.jetty.server.RequestLog; import org.junit.Test; @@ -42,6 +42,7 @@ public class TestHttpRequestLog { RequestLog requestLog = HttpRequestLog.getRequestLog("test"); Logger.getLogger("http.requests.test").removeAppender(requestLogAppender); assertNotNull("RequestLog should not be null", requestLog); - assertEquals("Class mismatch", NCSARequestLog.class, requestLog.getClass()); + assertEquals("Class mismatch", + CustomRequestLog.class, requestLog.getClass()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java index 1da3901b395..e4df1184356 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java @@ -105,7 +105,8 @@ public class TestJettyHelper implements MethodRule { conn.setHost(host); conn.setPort(port); if (ssl) { - SslContextFactory sslContextFactory = new SslContextFactory(); + SslContextFactory.Server sslContextFactory = + new SslContextFactory.Server(); sslContextFactory.setNeedClientAuth(false); sslContextFactory.setKeyStorePath(keyStore); sslContextFactory.setKeyStoreType(keyStoreType); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java index 4349c26b053..51a3137f3b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java @@ -93,8 +93,9 @@ public class DatanodeHttpServer implements Closeable { // set them to the minimum possible private static final int HTTP_SELECTOR_THREADS = 1; private static final int HTTP_ACCEPTOR_THREADS = 1; + // Jetty 9.4.x: Adding one more thread to HTTP_MAX_THREADS. private static final int HTTP_MAX_THREADS = - HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 1; + HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 2; public DatanodeHttpServer(final Configuration conf, final DataNode datanode, diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b193c198e6f..913a56c0c07 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -35,7 +35,7 @@ false true - 9.3.27.v20190418 + 9.4.20.v20190813 _ _