From 3d41f330186f6481850b46e0c345d3ecf7b1b818 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Thu, 17 Oct 2019 13:09:53 -0700 Subject: [PATCH] HADOOP-16152. Upgrade Eclipse Jetty version to 9.4.x. Contributed by Yuming Wang, Siyao Meng. Co-authored-By: Siyao Meng Signed-off-by: Wei-Chiu Chuang --- .../hadoop-client-minicluster/pom.xml | 16 +++++++ .../hadoop-client-runtime/pom.xml | 3 ++ .../apache/hadoop/http/HttpRequestLog.java | 12 ++--- .../org/apache/hadoop/http/HttpServer2.java | 44 +++++++------------ .../hadoop/http/TestHttpRequestLog.java | 5 ++- .../apache/hadoop/test/TestJettyHelper.java | 3 +- .../datanode/web/DatanodeHttpServer.java | 3 +- hadoop-project/pom.xml | 2 +- .../org/apache/hadoop/yarn/sls/SLSRunner.java | 3 +- .../pom.xml | 16 +++++++ 10 files changed, 67 insertions(+), 40 deletions(-) diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 918b3741377..5641b62e4b4 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -811,6 +811,15 @@ */** + + + org.eclipse.jetty:jetty-client + + */** + + @@ -939,6 +948,13 @@ **/pom.xml + + javax/websocket/ + ${shaded.dependency.prefix}.javax.websocket. + + **/pom.xml + + jersey/ ${shaded.dependency.prefix}.jersey. diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml index 565e9682d51..552cd9c1d88 100644 --- a/hadoop-client-modules/hadoop-client-runtime/pom.xml +++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml @@ -161,6 +161,9 @@ org.eclipse.jetty.websocket:* org.eclipse.jetty:jetty-servlet org.eclipse.jetty:jetty-security + org.eclipse.jetty:jetty-client + org.eclipse.jetty:jetty-http + org.eclipse.jetty:jetty-xml org.ow2.asm:* org.bouncycastle:* diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java index 05573a8de95..b2f18538b6c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java @@ -24,7 +24,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogConfigurationException; import org.apache.commons.logging.LogFactory; import org.apache.log4j.Appender; -import org.eclipse.jetty.server.NCSARequestLog; +import org.eclipse.jetty.server.AsyncRequestLogWriter; +import org.eclipse.jetty.server.CustomRequestLog; import org.eclipse.jetty.server.RequestLog; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,10 +86,11 @@ public class HttpRequestLog { if (appender instanceof HttpRequestLogAppender) { HttpRequestLogAppender requestLogAppender = (HttpRequestLogAppender)appender; - NCSARequestLog requestLog = new NCSARequestLog(); - requestLog.setFilename(requestLogAppender.getFilename()); - requestLog.setRetainDays(requestLogAppender.getRetainDays()); - return requestLog; + AsyncRequestLogWriter logWriter = new AsyncRequestLogWriter(); + logWriter.setFilename(requestLogAppender.getFilename()); + logWriter.setRetainDays(requestLogAppender.getRetainDays()); + return new CustomRequestLog(logWriter, + CustomRequestLog.EXTENDED_NCSA_FORMAT); } else { LOG.warn("Jetty request log for {} was of the wrong class", loggerName); return null; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 74c81331d7f..7e7d64423f1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -88,13 +88,11 @@ import org.eclipse.jetty.server.RequestLog; import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.SessionManager; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.AllowSymLinkAliasChecker; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.server.handler.HandlerCollection; import org.eclipse.jetty.server.handler.RequestLogHandler; -import org.eclipse.jetty.server.session.AbstractSessionManager; import org.eclipse.jetty.server.session.SessionHandler; import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.FilterMapping; @@ -519,7 +517,8 @@ public final class HttpServer2 implements FilterContainer { httpConfig.addCustomizer(new SecureRequestCustomizer()); ServerConnector conn = createHttpChannelConnector(server, httpConfig); - SslContextFactory sslContextFactory = new SslContextFactory(); + SslContextFactory.Server sslContextFactory = + new SslContextFactory.Server(); sslContextFactory.setNeedClientAuth(needsClientAuth); sslContextFactory.setKeyManagerPassword(keyPassword); if (keyStore != null) { @@ -621,12 +620,9 @@ public final class HttpServer2 implements FilterContainer { threadPool.setMaxThreads(maxThreads); } - SessionManager sm = webAppContext.getSessionHandler().getSessionManager(); - if (sm instanceof AbstractSessionManager) { - AbstractSessionManager asm = (AbstractSessionManager)sm; - asm.setHttpOnly(true); - asm.getSessionCookieConfig().setSecure(true); - } + SessionHandler handler = webAppContext.getSessionHandler(); + handler.setHttpOnly(true); + handler.getSessionCookieConfig().setSecure(true); ContextHandlerCollection contexts = new ContextHandlerCollection(); RequestLog requestLog = HttpRequestLog.getRequestLog(name); @@ -777,12 +773,8 @@ public final class HttpServer2 implements FilterContainer { } logContext.setDisplayName("logs"); SessionHandler handler = new SessionHandler(); - SessionManager sm = handler.getSessionManager(); - if (sm instanceof AbstractSessionManager) { - AbstractSessionManager asm = (AbstractSessionManager) sm; - asm.setHttpOnly(true); - asm.getSessionCookieConfig().setSecure(true); - } + handler.setHttpOnly(true); + handler.getSessionCookieConfig().setSecure(true); logContext.setSessionHandler(handler); logContext.addAliasCheck(new AllowSymLinkAliasChecker()); setContextAttributes(logContext, conf); @@ -800,12 +792,8 @@ public final class HttpServer2 implements FilterContainer { params.put("org.eclipse.jetty.servlet.Default.dirAllowed", "false"); params.put("org.eclipse.jetty.servlet.Default.gzip", "true"); SessionHandler handler = new SessionHandler(); - SessionManager sm = handler.getSessionManager(); - if (sm instanceof AbstractSessionManager) { - AbstractSessionManager asm = (AbstractSessionManager) sm; - asm.setHttpOnly(true); - asm.getSessionCookieConfig().setSecure(true); - } + handler.setHttpOnly(true); + handler.getSessionCookieConfig().setSecure(true); staticContext.setSessionHandler(handler); staticContext.addAliasCheck(new AllowSymLinkAliasChecker()); setContextAttributes(staticContext, conf); @@ -1268,7 +1256,7 @@ public final class HttpServer2 implements FilterContainer { * @return */ private static BindException constructBindException(ServerConnector listener, - BindException ex) { + IOException ex) { BindException be = new BindException("Port in use: " + listener.getHost() + ":" + listener.getPort()); if (ex != null) { @@ -1290,7 +1278,7 @@ public final class HttpServer2 implements FilterContainer { try { bindListener(listener); break; - } catch (BindException ex) { + } catch (IOException ex) { if (port == 0 || !findPort) { throw constructBindException(listener, ex); } @@ -1310,13 +1298,13 @@ public final class HttpServer2 implements FilterContainer { */ private void bindForPortRange(ServerConnector listener, int startPort) throws Exception { - BindException bindException = null; + IOException ioException = null; try { bindListener(listener); return; - } catch (BindException ex) { + } catch (IOException ex) { // Ignore exception. - bindException = ex; + ioException = ex; } for(Integer port : portRanges) { if (port == startPort) { @@ -1329,10 +1317,10 @@ public final class HttpServer2 implements FilterContainer { return; } catch (BindException ex) { // Ignore exception. Move to next port. - bindException = ex; + ioException = ex; } } - throw constructBindException(listener, bindException); + throw constructBindException(listener, ioException); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java index 212807f78ef..d0123e32039 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java @@ -18,7 +18,7 @@ package org.apache.hadoop.http; import org.apache.log4j.Logger; -import org.eclipse.jetty.server.NCSARequestLog; +import org.eclipse.jetty.server.CustomRequestLog; import org.eclipse.jetty.server.RequestLog; import org.junit.Test; @@ -42,6 +42,7 @@ public class TestHttpRequestLog { RequestLog requestLog = HttpRequestLog.getRequestLog("test"); Logger.getLogger("http.requests.test").removeAppender(requestLogAppender); assertNotNull("RequestLog should not be null", requestLog); - assertEquals("Class mismatch", NCSARequestLog.class, requestLog.getClass()); + assertEquals("Class mismatch", + CustomRequestLog.class, requestLog.getClass()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java index 1da3901b395..e4df1184356 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java @@ -105,7 +105,8 @@ public class TestJettyHelper implements MethodRule { conn.setHost(host); conn.setPort(port); if (ssl) { - SslContextFactory sslContextFactory = new SslContextFactory(); + SslContextFactory.Server sslContextFactory = + new SslContextFactory.Server(); sslContextFactory.setNeedClientAuth(false); sslContextFactory.setKeyStorePath(keyStore); sslContextFactory.setKeyStoreType(keyStoreType); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java index 86672b403c9..e4a0c209762 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java @@ -83,8 +83,9 @@ public class DatanodeHttpServer implements Closeable { // set them to the minimum possible private static final int HTTP_SELECTOR_THREADS = 1; private static final int HTTP_ACCEPTOR_THREADS = 1; + // Jetty 9.4.x: Adding one more thread to HTTP_MAX_THREADS. private static final int HTTP_MAX_THREADS = - HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 1; + HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 2; private final HttpServer2 infoServer; private final EventLoopGroup bossGroup; private final EventLoopGroup workerGroup; diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 9c83560ae69..803cf7cc65f 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -35,7 +35,7 @@ false true - 9.3.27.v20190418 + 9.4.20.v20190813 _ _ diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index f99038ebe5f..6f75bd17c6f 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -97,7 +97,6 @@ import org.apache.hadoop.yarn.sls.utils.SLSUtils; import org.apache.hadoop.yarn.util.UTCClock; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; -import org.eclipse.jetty.util.ConcurrentHashSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -371,7 +370,7 @@ public class SLSRunner extends Configured implements Tool { // create NM simulators Random random = new Random(); - Set rackSet = new ConcurrentHashSet<>(); + Set rackSet = ConcurrentHashMap.newKeySet(); int threadPoolSize = Math.max(poolSize, SLSConfiguration.RUNNER_POOL_SIZE_DEFAULT); ExecutorService executorService = Executors. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml index 35718da4d28..0924f6e59dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml @@ -126,6 +126,14 @@ org.eclipse.jetty jetty-io + + org.eclipse.jetty + jetty-xml + + + org.eclipse.jetty + jetty-http + test @@ -143,6 +151,14 @@ org.eclipse.jetty jetty-io + + org.eclipse.jetty + jetty-xml + + + org.eclipse.jetty + jetty-http + test