Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-16703. Backport HADOOP-16152 to branch-3.1. Contributed by Siyao Meng. 

Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
This commit is contained in:
Siyao Meng 2019-11-19 12:14:00 -08:00 committed by Wei-Chiu Chuang
parent 1669a5c1cb
commit df13f6d485
8 changed files with 49 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-client-modules/hadoop-client-minicluster/pom.xml
View File

@ -762,7 +762,6 @@
<exclude>ehcache-core.xsd</exclude>
</excludes>
</filter>
<!-- remove utility classes which are not required from
dnsjava -->
<filter>
@ -774,7 +773,15 @@
<exclude>update*</exclude>
</excludes>
</filter>
<!-- Jetty 9.4.x: jetty-client and jetty-xml are depended by org.eclipse.jetty.websocket:websocket-client.
But we are only excluding jetty-client not jetty-xml because HttpServer2 implicitly uses the shaded package name.
-->
<filter>
<artifact>org.eclipse.jetty:jetty-client</artifact>
<excludes>
<exclude>*/**</exclude>
</excludes>
</filter>
</filters>
<!-- relocate classes from mssql-jdbc -->
@ -903,6 +910,13 @@
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
<relocation>
<pattern>javax/websocket/</pattern>
<shadedPattern>${shaded.dependency.prefix}.javax.websocket.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
<relocation>
<pattern>jersey/</pattern>
<shadedPattern>${shaded.dependency.prefix}.jersey.</shadedPattern>

2
hadoop-client-modules/hadoop-client-runtime/pom.xml
View File

@ -157,6 +157,8 @@
<!-- Leave javax APIs that are stable -->
<!-- the jdk ships part of the javax.annotation namespace, so if we want to relocate this we'll have to care it out by class :( -->
<exclude>com.google.code.findbugs:jsr305</exclude>
<exclude>org.eclipse.jetty:jetty-client</exclude>
<exclude>org.eclipse.jetty:jetty-http</exclude>
</excludes>
</artifactSet>
<filters>

12
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java
View File

@ -24,7 +24,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogConfigurationException;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Appender;
import org.eclipse.jetty.server.NCSARequestLog;
import org.eclipse.jetty.server.AsyncRequestLogWriter;
import org.eclipse.jetty.server.CustomRequestLog;
import org.eclipse.jetty.server.RequestLog;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -85,10 +86,11 @@ public class HttpRequestLog {
if (appender instanceof HttpRequestLogAppender) {
HttpRequestLogAppender requestLogAppender
= (HttpRequestLogAppender)appender;
NCSARequestLog requestLog = new NCSARequestLog();
requestLog.setFilename(requestLogAppender.getFilename());
requestLog.setRetainDays(requestLogAppender.getRetainDays());
return requestLog;
AsyncRequestLogWriter logWriter = new AsyncRequestLogWriter();
logWriter.setFilename(requestLogAppender.getFilename());
logWriter.setRetainDays(requestLogAppender.getRetainDays());
return new CustomRequestLog(logWriter,
CustomRequestLog.EXTENDED_NCSA_FORMAT);
} else {
LOG.warn("Jetty request log for {} was of the wrong class", loggerName);
return null;

44
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
View File

@ -80,12 +80,10 @@ import org.eclipse.jetty.server.RequestLog;
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SessionManager;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.handler.ContextHandlerCollection;
import org.eclipse.jetty.server.handler.HandlerCollection;
import org.eclipse.jetty.server.handler.RequestLogHandler;
import org.eclipse.jetty.server.session.AbstractSessionManager;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.servlet.DefaultServlet;
import org.eclipse.jetty.servlet.FilterHolder;
@ -498,7 +496,8 @@ public final class HttpServer2 implements FilterContainer {
httpConfig.addCustomizer(new SecureRequestCustomizer());
ServerConnector conn = createHttpChannelConnector(server, httpConfig);
SslContextFactory sslContextFactory = new SslContextFactory();
SslContextFactory.Server sslContextFactory =
new SslContextFactory.Server();
sslContextFactory.setNeedClientAuth(needsClientAuth);
sslContextFactory.setKeyManagerPassword(keyPassword);
if (keyStore != null) {
@ -566,12 +565,9 @@ public final class HttpServer2 implements FilterContainer {
threadPool.setMaxThreads(maxThreads);
}
SessionManager sm = webAppContext.getSessionHandler().getSessionManager();
if (sm instanceof AbstractSessionManager) {
AbstractSessionManager asm = (AbstractSessionManager)sm;
asm.setHttpOnly(true);
asm.getSessionCookieConfig().setSecure(true);
}
SessionHandler handler = webAppContext.getSessionHandler();
handler.setHttpOnly(true);
handler.getSessionCookieConfig().setSecure(true);
ContextHandlerCollection contexts = new ContextHandlerCollection();
RequestLog requestLog = HttpRequestLog.getRequestLog(name);
@ -713,12 +709,8 @@ public final class HttpServer2 implements FilterContainer {
}
logContext.setDisplayName("logs");
SessionHandler handler = new SessionHandler();
SessionManager sm = handler.getSessionManager();
if (sm instanceof AbstractSessionManager) {
AbstractSessionManager asm = (AbstractSessionManager) sm;
asm.setHttpOnly(true);
asm.getSessionCookieConfig().setSecure(true);
}
handler.setHttpOnly(true);
handler.getSessionCookieConfig().setSecure(true);
logContext.setSessionHandler(handler);
setContextAttributes(logContext, conf);
addNoCacheFilter(logContext);
@ -735,12 +727,8 @@ public final class HttpServer2 implements FilterContainer {
params.put("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
params.put("org.eclipse.jetty.servlet.Default.gzip", "true");
SessionHandler handler = new SessionHandler();
SessionManager sm = handler.getSessionManager();
if (sm instanceof AbstractSessionManager) {
AbstractSessionManager asm = (AbstractSessionManager) sm;
asm.setHttpOnly(true);
asm.getSessionCookieConfig().setSecure(true);
}
handler.setHttpOnly(true);
handler.getSessionCookieConfig().setSecure(true);
staticContext.setSessionHandler(handler);
setContextAttributes(staticContext, conf);
defaultContexts.put(staticContext, true);
@ -1208,7 +1196,7 @@ public final class HttpServer2 implements FilterContainer {
* @return
*/
private static BindException constructBindException(ServerConnector listener,
BindException ex) {
IOException ex) {
BindException be = new BindException("Port in use: "
+ listener.getHost() + ":" + listener.getPort());
if (ex != null) {
@ -1230,7 +1218,7 @@ public final class HttpServer2 implements FilterContainer {
try {
bindListener(listener);
break;
} catch (BindException ex) {
} catch (IOException ex) {
if (port == 0 || !findPort) {
throw constructBindException(listener, ex);
}
@ -1250,13 +1238,13 @@ public final class HttpServer2 implements FilterContainer {
*/
private void bindForPortRange(ServerConnector listener, int startPort)
throws Exception {
BindException bindException = null;
IOException ioException = null;
try {
bindListener(listener);
return;
} catch (BindException ex) {
} catch (IOException ex) {
// Ignore exception.
bindException = ex;
ioException = ex;
}
for(Integer port : portRanges) {
if (port == startPort) {
@ -1269,10 +1257,10 @@ public final class HttpServer2 implements FilterContainer {
return;
} catch (BindException ex) {
// Ignore exception. Move to next port.
bindException = ex;
ioException = ex;
}
}
throw constructBindException(listener, bindException);
throw constructBindException(listener, ioException);
}
/**

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java
View File

@ -18,7 +18,7 @@
package org.apache.hadoop.http;
import org.apache.log4j.Logger;
import org.eclipse.jetty.server.NCSARequestLog;
import org.eclipse.jetty.server.CustomRequestLog;
import org.eclipse.jetty.server.RequestLog;
import org.junit.Test;
@ -42,6 +42,7 @@ public class TestHttpRequestLog {
RequestLog requestLog = HttpRequestLog.getRequestLog("test");
Logger.getLogger("http.requests.test").removeAppender(requestLogAppender);
assertNotNull("RequestLog should not be null", requestLog);
assertEquals("Class mismatch", NCSARequestLog.class, requestLog.getClass());
assertEquals("Class mismatch",
CustomRequestLog.class, requestLog.getClass());
}
}

3
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java
View File

@ -105,7 +105,8 @@ public class TestJettyHelper implements MethodRule {
conn.setHost(host);
conn.setPort(port);
if (ssl) {
SslContextFactory sslContextFactory = new SslContextFactory();
SslContextFactory.Server sslContextFactory =
new SslContextFactory.Server();
sslContextFactory.setNeedClientAuth(false);
sslContextFactory.setKeyStorePath(keyStore);
sslContextFactory.setKeyStoreType(keyStoreType);

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
View File

@ -93,8 +93,9 @@ public class DatanodeHttpServer implements Closeable {
// set them to the minimum possible
private static final int HTTP_SELECTOR_THREADS = 1;
private static final int HTTP_ACCEPTOR_THREADS = 1;
// Jetty 9.4.x: Adding one more thread to HTTP_MAX_THREADS.
private static final int HTTP_MAX_THREADS =
HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 1;
HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 2;
public DatanodeHttpServer(final Configuration conf,
final DataNode datanode,

2
hadoop-project/pom.xml
View File

@ -35,7 +35,7 @@
<failIfNoTests>false</failIfNoTests>
<maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
<jetty.version>9.3.27.v20190418</jetty.version>
<jetty.version>9.4.20.v20190813</jetty.version>
<test.exclude>_</test.exclude>
<test.exclude.pattern>_</test.exclude.pattern>