Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-12097. Allow port range to be specified while starting webapp. Contributed by Varun Saxena.

This commit is contained in:
Junping Du 2017-02-05 19:42:11 -08:00
parent d401e63b6c
commit cce35c3815
5 changed files with 227 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -1887,6 +1887,18 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
return result.toString();
}
/**
* Get range start for the first integer range.
* @return range start.
*/
public int getRangeStart() {
if (ranges == null || ranges.isEmpty()) {
return -1;
}
Range r = ranges.get(0);
return r.start;
}
@Override
public Iterator<Integer> iterator() {
return new RangeNumberIterator(ranges);

119
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
View File

@ -60,6 +60,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.ConfServlet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.jmx.JMXJsonServlet;
import org.apache.hadoop.log.LogLevel;
@ -151,6 +152,7 @@ public final class HttpServer2 implements FilterContainer {
protected final WebAppContext webAppContext;
protected final boolean findPort;
protected final IntegerRanges portRanges;
private final Map<ServletContextHandler, Boolean> defaultContexts =
new HashMap<>();
protected final List<String> filterNames = new ArrayList<>();
@ -189,6 +191,7 @@ public final class HttpServer2 implements FilterContainer {
private String keyPassword;
private boolean findPort;
private IntegerRanges portRanges = null;
private String hostName;
private boolean disallowFallbackToRandomSignerSecretProvider;
@ -261,6 +264,11 @@ public final class HttpServer2 implements FilterContainer {
return this;
}
public Builder setPortRanges(IntegerRanges ranges) {
this.portRanges = ranges;
return this;
}
public Builder setConf(Configuration conf) {
this.conf = conf;
return this;
@ -496,6 +504,7 @@ public final class HttpServer2 implements FilterContainer {
}
this.findPort = b.findPort;
this.portRanges = b.portRanges;
initializeWebServer(b.name, b.hostName, b.conf, b.pathSpecs);
}
@ -1079,6 +1088,93 @@ public final class HttpServer2 implements FilterContainer {
}
}
/**
* Bind listener by closing and opening the listener.
* @param listener
* @throws Exception
*/
private static void bindListener(ServerConnector listener) throws Exception {
// jetty has a bug where you can't reopen a listener that previously
// failed to open w/o issuing a close first, even if the port is changed
listener.close();
listener.open();
LOG.info("Jetty bound to port " + listener.getLocalPort());
}
/**
* Create bind exception by wrapping the bind exception thrown.
* @param listener
* @param ex
* @return
*/
private static BindException constructBindException(ServerConnector listener,
BindException ex) {
BindException be = new BindException("Port in use: "
+ listener.getHost() + ":" + listener.getPort());
if (ex != null) {
be.initCause(ex);
}
return be;
}
/**
* Bind using single configured port. If findPort is true, we will try to bind
* after incrementing port till a free port is found.
* @param listener jetty listener.
* @param port port which is set in the listener.
* @throws Exception
*/
private void bindForSinglePort(ServerConnector listener, int port)
throws Exception {
while (true) {
try {
bindListener(listener);
break;
} catch (BindException ex) {
if (port == 0 || !findPort) {
throw constructBindException(listener, ex);
}
}
// try the next port number
listener.setPort(++port);
Thread.sleep(100);
}
}
/**
* Bind using port ranges. Keep on looking for a free port in the port range
* and throw a bind exception if no port in the configured range binds.
* @param listener jetty listener.
* @param startPort initial port which is set in the listener.
* @throws Exception
*/
private void bindForPortRange(ServerConnector listener, int startPort)
throws Exception {
BindException bindException = null;
try {
bindListener(listener);
return;
} catch (BindException ex) {
// Ignore exception.
bindException = ex;
}
for(Integer port : portRanges) {
if (port == startPort) {
continue;
}
Thread.sleep(100);
listener.setPort(port);
try {
bindListener(listener);
return;
} catch (BindException ex) {
// Ignore exception. Move to next port.
bindException = ex;
}
}
throw constructBindException(listener, bindException);
}
/**
* Open the main listener for the server
* @throws Exception
@ -1091,25 +1187,10 @@ public final class HttpServer2 implements FilterContainer {
continue;
}
int port = listener.getPort();
while (true) {
// jetty has a bug where you can't reopen a listener that previously
// failed to open w/o issuing a close first, even if the port is changed
try {
listener.close();
listener.open();
LOG.info("Jetty bound to port " + listener.getLocalPort());
break;
} catch (BindException ex) {
if (port == 0 || !findPort) {
BindException be = new BindException("Port in use: "
+ listener.getHost() + ":" + listener.getPort());
be.initCause(ex);
throw be;
}
}
// try the next port number
listener.setPort(++port);
Thread.sleep(100);
if (portRanges != null && port != 0) {
bindForPortRange(listener, port);
} else {
bindForSinglePort(listener, port);
}
}
}

38
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
View File

@ -20,10 +20,12 @@ package org.apache.hadoop.http;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.http.HttpServer2.QuotingInputFilter.RequestQuoter;
import org.apache.hadoop.http.resource.JerseyResource;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
import org.apache.hadoop.security.UserGroupInformation;
@ -644,4 +646,40 @@ public class TestHttpServer extends HttpServerFunctionalTest {
assertNotNull(conn.getHeaderField("Date"));
assertEquals(conn.getHeaderField("Expires"), conn.getHeaderField("Date"));
}
private static void stopHttpServer(HttpServer2 server) throws Exception {
if (server != null) {
server.stop();
}
}
@Test
public void testPortRanges() throws Exception {
Configuration conf = new Configuration();
int port = ServerSocketUtil.waitForPort(49000, 60);
int endPort = 49500;
conf.set("abc", "49000-49500");
HttpServer2.Builder builder = new HttpServer2.Builder()
.setName("test").setConf(new Configuration()).setFindPort(false);
IntegerRanges ranges = conf.getRange("abc", "");
int startPort = 0;
if (ranges != null && !ranges.isEmpty()) {
startPort = ranges.getRangeStart();
builder.setPortRanges(ranges);
}
builder.addEndpoint(URI.create("http://localhost:" + startPort));
HttpServer2 myServer = builder.build();
HttpServer2 myServer2 = null;
try {
myServer.start();
assertEquals(port, myServer.getConnectorAddress(0).getPort());
myServer2 = builder.build();
myServer2.start();
assertTrue(myServer2.getConnectorAddress(0).getPort() > port &&
myServer2.getConnectorAddress(0).getPort() <= endPort);
} finally {
stopHttpServer(myServer);
stopHttpServer(myServer2);
}
}
}

38
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
View File

@ -35,6 +35,7 @@ import javax.servlet.http.HttpServlet;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.UserGroupInformation;
@ -92,6 +93,7 @@ public class WebApps {
boolean findPort = false;
Configuration conf;
Policy httpPolicy = null;
String portRangeConfigKey = null;
boolean devMode = false;
private String spnegoPrincipalKey;
private String spnegoKeytabKey;
@ -157,6 +159,19 @@ public class WebApps {
return this;
}
/**
* Set port range config key and associated configuration object.
* @param config configuration.
* @param portRangeConfKey port range config key.
* @return builder object.
*/
public Builder<T> withPortRange(Configuration config,
String portRangeConfKey) {
this.conf = config;
this.portRangeConfigKey = portRangeConfKey;
return this;
}
public Builder<T> withHttpSpnegoPrincipalKey(String spnegoPrincipalKey) {
this.spnegoPrincipalKey = spnegoPrincipalKey;
return this;
@ -265,15 +280,24 @@ public class WebApps {
: WebAppUtils.HTTP_PREFIX;
}
HttpServer2.Builder builder = new HttpServer2.Builder()
.setName(name)
.addEndpoint(
URI.create(httpScheme + bindAddress
+ ":" + port)).setConf(conf).setFindPort(findPort)
.setName(name).setConf(conf).setFindPort(findPort)
.setACL(new AccessControlList(conf.get(
YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)))
YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)))
.setPathSpec(pathList.toArray(new String[0]));
// Get port ranges from config.
IntegerRanges ranges = null;
if (portRangeConfigKey != null) {
ranges = conf.getRange(portRangeConfigKey, "");
}
int startPort = port;
if (ranges != null && !ranges.isEmpty()) {
// Set port ranges if its configured.
startPort = ranges.getRangeStart();
builder.setPortRanges(ranges);
}
builder.addEndpoint(URI.create(httpScheme + bindAddress +
":" + startPort));
boolean hasSpnegoConf = spnegoPrincipalKey != null
&& conf.get(spnegoPrincipalKey) != null && spnegoKeytabKey != null
&& conf.get(spnegoKeytabKey) != null;

46
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
View File

@ -35,6 +35,8 @@ import java.net.HttpURLConnection;
import java.net.URL;
import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.webapp.view.HtmlPage;
import org.apache.hadoop.yarn.webapp.view.JQueryUI;
@ -307,6 +309,50 @@ public class TestWebApp {
}
}
private static void stopWebApp(WebApp app) {
if (app != null) {
app.stop();
}
}
@Test
public void testPortRanges() throws Exception {
WebApp app = WebApps.$for("test", this).start();
String baseUrl = baseUrl(app);
WebApp app1 = null;
WebApp app2 = null;
WebApp app3 = null;
WebApp app4 = null;
WebApp app5 = null;
try {
int port = ServerSocketUtil.waitForPort(48000, 60);
assertEquals("foo", getContent(baseUrl +"test/foo").trim());
app1 = WebApps.$for("test", this).at(port).start();
assertEquals(port, app1.getListenerAddress().getPort());
app2 = WebApps.$for("test", this).at("0.0.0.0",port, true).start();
assertTrue(app2.getListenerAddress().getPort() > port);
Configuration conf = new Configuration();
port = ServerSocketUtil.waitForPort(47000, 60);
app3 = WebApps.$for("test", this).at(port).withPortRange(conf, "abc").
start();
assertEquals(port, app3.getListenerAddress().getPort());
ServerSocketUtil.waitForPort(46000, 60);
conf.set("abc", "46000-46500");
app4 = WebApps.$for("test", this).at(port).withPortRange(conf, "abc").
start();
assertEquals(46000, app4.getListenerAddress().getPort());
app5 = WebApps.$for("test", this).withPortRange(conf, "abc").start();
assertTrue(app5.getListenerAddress().getPort() > 46000);
} finally {
stopWebApp(app);
stopWebApp(app1);
stopWebApp(app2);
stopWebApp(app3);
stopWebApp(app4);
stopWebApp(app5);
}
}
static String baseUrl(WebApp app) {
return "http://localhost:"+ app.port() +"/";
}