diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 11966c71b6a..6581209d40c 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -183,6 +183,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4161. create sockets consistently (Daryn Sharp via bobby)
+ MAPREDUCE-4079. Allow MR AppMaster to limit ephemeral port range.
+ (bobby via tgraves)
+
OPTIMIZATIONS
BUG FIXES
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
index 19db2a51981..60b29e831df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
@@ -139,7 +139,8 @@ public class MRClientService extends AbstractService
rpc.getServer(MRClientProtocol.class, protocolHandler, address,
conf, secretManager,
conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT,
- MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT));
+ MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT),
+ MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE);
// Enable service authorization?
if (conf.getBoolean(
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index f5db15763e7..03efd7e9bf3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -373,6 +373,13 @@ public interface MRJobConfig {
MR_AM_PREFIX + "job.client.thread-count";
public static final int DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT = 1;
+ /**
+ * Range of ports that the MapReduce AM can use when binding. Leave blank
+ * if you want all possible ports.
+ */
+ public static final String MR_AM_JOB_CLIENT_PORT_RANGE =
+ MR_AM_PREFIX + "job.client.port-range";
+
/** Enable blacklisting of nodes in the job.*/
public static final String MR_AM_JOB_NODE_BLACKLISTING_ENABLE =
MR_AM_PREFIX + "job.node-blacklisting.enable";
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 231dcb1cd23..34e5a7a9bb1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1236,6 +1236,14 @@
MR AppMaster from remote tasks
+
+ yarn.app.mapreduce.am.job.client.port-range
+
+ Range of ports that the MapReduce AM can use when binding.
+ Leave blank if you want all possible ports.
+ For example 50000-50050,50100-50200
+
+
yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms
1000
diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java
index 28576d7628a..94d7476e9df 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java
+++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java
@@ -31,6 +31,6 @@ public interface RpcServerFactory {
public Server getServer(Class> protocol, Object instance,
InetSocketAddress addr, Configuration conf,
SecretManager extends TokenIdentifier> secretManager,
- int numHandlers)
+ int numHandlers, String portRangeConfig)
throws YarnException;
}
\ No newline at end of file
diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java
index 0ea9d1c65e9..c26e51a1f6d 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java
+++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java
@@ -62,11 +62,20 @@ public class RpcServerFactoryPBImpl implements RpcServerFactory {
private RpcServerFactoryPBImpl() {
}
- @Override
public Server getServer(Class> protocol, Object instance,
InetSocketAddress addr, Configuration conf,
SecretManager extends TokenIdentifier> secretManager, int numHandlers)
throws YarnException {
+ return getServer(protocol, instance, addr, conf, secretManager, numHandlers,
+ null);
+ }
+
+ @Override
+ public Server getServer(Class> protocol, Object instance,
+ InetSocketAddress addr, Configuration conf,
+ SecretManager extends TokenIdentifier> secretManager, int numHandlers,
+ String portRangeConfig)
+ throws YarnException {
Constructor> constructor = serviceCache.get(protocol);
if (constructor == null) {
@@ -122,7 +131,7 @@ public class RpcServerFactoryPBImpl implements RpcServerFactory {
try {
return createServer(pbProtocol, addr, conf, secretManager, numHandlers,
- (BlockingService)method.invoke(null, service));
+ (BlockingService)method.invoke(null, service), portRangeConfig);
} catch (InvocationTargetException e) {
throw new YarnException(e);
} catch (IllegalAccessException e) {
@@ -156,11 +165,11 @@ public class RpcServerFactoryPBImpl implements RpcServerFactory {
private Server createServer(Class> pbProtocol, InetSocketAddress addr, Configuration conf,
SecretManager extends TokenIdentifier> secretManager, int numHandlers,
- BlockingService blockingService) throws IOException {
+ BlockingService blockingService, String portRangeConfig) throws IOException {
RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class);
RPC.Server server = RPC.getServer(pbProtocol, blockingService,
addr.getHostName(), addr.getPort(), numHandlers, false, conf,
- secretManager);
+ secretManager, portRangeConfig);
LOG.info("Adding protocol "+pbProtocol.getCanonicalName()+" to the server");
server.addProtocol(RpcKind.RPC_PROTOCOL_BUFFER, pbProtocol, blockingService);
return server;
diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java
index 1e3ca272967..a367ea37ce9 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java
+++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java
@@ -56,12 +56,12 @@ public class HadoopYarnProtoRPC extends YarnRPC {
public Server getServer(Class protocol, Object instance,
InetSocketAddress addr, Configuration conf,
SecretManager extends TokenIdentifier> secretManager,
- int numHandlers) {
+ int numHandlers, String portRangeConfig) {
LOG.debug("Creating a HadoopYarnProtoRpc server for protocol " + protocol +
" with " + numHandlers + " handlers");
return RpcFactoryProvider.getServerFactory(conf).getServer(protocol,
- instance, addr, conf, secretManager, numHandlers);
+ instance, addr, conf, secretManager, numHandlers, portRangeConfig);
}
diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java
index 1434326e5f5..512b8d49ce5 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java
+++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java
@@ -43,8 +43,16 @@ public abstract class YarnRPC {
public abstract Server getServer(Class protocol, Object instance,
InetSocketAddress addr, Configuration conf,
SecretManager extends TokenIdentifier> secretManager,
- int numHandlers);
+ int numHandlers, String portRangeConfig);
+ public Server getServer(Class protocol, Object instance,
+ InetSocketAddress addr, Configuration conf,
+ SecretManager extends TokenIdentifier> secretManager,
+ int numHandlers) {
+ return getServer(protocol, instance, addr, conf, secretManager, numHandlers,
+ null);
+ }
+
public static YarnRPC create(Configuration conf) {
LOG.debug("Creating YarnRPC for " +
conf.get(YarnConfiguration.IPC_RPC_IMPL));