diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 11966c71b6a..6581209d40c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -183,6 +183,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4161. create sockets consistently (Daryn Sharp via bobby) + MAPREDUCE-4079. Allow MR AppMaster to limit ephemeral port range. + (bobby via tgraves) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java index 19db2a51981..60b29e831df 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java @@ -139,7 +139,8 @@ public void start() { rpc.getServer(MRClientProtocol.class, protocolHandler, address, conf, secretManager, conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, - MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT)); + MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT), + MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE); // Enable service authorization? if (conf.getBoolean( diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index f5db15763e7..03efd7e9bf3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -373,6 +373,13 @@ public interface MRJobConfig { MR_AM_PREFIX + "job.client.thread-count"; public static final int DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT = 1; + /** + * Range of ports that the MapReduce AM can use when binding. Leave blank + * if you want all possible ports. + */ + public static final String MR_AM_JOB_CLIENT_PORT_RANGE = + MR_AM_PREFIX + "job.client.port-range"; + /** Enable blacklisting of nodes in the job.*/ public static final String MR_AM_JOB_NODE_BLACKLISTING_ENABLE = MR_AM_PREFIX + "job.node-blacklisting.enable"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 231dcb1cd23..34e5a7a9bb1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1236,6 +1236,14 @@ MR AppMaster from remote tasks + + yarn.app.mapreduce.am.job.client.port-range + + Range of ports that the MapReduce AM can use when binding. + Leave blank if you want all possible ports. + For example 50000-50050,50100-50200 + + yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms 1000 diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java index 28576d7628a..94d7476e9df 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java @@ -31,6 +31,6 @@ public interface RpcServerFactory { public Server getServer(Class protocol, Object instance, InetSocketAddress addr, Configuration conf, SecretManager secretManager, - int numHandlers) + int numHandlers, String portRangeConfig) throws YarnException; } \ No newline at end of file diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java index 0ea9d1c65e9..c26e51a1f6d 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java @@ -62,11 +62,20 @@ public static RpcServerFactoryPBImpl get() { private RpcServerFactoryPBImpl() { } - @Override public Server getServer(Class protocol, Object instance, InetSocketAddress addr, Configuration conf, SecretManager secretManager, int numHandlers) throws YarnException { + return getServer(protocol, instance, addr, conf, secretManager, numHandlers, + null); + } + + @Override + public Server getServer(Class protocol, Object instance, + InetSocketAddress addr, Configuration conf, + SecretManager secretManager, int numHandlers, + String portRangeConfig) + throws YarnException { Constructor constructor = serviceCache.get(protocol); if (constructor == null) { @@ -122,7 +131,7 @@ public Server getServer(Class protocol, Object instance, try { return createServer(pbProtocol, addr, conf, secretManager, numHandlers, - (BlockingService)method.invoke(null, service)); + (BlockingService)method.invoke(null, service), portRangeConfig); } catch (InvocationTargetException e) { throw new YarnException(e); } catch (IllegalAccessException e) { @@ -156,11 +165,11 @@ private String getPackageName(Class clazz) { private Server createServer(Class pbProtocol, InetSocketAddress addr, Configuration conf, SecretManager secretManager, int numHandlers, - BlockingService blockingService) throws IOException { + BlockingService blockingService, String portRangeConfig) throws IOException { RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class); RPC.Server server = RPC.getServer(pbProtocol, blockingService, addr.getHostName(), addr.getPort(), numHandlers, false, conf, - secretManager); + secretManager, portRangeConfig); LOG.info("Adding protocol "+pbProtocol.getCanonicalName()+" to the server"); server.addProtocol(RpcKind.RPC_PROTOCOL_BUFFER, pbProtocol, blockingService); return server; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java index 1e3ca272967..a367ea37ce9 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java @@ -56,12 +56,12 @@ public void stopProxy(Object proxy, Configuration conf) { public Server getServer(Class protocol, Object instance, InetSocketAddress addr, Configuration conf, SecretManager secretManager, - int numHandlers) { + int numHandlers, String portRangeConfig) { LOG.debug("Creating a HadoopYarnProtoRpc server for protocol " + protocol + " with " + numHandlers + " handlers"); return RpcFactoryProvider.getServerFactory(conf).getServer(protocol, - instance, addr, conf, secretManager, numHandlers); + instance, addr, conf, secretManager, numHandlers, portRangeConfig); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java index 1434326e5f5..512b8d49ce5 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java @@ -43,8 +43,16 @@ public abstract Object getProxy(Class protocol, InetSocketAddress addr, public abstract Server getServer(Class protocol, Object instance, InetSocketAddress addr, Configuration conf, SecretManager secretManager, - int numHandlers); + int numHandlers, String portRangeConfig); + public Server getServer(Class protocol, Object instance, + InetSocketAddress addr, Configuration conf, + SecretManager secretManager, + int numHandlers) { + return getServer(protocol, instance, addr, conf, secretManager, numHandlers, + null); + } + public static YarnRPC create(Configuration conf) { LOG.debug("Creating YarnRPC for " + conf.get(YarnConfiguration.IPC_RPC_IMPL));