From e23d5672a75742481c123d5d7d1f4cf08bab5185 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Wed, 25 Sep 2013 00:37:14 +0000 Subject: [PATCH] merge YARN-1229 from trunk. Define constraints on Auxiliary Service names. Change ShuffleHandler service name from mapreduce.shuffle to mapreduce_shuffle. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1526066 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 12 +++++++ .../src/site/apt/ClusterSetup.apt.vm | 2 +- .../src/site/apt/SingleCluster.apt.vm | 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 +++++++ hadoop-mapreduce-project/CHANGES.txt | 12 +++++++ hadoop-mapreduce-project/INSTALL | 4 +-- .../PluggableShuffleAndPluggableSort.apt.vm | 8 ++--- .../apache/hadoop/mapred/ShuffleHandler.java | 2 +- hadoop-yarn-project/CHANGES.txt | 16 ++++++++++ .../src/main/resources/yarn-default.xml | 5 +-- .../containermanager/AuxServices.java | 18 +++++++++++ .../containermanager/TestAuxServices.java | 31 +++++++++++++++++++ 12 files changed, 113 insertions(+), 11 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7abb1179ec4..2f3bcde5d81 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -96,6 +96,18 @@ Release 2.2.0 - UNRELEASED BUG FIXES +Release 2.1.2 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 2.1.1-beta - 2013-09-23 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm index cae366abc77..1e2a2c0b994 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm @@ -311,7 +311,7 @@ Hadoop MapReduce Next Generation - Cluster Setup | | | Only applicable if log-aggregation is enabled. | *-------------------------+-------------------------+------------------------+ | <<>> | | | -| | mapreduce.shuffle | | +| | mapreduce_shuffle | | | | | Shuffle service that needs to be set for Map Reduce applications. | *-------------------------+-------------------------+------------------------+ diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm index d7058d94914..c86cd6defdf 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm @@ -140,7 +140,7 @@ Add the following configs to your <<>> yarn.nodemanager.aux-services - mapreduce.shuffle + mapreduce_shuffle shuffle service that needs to be set for Map Reduce to run +---+ diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6527f766a69..a1e887ff14c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -85,6 +85,18 @@ Release 2.2.0 - UNRELEASED OPTIMIZATIONS + BUG FIXES + +Release 2.1.2 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + BUG FIXES HDFS-5139. Remove redundant -R option from setrep. diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index c9152c326db..250f032b019 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -54,6 +54,18 @@ Release 2.2.0 - UNRELEASED OPTIMIZATIONS + BUG FIXES + +Release 2.1.2 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + BUG FIXES MAPREDUCE-5504. mapred queue -info inconsistent with types (Kousuke Saruta diff --git a/hadoop-mapreduce-project/INSTALL b/hadoop-mapreduce-project/INSTALL index fae237045b9..f52fe2056fe 100644 --- a/hadoop-mapreduce-project/INSTALL +++ b/hadoop-mapreduce-project/INSTALL @@ -39,11 +39,11 @@ export YARN_CONF_DIR=$HADOOP_CONF_DIR Step 7) Setup config: for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager. yarn.nodemanager.aux-services - mapreduce.shuffle + mapreduce_shuffle - yarn.nodemanager.aux-services.mapreduce.shuffle.class + yarn.nodemanager.aux-services.mapreduce_shuffle.class org.apache.hadoop.mapred.ShuffleHandler diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm index 8dd2f2eceff..c37fc071d87 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm @@ -83,14 +83,14 @@ Hadoop MapReduce Next Generation - Pluggable Shuffle and Pluggable Sort *--------------------------------------+---------------------+-----------------+ | <> | <> | <> | *--------------------------------------+---------------------+-----------------+ -| <<>> | <<<...,mapreduce.shuffle>>> | The auxiliary service name | +| <<>> | <<<...,mapreduce_shuffle>>> | The auxiliary service name | *--------------------------------------+---------------------+-----------------+ -| <<>> | <<>> | The auxiliary service class to use | +| <<>> | <<>> | The auxiliary service class to use | *--------------------------------------+---------------------+-----------------+ <> If setting an auxiliary service in addition the default - <<>> service, then a new service key should be added to the + <<>> service, then a new service key should be added to the <<>> property, for example <<>>. Then the property defining the corresponding class must be - <<>>. + <<>>. \ No newline at end of file diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index f9d1203ba4d..82fd59e5516 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -146,7 +146,7 @@ public class ShuffleHandler extends AuxiliaryService { private ReadaheadPool readaheadPool = ReadaheadPool.getInstance(); public static final String MAPREDUCE_SHUFFLE_SERVICEID = - "mapreduce.shuffle"; + "mapreduce_shuffle"; private static final Map userRsrc = new ConcurrentHashMap(); diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e751b2be002..cd9aa9c4b1f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -41,6 +41,22 @@ Release 2.2.0 - UNRELEASED OPTIMIZATIONS + BUG FIXES + +Release 2.1.2 - UNRELEASED + + INCOMPATIBLE CHANGES + + YARN-1229. Define constraints on Auxiliary Service names. Change + ShuffleHandler service name from mapreduce.shuffle to + mapreduce_shuffle (Xuan Gong via sseth) + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + BUG FIXES YARN-1128. FifoPolicy.computeShares throws NPE on empty list of Schedulables diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 8aa7d01735b..65718a9d69c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -705,9 +705,10 @@ + the valid service name should only contain a-zA-Z0-9_ and can not start with numbers yarn.nodemanager.aux-services - + @@ -763,7 +764,7 @@ - yarn.nodemanager.aux-services.mapreduce.shuffle.class + yarn.nodemanager.aux-services.mapreduce_shuffle.class org.apache.hadoop.mapred.ShuffleHandler diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index 13f43650a47..0e0e7668f48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -40,6 +41,8 @@ import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext; import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; import org.apache.hadoop.yarn.server.api.ContainerTerminationContext; +import com.google.common.base.Preconditions; + public class AuxServices extends AbstractService implements ServiceStateChangeListener, EventHandler { @@ -48,6 +51,8 @@ public class AuxServices extends AbstractService protected final Map serviceMap; protected final Map serviceMetaData; + private final Pattern p = Pattern.compile("^[A-Za-z_]+[A-Za-z0-9_]*$"); + public AuxServices() { super(AuxServices.class.getName()); serviceMap = @@ -90,6 +95,13 @@ public class AuxServices extends AbstractService YarnConfiguration.NM_AUX_SERVICES); for (final String sName : auxNames) { try { + Preconditions + .checkArgument( + validateAuxServiceName(sName), + "The ServiceName: " + sName + " set in " + + YarnConfiguration.NM_AUX_SERVICES +" is invalid." + + "The valid service name should only contain a-zA-Z0-9_ " + + "and can not start with numbers"); Class sClass = conf.getClass( String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, sName), null, AuxiliaryService.class); @@ -199,4 +211,10 @@ public class AuxServices extends AbstractService } } + private boolean validateAuxServiceName(String name) { + if (name == null || name.trim().isEmpty()) { + return false; + } + return p.matcher(name).matches(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java index 81f758ee063..a58f3bda66c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java @@ -31,6 +31,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Map; +import junit.framework.Assert; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -288,4 +290,33 @@ public class TestAuxServices { assertTrue(aux.getServices().isEmpty()); } + @Test + public void testValidAuxServiceName() { + final AuxServices aux = new AuxServices(); + Configuration conf = new Configuration(); + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] {"Asrv1", "Bsrv_2"}); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv1"), + ServiceA.class, Service.class); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv_2"), + ServiceB.class, Service.class); + try { + aux.init(conf); + } catch (Exception ex) { + Assert.fail("Should not receive the exception."); + } + + //Test bad auxService Name + final AuxServices aux1 = new AuxServices(); + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] {"1Asrv1"}); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "1Asrv1"), + ServiceA.class, Service.class); + try { + aux1.init(conf); + Assert.fail("Should receive the exception."); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("The ServiceName: 1Asrv1 set in " + + "yarn.nodemanager.aux-services is invalid.The valid service name " + + "should only contain a-zA-Z0-9_ and can not start with numbers")); + } + } }