diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index a6bbca7c56f..422fd296c82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -424,9 +424,9 @@ public abstract class ApplicationSubmissionContext { /** * Set tags for the application. A maximum of - * {@link YarnConfiguration#APPLICATION_MAX_TAGS} are allowed + * {@link YarnConfiguration#RM_APPLICATION_MAX_TAGS} are allowed * per application. Each tag can be at most - * {@link YarnConfiguration#APPLICATION_MAX_TAG_LENGTH} + * {@link YarnConfiguration#RM_APPLICATION_MAX_TAG_LENGTH} * characters, and can contain only ASCII characters. * * @param tags tags to set diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 029cfef7113..25dd9359cc6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -92,10 +92,10 @@ public class YarnConfiguration extends Configuration { YARN_SITE_CONFIGURATION_FILE, CORE_SITE_CONFIGURATION_FILE)); - @Evolving + @Deprecated public static final int APPLICATION_MAX_TAGS = 10; - @Evolving + @Deprecated public static final int APPLICATION_MAX_TAG_LENGTH = 100; static { @@ -198,6 +198,18 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_RM_ADDRESS = "0.0.0.0:" + DEFAULT_RM_PORT; + /**Max number of application tags.*/ + public static final String RM_APPLICATION_MAX_TAGS = RM_PREFIX + + "application.max-tags"; + + public static final int DEFAULT_RM_APPLICATION_MAX_TAGS = 10; + + /**Max length of each application tag.*/ + public static final String RM_APPLICATION_MAX_TAG_LENGTH = RM_PREFIX + + "application.max-tag.length"; + + public static final int DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH = 100; + public static final String RM_APPLICATION_MASTER_SERVICE_PROCESSORS = RM_PREFIX + "application-master-service.processors"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index c2b9111c032..53e1ed49e76 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder; @@ -273,24 +272,6 @@ extends ApplicationSubmissionContext { builder.setApplicationType((applicationType)); } - private void checkTags(Set tags) { - if (tags.size() > YarnConfiguration.APPLICATION_MAX_TAGS) { - throw new IllegalArgumentException("Too many applicationTags, a maximum of only " - + YarnConfiguration.APPLICATION_MAX_TAGS + " are allowed!"); - } - for (String tag : tags) { - if (tag.length() > YarnConfiguration.APPLICATION_MAX_TAG_LENGTH) { - throw new IllegalArgumentException("Tag " + tag + " is too long, " + - "maximum allowed length of a tag is " + - YarnConfiguration.APPLICATION_MAX_TAG_LENGTH); - } - if (!CharMatcher.ASCII.matchesAllOf(tag)) { - throw new IllegalArgumentException("A tag can only have ASCII " + - "characters! Invalid tag - " + tag); - } - } - } - @Override public void setApplicationTags(Set tags) { maybeInitBuilder(); @@ -299,7 +280,6 @@ extends ApplicationSubmissionContext { this.applicationTags = null; return; } - checkTags(tags); // Convert applicationTags to lower case and add this.applicationTags = new HashSet(); for (String tag : tags) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 3f6bee32bf3..1aaeab92af1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3609,4 +3609,17 @@ + + Max number of application tags set by user in ApplicationSubmissionContext + while submitting application + yarn.resourcemanager.application.max-tags + 10 + + + + Max length of each application tag set by user in ApplicationSubmissionContext + while submitting application. + yarn.resourcemanager.application.max-tag.length + 100 + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 8c035267913..da77b3001b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -181,6 +181,7 @@ import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.CharMatcher; /** * The client interface to the Resource Manager. This module handles all the rpc @@ -584,6 +585,8 @@ public class ClientRMService extends AbstractService implements throw RPCUtil.getRemoteException(ie); } + checkTags(submissionContext.getApplicationTags()); + if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { // Sanity check for flow run String value = null; @@ -725,6 +728,31 @@ public class ClientRMService extends AbstractService implements return response; } + private void checkTags(Set tags) throws YarnException { + int appMaxTags = getConfig().getInt( + YarnConfiguration.RM_APPLICATION_MAX_TAGS, + YarnConfiguration.DEFAULT_RM_APPLICATION_MAX_TAGS); + int appMaxTagLength = getConfig().getInt( + YarnConfiguration.RM_APPLICATION_MAX_TAG_LENGTH, + YarnConfiguration.DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH); + if (tags.size() > appMaxTags) { + throw RPCUtil.getRemoteException(new IllegalArgumentException( + "Too many applicationTags, a maximum of only " + appMaxTags + + " are allowed!")); + } + for (String tag : tags) { + if (tag.length() > appMaxTagLength) { + throw RPCUtil.getRemoteException( + new IllegalArgumentException("Tag " + tag + " is too long, " + + "maximum allowed length of a tag is " + appMaxTagLength)); + } + if (!CharMatcher.ASCII.matchesAllOf(tag)) { + throw RPCUtil.getRemoteException(new IllegalArgumentException( + "A tag can only have ASCII " + "characters! Invalid tag - " + tag)); + } + } + } + @SuppressWarnings("unchecked") @Override public KillApplicationResponse forceKillApplication( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index b0a410b4ada..1b13acbc9bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -521,6 +521,51 @@ public class TestClientRMService { } } + @Test + public void testApplicationTagsValidation() throws IOException { + YarnConfiguration conf = new YarnConfiguration(); + int maxtags = 3, appMaxTagLength = 5; + conf.setInt(YarnConfiguration.RM_APPLICATION_MAX_TAGS, maxtags); + conf.setInt(YarnConfiguration.RM_APPLICATION_MAX_TAG_LENGTH, + appMaxTagLength); + MockRM rm = new MockRM(conf); + rm.init(conf); + rm.start(); + + ClientRMService rmService = rm.getClientRMService(); + + List tags = Arrays.asList("Tag1", "Tag2", "Tag3", "Tag4"); + validateApplicationTag(rmService, tags, + "Too many applicationTags, a maximum of only " + maxtags + + " are allowed!"); + + tags = Arrays.asList("ApplicationTag1", "ApplicationTag2", + "ApplicationTag3"); + // tags are converted to lowercase in + // ApplicationSubmissionContext#setApplicationTags + validateApplicationTag(rmService, tags, + "Tag applicationtag1 is too long, maximum allowed length of a tag is " + + appMaxTagLength); + + tags = Arrays.asList("tãg1", "tag2#"); + validateApplicationTag(rmService, tags, + "A tag can only have ASCII characters! Invalid tag - tãg1"); + rm.close(); + } + + private void validateApplicationTag(ClientRMService rmService, + List tags, String errorMsg) { + SubmitApplicationRequest submitRequest = mockSubmitAppRequest( + getApplicationId(101), MockApps.newAppName(), QUEUE_1, + new HashSet(tags)); + try { + rmService.submitApplication(submitRequest); + Assert.fail(); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains(errorMsg)); + } + } + @Test public void testForceKillApplication() throws Exception { YarnConfiguration conf = new YarnConfiguration();