From 6dc1f7e15441c91487a7fe655a22dcf01df80316 Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Fri, 17 Apr 2020 09:43:36 -0700 Subject: [PATCH] YARN-9954. Configurable max application tags and max tag length. Contributed by Bilwa S T (cherry picked from commit 49ae9b2137dddaff1b579586e593a24da21a9e7d) (cherry picked from commit d1af4e0fae842d46c29562444ebb16905f2caf0d) --- .../records/ApplicationSubmissionContext.java | 4 +- .../hadoop/yarn/conf/YarnConfiguration.java | 16 ++++++- .../ApplicationSubmissionContextPBImpl.java | 20 --------- .../src/main/resources/yarn-default.xml | 13 ++++++ .../resourcemanager/ClientRMService.java | 27 +++++++++++ .../resourcemanager/TestClientRMService.java | 45 +++++++++++++++++++ 6 files changed, 101 insertions(+), 24 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index d2adfdc478a..1d4ffc9b5b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -424,9 +424,9 @@ public abstract class ApplicationSubmissionContext { /** * Set tags for the application. A maximum of - * {@link YarnConfiguration#APPLICATION_MAX_TAGS} are allowed + * {@link YarnConfiguration#RM_APPLICATION_MAX_TAGS} are allowed * per application. Each tag can be at most - * {@link YarnConfiguration#APPLICATION_MAX_TAG_LENGTH} + * {@link YarnConfiguration#RM_APPLICATION_MAX_TAG_LENGTH} * characters, and can contain only ASCII characters. * * @param tags tags to set diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 6e9297a5830..13a3506bd8a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -92,10 +92,10 @@ public class YarnConfiguration extends Configuration { YARN_SITE_CONFIGURATION_FILE, CORE_SITE_CONFIGURATION_FILE)); - @Evolving + @Deprecated public static final int APPLICATION_MAX_TAGS = 10; - @Evolving + @Deprecated public static final int APPLICATION_MAX_TAG_LENGTH = 100; static { @@ -202,6 +202,18 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_RM_ADDRESS = "0.0.0.0:" + DEFAULT_RM_PORT; + /**Max number of application tags.*/ + public static final String RM_APPLICATION_MAX_TAGS = RM_PREFIX + + "application.max-tags"; + + public static final int DEFAULT_RM_APPLICATION_MAX_TAGS = 10; + + /**Max length of each application tag.*/ + public static final String RM_APPLICATION_MAX_TAG_LENGTH = RM_PREFIX + + "application.max-tag.length"; + + public static final int DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH = 100; + public static final String RM_APPLICATION_MASTER_SERVICE_PROCESSORS = RM_PREFIX + "application-master-service.processors"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index b30224e78fb..01f09838ca2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder; @@ -277,24 +276,6 @@ extends ApplicationSubmissionContext { builder.setApplicationType((applicationType)); } - private void checkTags(Set tags) { - if (tags.size() > YarnConfiguration.APPLICATION_MAX_TAGS) { - throw new IllegalArgumentException("Too many applicationTags, a maximum of only " - + YarnConfiguration.APPLICATION_MAX_TAGS + " are allowed!"); - } - for (String tag : tags) { - if (tag.length() > YarnConfiguration.APPLICATION_MAX_TAG_LENGTH) { - throw new IllegalArgumentException("Tag " + tag + " is too long, " + - "maximum allowed length of a tag is " + - YarnConfiguration.APPLICATION_MAX_TAG_LENGTH); - } - if (!org.apache.commons.lang3.StringUtils.isAsciiPrintable(tag)) { - throw new IllegalArgumentException("A tag can only have ASCII " + - "characters! Invalid tag - " + tag); - } - } - } - @Override public synchronized void setApplicationTags(Set tags) { maybeInitBuilder(); @@ -303,7 +284,6 @@ extends ApplicationSubmissionContext { this.applicationTags = null; return; } - checkTags(tags); // Convert applicationTags to lower case and add this.applicationTags = new HashSet(); for (String tag : tags) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a6d0d08c15c..2a0d8b59c8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3909,4 +3909,17 @@ + + Max number of application tags set by user in ApplicationSubmissionContext + while submitting application + yarn.resourcemanager.application.max-tags + 10 + + + + Max length of each application tag set by user in ApplicationSubmissionContext + while submitting application. + yarn.resourcemanager.application.max-tag.length + 100 + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 1c10e89839f..db4990ea523 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -595,6 +595,8 @@ public class ClientRMService extends AbstractService implements throw RPCUtil.getRemoteException(ie); } + checkTags(submissionContext.getApplicationTags()); + if (timelineServiceV2Enabled) { // Sanity check for flow run String value = null; @@ -737,6 +739,31 @@ public class ClientRMService extends AbstractService implements return response; } + private void checkTags(Set tags) throws YarnException { + int appMaxTags = getConfig().getInt( + YarnConfiguration.RM_APPLICATION_MAX_TAGS, + YarnConfiguration.DEFAULT_RM_APPLICATION_MAX_TAGS); + int appMaxTagLength = getConfig().getInt( + YarnConfiguration.RM_APPLICATION_MAX_TAG_LENGTH, + YarnConfiguration.DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH); + if (tags.size() > appMaxTags) { + throw RPCUtil.getRemoteException(new IllegalArgumentException( + "Too many applicationTags, a maximum of only " + appMaxTags + + " are allowed!")); + } + for (String tag : tags) { + if (tag.length() > appMaxTagLength) { + throw RPCUtil.getRemoteException( + new IllegalArgumentException("Tag " + tag + " is too long, " + + "maximum allowed length of a tag is " + appMaxTagLength)); + } + if (!org.apache.commons.lang3.StringUtils.isAsciiPrintable(tag)) { + throw RPCUtil.getRemoteException(new IllegalArgumentException( + "A tag can only have ASCII " + "characters! Invalid tag - " + tag)); + } + } + } + @SuppressWarnings("unchecked") @Override public KillApplicationResponse forceKillApplication( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index b0e2502c054..3507b968a04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -574,6 +574,51 @@ public class TestClientRMService { } } + @Test + public void testApplicationTagsValidation() throws IOException { + YarnConfiguration conf = new YarnConfiguration(); + int maxtags = 3, appMaxTagLength = 5; + conf.setInt(YarnConfiguration.RM_APPLICATION_MAX_TAGS, maxtags); + conf.setInt(YarnConfiguration.RM_APPLICATION_MAX_TAG_LENGTH, + appMaxTagLength); + MockRM rm = new MockRM(conf); + rm.init(conf); + rm.start(); + + ClientRMService rmService = rm.getClientRMService(); + + List tags = Arrays.asList("Tag1", "Tag2", "Tag3", "Tag4"); + validateApplicationTag(rmService, tags, + "Too many applicationTags, a maximum of only " + maxtags + + " are allowed!"); + + tags = Arrays.asList("ApplicationTag1", "ApplicationTag2", + "ApplicationTag3"); + // tags are converted to lowercase in + // ApplicationSubmissionContext#setApplicationTags + validateApplicationTag(rmService, tags, + "Tag applicationtag1 is too long, maximum allowed length of a tag is " + + appMaxTagLength); + + tags = Arrays.asList("tãg1", "tag2#"); + validateApplicationTag(rmService, tags, + "A tag can only have ASCII characters! Invalid tag - tãg1"); + rm.close(); + } + + private void validateApplicationTag(ClientRMService rmService, + List tags, String errorMsg) { + SubmitApplicationRequest submitRequest = mockSubmitAppRequest( + getApplicationId(101), MockApps.newAppName(), QUEUE_1, + new HashSet(tags)); + try { + rmService.submitApplication(submitRequest); + Assert.fail(); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains(errorMsg)); + } + } + @Test public void testForceKillApplication() throws Exception { YarnConfiguration conf = new YarnConfiguration();