Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9954. Configurable max application tags and max tag length. Contributed by Bilwa S T 

(cherry picked from commit cd6c10de442fc3a53c9ed5521ac1d944a6ac95c6)
(cherry picked from commit 2c79865b951d0fdea7f576ce31e310b4074ecedd)
This commit is contained in:
Jonathan Hung 2020-04-17 09:43:36 -07:00
parent 9ec141c77f
commit ebce5c74e6
6 changed files with 102 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
View File

@ -424,9 +424,9 @@ public abstract class ApplicationSubmissionContext {
/**
* Set tags for the application. A maximum of
* {@link YarnConfiguration#APPLICATION_MAX_TAGS} are allowed
* {@link YarnConfiguration#RM_APPLICATION_MAX_TAGS} are allowed
* per application. Each tag can be at most
* {@link YarnConfiguration#APPLICATION_MAX_TAG_LENGTH}
* {@link YarnConfiguration#RM_APPLICATION_MAX_TAG_LENGTH}
* characters, and can contain only ASCII characters.
*
* @param tags tags to set

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -92,10 +92,10 @@ public class YarnConfiguration extends Configuration {
YARN_SITE_CONFIGURATION_FILE,
CORE_SITE_CONFIGURATION_FILE));
@Evolving
@Deprecated
public static final int APPLICATION_MAX_TAGS = 10;
@Evolving
@Deprecated
public static final int APPLICATION_MAX_TAG_LENGTH = 100;
static {
@ -198,6 +198,18 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_ADDRESS =
"0.0.0.0:" + DEFAULT_RM_PORT;
/**Max number of application tags.*/
public static final String RM_APPLICATION_MAX_TAGS = RM_PREFIX
+ "application.max-tags";
public static final int DEFAULT_RM_APPLICATION_MAX_TAGS = 10;
/**Max length of each application tag.*/
public static final String RM_APPLICATION_MAX_TAG_LENGTH = RM_PREFIX
+ "application.max-tag.length";
public static final int DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH = 100;
public static final String RM_APPLICATION_MASTER_SERVICE_PROCESSORS =
RM_PREFIX + "application-master-service.processors";

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
View File

@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder;
@ -273,24 +272,6 @@ extends ApplicationSubmissionContext {
builder.setApplicationType((applicationType));
}
private void checkTags(Set<String> tags) {
if (tags.size() > YarnConfiguration.APPLICATION_MAX_TAGS) {
throw new IllegalArgumentException("Too many applicationTags, a maximum of only "
+ YarnConfiguration.APPLICATION_MAX_TAGS + " are allowed!");
}
for (String tag : tags) {
if (tag.length() > YarnConfiguration.APPLICATION_MAX_TAG_LENGTH) {
throw new IllegalArgumentException("Tag " + tag + " is too long, " +
"maximum allowed length of a tag is " +
YarnConfiguration.APPLICATION_MAX_TAG_LENGTH);
}
if (!CharMatcher.ASCII.matchesAllOf(tag)) {
throw new IllegalArgumentException("A tag can only have ASCII " +
"characters! Invalid tag - " + tag);
}
}
}
@Override
public void setApplicationTags(Set<String> tags) {
maybeInitBuilder();
@ -299,7 +280,6 @@ extends ApplicationSubmissionContext {
this.applicationTags = null;
return;
}
checkTags(tags);
// Convert applicationTags to lower case and add
this.applicationTags = new HashSet<String>();
for (String tag : tags) {

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -3609,4 +3609,17 @@
<value></value>
</property>
<property>
<description>Max number of application tags set by user in ApplicationSubmissionContext
while submitting application</description>
<name>yarn.resourcemanager.application.max-tags</name>
<value>10</value>
</property>
<property>
<description>Max length of each application tag set by user in ApplicationSubmissionContext
while submitting application.</description>
<name>yarn.resourcemanager.application.max-tag.length</name>
<value>100</value>
</property>
</configuration>

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -181,6 +181,7 @@ import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.CharMatcher;
/**
* The client interface to the Resource Manager. This module handles all the rpc
@ -584,6 +585,8 @@ public class ClientRMService extends AbstractService implements
throw RPCUtil.getRemoteException(ie);
}
checkTags(submissionContext.getApplicationTags());
if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
// Sanity check for flow run
String value = null;
@ -725,6 +728,31 @@ public class ClientRMService extends AbstractService implements
return response;
}
private void checkTags(Set<String> tags) throws YarnException {
int appMaxTags = getConfig().getInt(
YarnConfiguration.RM_APPLICATION_MAX_TAGS,
YarnConfiguration.DEFAULT_RM_APPLICATION_MAX_TAGS);
int appMaxTagLength = getConfig().getInt(
YarnConfiguration.RM_APPLICATION_MAX_TAG_LENGTH,
YarnConfiguration.DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH);
if (tags.size() > appMaxTags) {
throw RPCUtil.getRemoteException(new IllegalArgumentException(
"Too many applicationTags, a maximum of only " + appMaxTags
+ " are allowed!"));
}
for (String tag : tags) {
if (tag.length() > appMaxTagLength) {
throw RPCUtil.getRemoteException(
new IllegalArgumentException("Tag " + tag + " is too long, "
+ "maximum allowed length of a tag is " + appMaxTagLength));
}
if (!CharMatcher.ASCII.matchesAllOf(tag)) {
throw RPCUtil.getRemoteException(new IllegalArgumentException(
"A tag can only have ASCII " + "characters! Invalid tag - " + tag));
}
}
}
@SuppressWarnings("unchecked")
@Override
public KillApplicationResponse forceKillApplication(

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -521,6 +521,51 @@ public class TestClientRMService {
}
}
@Test
public void testApplicationTagsValidation() throws IOException {
YarnConfiguration conf = new YarnConfiguration();
int maxtags = 3, appMaxTagLength = 5;
conf.setInt(YarnConfiguration.RM_APPLICATION_MAX_TAGS, maxtags);
conf.setInt(YarnConfiguration.RM_APPLICATION_MAX_TAG_LENGTH,
appMaxTagLength);
MockRM rm = new MockRM(conf);
rm.init(conf);
rm.start();
ClientRMService rmService = rm.getClientRMService();
List<String> tags = Arrays.asList("Tag1", "Tag2", "Tag3", "Tag4");
validateApplicationTag(rmService, tags,
"Too many applicationTags, a maximum of only " + maxtags
+ " are allowed!");
tags = Arrays.asList("ApplicationTag1", "ApplicationTag2",
"ApplicationTag3");
// tags are converted to lowercase in
// ApplicationSubmissionContext#setApplicationTags
validateApplicationTag(rmService, tags,
"Tag applicationtag1 is too long, maximum allowed length of a tag is "
+ appMaxTagLength);
tags = Arrays.asList("tãg1", "tag2#");
validateApplicationTag(rmService, tags,
"A tag can only have ASCII characters! Invalid tag - tãg1");
rm.close();
}
private void validateApplicationTag(ClientRMService rmService,
List<String> tags, String errorMsg) {
SubmitApplicationRequest submitRequest = mockSubmitAppRequest(
getApplicationId(101), MockApps.newAppName(), QUEUE_1,
new HashSet<String>(tags));
try {
rmService.submitApplication(submitRequest);
Assert.fail();
} catch (Exception ex) {
Assert.assertTrue(ex.getMessage().contains(errorMsg));
}
}
@Test
public void testForceKillApplication() throws Exception {
YarnConfiguration conf = new YarnConfiguration();