YARN-9954. Configurable max application tags and max tag length. Contributed by Bilwa S T
This commit is contained in:
parent
96d7ceb39a
commit
49ae9b2137
|
@ -424,9 +424,9 @@ public abstract class ApplicationSubmissionContext {
|
|||
|
||||
/**
|
||||
* Set tags for the application. A maximum of
|
||||
* {@link YarnConfiguration#APPLICATION_MAX_TAGS} are allowed
|
||||
* {@link YarnConfiguration#RM_APPLICATION_MAX_TAGS} are allowed
|
||||
* per application. Each tag can be at most
|
||||
* {@link YarnConfiguration#APPLICATION_MAX_TAG_LENGTH}
|
||||
* {@link YarnConfiguration#RM_APPLICATION_MAX_TAG_LENGTH}
|
||||
* characters, and can contain only ASCII characters.
|
||||
*
|
||||
* @param tags tags to set
|
||||
|
|
|
@ -93,10 +93,10 @@ public class YarnConfiguration extends Configuration {
|
|||
YARN_SITE_CONFIGURATION_FILE,
|
||||
CORE_SITE_CONFIGURATION_FILE));
|
||||
|
||||
@Evolving
|
||||
@Deprecated
|
||||
public static final int APPLICATION_MAX_TAGS = 10;
|
||||
|
||||
@Evolving
|
||||
@Deprecated
|
||||
public static final int APPLICATION_MAX_TAG_LENGTH = 100;
|
||||
|
||||
static {
|
||||
|
@ -207,6 +207,18 @@ public class YarnConfiguration extends Configuration {
|
|||
public static final String DEFAULT_RM_ADDRESS =
|
||||
"0.0.0.0:" + DEFAULT_RM_PORT;
|
||||
|
||||
/**Max number of application tags.*/
|
||||
public static final String RM_APPLICATION_MAX_TAGS = RM_PREFIX
|
||||
+ "application.max-tags";
|
||||
|
||||
public static final int DEFAULT_RM_APPLICATION_MAX_TAGS = 10;
|
||||
|
||||
/**Max length of each application tag.*/
|
||||
public static final String RM_APPLICATION_MAX_TAG_LENGTH = RM_PREFIX
|
||||
+ "application.max-tag.length";
|
||||
|
||||
public static final int DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH = 100;
|
||||
|
||||
public static final String RM_APPLICATION_MASTER_SERVICE_PROCESSORS =
|
||||
RM_PREFIX + "application-master-service.processors";
|
||||
|
||||
|
|
|
@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
|
|||
import org.apache.hadoop.yarn.api.records.ReservationId;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceRequest;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder;
|
||||
|
@ -277,24 +276,6 @@ extends ApplicationSubmissionContext {
|
|||
builder.setApplicationType((applicationType));
|
||||
}
|
||||
|
||||
private void checkTags(Set<String> tags) {
|
||||
if (tags.size() > YarnConfiguration.APPLICATION_MAX_TAGS) {
|
||||
throw new IllegalArgumentException("Too many applicationTags, a maximum of only "
|
||||
+ YarnConfiguration.APPLICATION_MAX_TAGS + " are allowed!");
|
||||
}
|
||||
for (String tag : tags) {
|
||||
if (tag.length() > YarnConfiguration.APPLICATION_MAX_TAG_LENGTH) {
|
||||
throw new IllegalArgumentException("Tag " + tag + " is too long, " +
|
||||
"maximum allowed length of a tag is " +
|
||||
YarnConfiguration.APPLICATION_MAX_TAG_LENGTH);
|
||||
}
|
||||
if (!org.apache.commons.lang3.StringUtils.isAsciiPrintable(tag)) {
|
||||
throw new IllegalArgumentException("A tag can only have ASCII " +
|
||||
"characters! Invalid tag - " + tag);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void setApplicationTags(Set<String> tags) {
|
||||
maybeInitBuilder();
|
||||
|
@ -303,7 +284,6 @@ extends ApplicationSubmissionContext {
|
|||
this.applicationTags = null;
|
||||
return;
|
||||
}
|
||||
checkTags(tags);
|
||||
// Convert applicationTags to lower case and add
|
||||
this.applicationTags = new TreeSet<>();
|
||||
for (String tag : tags) {
|
||||
|
|
|
@ -4547,4 +4547,17 @@
|
|||
<value></value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>Max number of application tags set by user in ApplicationSubmissionContext
|
||||
while submitting application</description>
|
||||
<name>yarn.resourcemanager.application.max-tags</name>
|
||||
<value>10</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>Max length of each application tag set by user in ApplicationSubmissionContext
|
||||
while submitting application.</description>
|
||||
<name>yarn.resourcemanager.application.max-tag.length</name>
|
||||
<value>100</value>
|
||||
</property>
|
||||
</configuration>
|
||||
|
|
|
@ -610,6 +610,8 @@ public class ClientRMService extends AbstractService implements
|
|||
throw RPCUtil.getRemoteException(ie);
|
||||
}
|
||||
|
||||
checkTags(submissionContext.getApplicationTags());
|
||||
|
||||
if (timelineServiceV2Enabled) {
|
||||
// Sanity check for flow run
|
||||
String value = null;
|
||||
|
@ -752,6 +754,31 @@ public class ClientRMService extends AbstractService implements
|
|||
return response;
|
||||
}
|
||||
|
||||
private void checkTags(Set<String> tags) throws YarnException {
|
||||
int appMaxTags = getConfig().getInt(
|
||||
YarnConfiguration.RM_APPLICATION_MAX_TAGS,
|
||||
YarnConfiguration.DEFAULT_RM_APPLICATION_MAX_TAGS);
|
||||
int appMaxTagLength = getConfig().getInt(
|
||||
YarnConfiguration.RM_APPLICATION_MAX_TAG_LENGTH,
|
||||
YarnConfiguration.DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH);
|
||||
if (tags.size() > appMaxTags) {
|
||||
throw RPCUtil.getRemoteException(new IllegalArgumentException(
|
||||
"Too many applicationTags, a maximum of only " + appMaxTags
|
||||
+ " are allowed!"));
|
||||
}
|
||||
for (String tag : tags) {
|
||||
if (tag.length() > appMaxTagLength) {
|
||||
throw RPCUtil.getRemoteException(
|
||||
new IllegalArgumentException("Tag " + tag + " is too long, "
|
||||
+ "maximum allowed length of a tag is " + appMaxTagLength));
|
||||
}
|
||||
if (!org.apache.commons.lang3.StringUtils.isAsciiPrintable(tag)) {
|
||||
throw RPCUtil.getRemoteException(new IllegalArgumentException(
|
||||
"A tag can only have ASCII " + "characters! Invalid tag - " + tag));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public KillApplicationResponse forceKillApplication(
|
||||
|
|
|
@ -593,6 +593,51 @@ public class TestClientRMService {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testApplicationTagsValidation() throws IOException {
|
||||
YarnConfiguration conf = new YarnConfiguration();
|
||||
int maxtags = 3, appMaxTagLength = 5;
|
||||
conf.setInt(YarnConfiguration.RM_APPLICATION_MAX_TAGS, maxtags);
|
||||
conf.setInt(YarnConfiguration.RM_APPLICATION_MAX_TAG_LENGTH,
|
||||
appMaxTagLength);
|
||||
MockRM rm = new MockRM(conf);
|
||||
rm.init(conf);
|
||||
rm.start();
|
||||
|
||||
ClientRMService rmService = rm.getClientRMService();
|
||||
|
||||
List<String> tags = Arrays.asList("Tag1", "Tag2", "Tag3", "Tag4");
|
||||
validateApplicationTag(rmService, tags,
|
||||
"Too many applicationTags, a maximum of only " + maxtags
|
||||
+ " are allowed!");
|
||||
|
||||
tags = Arrays.asList("ApplicationTag1", "ApplicationTag2",
|
||||
"ApplicationTag3");
|
||||
// tags are converted to lowercase in
|
||||
// ApplicationSubmissionContext#setApplicationTags
|
||||
validateApplicationTag(rmService, tags,
|
||||
"Tag applicationtag1 is too long, maximum allowed length of a tag is "
|
||||
+ appMaxTagLength);
|
||||
|
||||
tags = Arrays.asList("tãg1", "tag2#");
|
||||
validateApplicationTag(rmService, tags,
|
||||
"A tag can only have ASCII characters! Invalid tag - tãg1");
|
||||
rm.close();
|
||||
}
|
||||
|
||||
private void validateApplicationTag(ClientRMService rmService,
|
||||
List<String> tags, String errorMsg) {
|
||||
SubmitApplicationRequest submitRequest = mockSubmitAppRequest(
|
||||
getApplicationId(101), MockApps.newAppName(), QUEUE_1,
|
||||
new HashSet<String>(tags));
|
||||
try {
|
||||
rmService.submitApplication(submitRequest);
|
||||
Assert.fail();
|
||||
} catch (Exception ex) {
|
||||
Assert.assertTrue(ex.getMessage().contains(errorMsg));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testForceKillApplication() throws Exception {
|
||||
YarnConfiguration conf = new YarnConfiguration();
|
||||
|
|
Loading…
Reference in New Issue