diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 833aeccd0bd..2ebf79cfae3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1954,7 +1954,10 @@ public class YarnConfiguration extends Configuration { false; public static final String APPLICATION_TAG_BASED_PLACEMENT_USER_WHITELIST = APPLICATION_TAG_BASED_PLACEMENT_PREFIX + ".username.whitelist"; - + public static final String APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION = + APPLICATION_TAG_BASED_PLACEMENT_PREFIX + ".force-lowercase"; + public static final boolean DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION = + true; /** Enable switch for container log monitoring. */ public static final String NM_CONTAINER_LOG_MONITOR_ENABLED = NM_PREFIX + "container-log-monitor.enable"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java index 32b576aeec1..97f4ad0338c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java @@ -18,6 +18,9 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION; + import java.util.EnumSet; import java.util.HashSet; import java.util.List; @@ -25,6 +28,7 @@ import java.util.Set; import org.apache.commons.lang3.Range; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; @@ -33,12 +37,14 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder; - +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.protobuf.TextFormat; @Private @Unstable public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { + private static volatile Boolean forceLowerCaseTags; + GetApplicationsRequestProto proto = GetApplicationsRequestProto.getDefaultInstance(); GetApplicationsRequestProto.Builder builder = null; boolean viaProto = false; @@ -56,11 +62,23 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { public GetApplicationsRequestPBImpl() { builder = GetApplicationsRequestProto.newBuilder(); + initLowerCaseConfig(); } public GetApplicationsRequestPBImpl(GetApplicationsRequestProto proto) { this.proto = proto; viaProto = true; + initLowerCaseConfig(); + } + + private static void initLowerCaseConfig() { + if (forceLowerCaseTags == null) { + Configuration conf = new Configuration(); + + forceLowerCaseTags = + conf.getBoolean(APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION, + DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION); + } } public synchronized GetApplicationsRequestProto getProto() { @@ -210,7 +228,8 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { // Convert applicationTags to lower case and add this.applicationTags = new HashSet(); for (String tag : tags) { - this.applicationTags.add(StringUtils.toLowerCase(tag)); + this.applicationTags.add( + forceLowerCaseTags ? StringUtils.toLowerCase(tag) : tag); } } @@ -406,4 +425,9 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { public String toString() { return TextFormat.shortDebugString(getProto()); } + + @VisibleForTesting + static void setForceLowerCaseTags(boolean convert) { + GetApplicationsRequestPBImpl.forceLowerCaseTags = convert; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index 5f9a71bd442..3eb139b18ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -18,6 +18,9 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -29,6 +32,7 @@ import java.util.TreeSet; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -50,13 +54,15 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto; import org.apache.hadoop.yarn.proto.YarnProtos.StringStringMapProto; - +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.protobuf.TextFormat; @Private @Unstable public class ApplicationSubmissionContextPBImpl extends ApplicationSubmissionContext { + private static volatile Boolean forceLowerCaseTags; + ApplicationSubmissionContextProto proto = ApplicationSubmissionContextProto.getDefaultInstance(); ApplicationSubmissionContextProto.Builder builder = null; @@ -75,14 +81,26 @@ extends ApplicationSubmissionContext { public ApplicationSubmissionContextPBImpl() { builder = ApplicationSubmissionContextProto.newBuilder(); + initLowerCaseConfig(); } public ApplicationSubmissionContextPBImpl( ApplicationSubmissionContextProto proto) { this.proto = proto; viaProto = true; + initLowerCaseConfig(); } - + + private static void initLowerCaseConfig() { + if (forceLowerCaseTags == null) { + Configuration conf = new Configuration(); + + forceLowerCaseTags = + conf.getBoolean(APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION, + DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION); + } + } + public synchronized ApplicationSubmissionContextProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); @@ -287,7 +305,8 @@ extends ApplicationSubmissionContext { // Convert applicationTags to lower case and add this.applicationTags = new TreeSet<>(); for (String tag : tags) { - this.applicationTags.add(StringUtils.toLowerCase(tag)); + this.applicationTags.add( + forceLowerCaseTags ? StringUtils.toLowerCase(tag) : tag); } } @@ -719,4 +738,9 @@ extends ApplicationSubmissionContext { this.schedulingProperties.clear(); this.schedulingProperties.putAll(schedulingPropertyMap); } + + @VisibleForTesting + static void setForceLowerCaseTags(boolean convert) { + ApplicationSubmissionContextPBImpl.forceLowerCaseTags = convert; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0e9c00beead..b64efba5dca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -4773,4 +4773,10 @@ yarn.resourcemanager.application.max-tag.length 100 + + + Specifies whether application tags should be converted to lowercase or not. + yarn.resourcemanager.application-tag-based-placement.force-lowercase + true + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TestGetApplicationsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TestGetApplicationsRequestPBImpl.java new file mode 100644 index 00000000000..ff9c778139f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TestGetApplicationsRequestPBImpl.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class TestGetApplicationsRequestPBImpl { + @Parameter + @SuppressWarnings("checkstyle:visibilitymodifier") + public GetApplicationsRequestPBImpl impl; + + @Test + public void testAppTagsLowerCaseConversionDefault() { + impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH")); + impl.getApplicationTags().forEach(s -> + assertEquals(s, s.toLowerCase())); + } + + @Test + public void testAppTagsLowerCaseConversionDisabled() { + GetApplicationsRequestPBImpl.setForceLowerCaseTags(false); + impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH")); + impl.getApplicationTags().forEach(s -> + assertNotEquals(s, s.toLowerCase())); + } + + @Test + public void testAppTagsLowerCaseConversionEnabled() { + GetApplicationsRequestPBImpl.setForceLowerCaseTags(true); + impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH")); + impl.getApplicationTags().forEach(s -> + assertEquals(s, s.toLowerCase())); + } + + @Parameters + public static Collection data() { + List list = new ArrayList<>(); + list.add(new Object[] {new GetApplicationsRequestPBImpl()}); + list.add(new Object[] {new GetApplicationsRequestPBImpl( + GetApplicationsRequestProto.newBuilder().build())}); + + return list; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationSubmissionContextPBImpl.java new file mode 100644 index 00000000000..a0d8bb3d692 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationSubmissionContextPBImpl.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class TestApplicationSubmissionContextPBImpl { + @Parameter + @SuppressWarnings("checkstyle:visibilitymodifier") + public ApplicationSubmissionContextPBImpl impl; + + @Test + public void testAppTagsLowerCaseConversionDefault() { + impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH")); + impl.getApplicationTags().forEach(s -> + assertEquals(s, s.toLowerCase())); + } + + @Test + public void testAppTagsLowerCaseConversionDisabled() { + ApplicationSubmissionContextPBImpl.setForceLowerCaseTags(false); + impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH")); + impl.getApplicationTags().forEach(s -> + assertNotEquals(s, s.toLowerCase())); + } + + @Test + public void testAppTagsLowerCaseConversionEnabled() { + ApplicationSubmissionContextPBImpl.setForceLowerCaseTags(true); + impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH")); + impl.getApplicationTags().forEach(s -> + assertEquals(s, s.toLowerCase())); + } + + @Parameters + public static Collection data() { + List list = new ArrayList<>(); + list.add(new Object[] {new ApplicationSubmissionContextPBImpl()}); + list.add(new Object[] {new ApplicationSubmissionContextPBImpl( + ApplicationSubmissionContextProto.newBuilder().build())}); + + return list; + } +}