From 133eb31d549b8e54b35a6cfccb4e81a70195af23 Mon Sep 17 00:00:00 2001 From: Varun Vasudev Date: Fri, 11 Mar 2016 15:03:15 +0530 Subject: [PATCH] YARN-4715. Add support to read resource types from a config file. Contributed by Varun Vasudev. (cherry picked from commit ceb12c59a34677518a198e19acb4e29cdfc8a0e1) --- .../hadoop/yarn/api/records/Resource.java | 24 +- .../yarn/api/records/ResourceInformation.java | 8 +- .../hadoop/yarn/conf/YarnConfiguration.java | 12 + .../exceptions/ResourceNotFoundException.java | 2 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 8 + .../api/records/impl/pb/ResourcePBImpl.java | 72 ++--- .../resource/DominantResourceCalculator.java | 5 +- .../yarn/util/resource/ResourceUtils.java | 229 ++++++++++++++++ .../hadoop/yarn/util/resource/Resources.java | 18 +- .../src/main/resources/yarn-default.xml | 10 + .../yarn/util/resource/TestResourceUtils.java | 248 ++++++++++++++++++ .../resource-types/resource-types-1.xml | 18 ++ .../resource-types/resource-types-2.xml | 29 ++ .../resource-types/resource-types-3.xml | 24 ++ .../resource-types/resource-types-4.xml | 34 +++ .../resource-types/resource-types-error-1.xml | 29 ++ .../resource-types/resource-types-error-2.xml | 29 ++ .../resource-types/resource-types-error-3.xml | 29 ++ .../resource-types/resource-types-error-4.xml | 24 ++ 19 files changed, 762 insertions(+), 90 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-1.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-2.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-3.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-1.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-4.xml diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index fab80630e7b..f8d250bf55f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -101,15 +101,6 @@ public abstract class Resource implements Comparable { return new SimpleResource(memory, vCores); } - @Public - @Stable - public static Resource newInstance( - Map resources) { - Resource resource = Records.newRecord(Resource.class); - resource.setResources(resources); - return resource; - } - /** * This method is DEPRECATED: * Use {@link Resource#getMemorySize()} instead @@ -233,15 +224,6 @@ public abstract class Resource implements Comparable { @Evolving public abstract Long getResourceValue(String resource) throws YarnException; - /** - * Set the resources to the map specified. - * - * @param resources Desired resources - */ - @Public - @Evolving - public abstract void setResources(Map resources); - /** * Set the ResourceInformation object for a particular resource. * @@ -276,8 +258,8 @@ public abstract class Resource implements Comparable { result = prime * result + getVirtualCores(); for (Map.Entry entry : getResources() .entrySet()) { - if (entry.getKey().equals(ResourceInformation.MEMORY.getName()) || entry - .getKey().equals(ResourceInformation.VCORES.getName())) { + if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) + || entry.getKey().equals(ResourceInformation.VCORES.getName())) { continue; } result = prime * result + entry.getValue().hashCode(); @@ -320,7 +302,7 @@ public abstract class Resource implements Comparable { .append(getVirtualCores()); for (Map.Entry entry : getResources() .entrySet()) { - if (entry.getKey().equals(ResourceInformation.MEMORY.getName()) + if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) && entry.getValue().getUnits() .equals(ResourceInformation.MEMORY_MB.getUnits())) { continue; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 4e780c15fee..80e3192537e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -32,11 +32,9 @@ public class ResourceInformation implements Comparable { private ResourceTypes resourceType; private Long value; - private static final String MEMORY_URI = "yarn.io/memory"; - private static final String VCORES_URI = "yarn.io/vcores"; + private static final String MEMORY_URI = "memory-mb"; + private static final String VCORES_URI = "vcores"; - public static final ResourceInformation MEMORY = - ResourceInformation.newInstance(MEMORY_URI); public static final ResourceInformation MEMORY_MB = ResourceInformation.newInstance(MEMORY_URI, "M"); public static final ResourceInformation VCORES = @@ -77,7 +75,7 @@ public class ResourceInformation implements Comparable { public void setUnits(String rUnits) { if (!UnitsConversionUtil.KNOWN_UNITS.contains(rUnits)) { throw new IllegalArgumentException( - "Unknown unit '" + units + "'. Known units are " + "Unknown unit '" + rUnits + "'. Known units are " + UnitsConversionUtil.KNOWN_UNITS); } this.units = rUnits; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 2168ca52358..33e3d7dfaf5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -62,9 +62,14 @@ public class YarnConfiguration extends Configuration { @Private public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; + @Private + public static final String RESOURCE_TYPES_CONFIGURATION_FILE = + "resource-types.xml"; + @Private public static final List RM_CONFIGURATION_FILES = Collections.unmodifiableList(Arrays.asList( + RESOURCE_TYPES_CONFIGURATION_FILE, DR_CONFIGURATION_FILE, CS_CONFIGURATION_FILE, HADOOP_POLICY_CONFIGURATION_FILE, @@ -108,6 +113,13 @@ public class YarnConfiguration extends Configuration { public static final String YARN_PREFIX = "yarn."; + ///////////////////////////// + // Scheduler resource types configs + //////////////////////////// + + public static final String RESOURCE_TYPES = + YarnConfiguration.YARN_PREFIX + "resource-types"; + /** Delay before deleting resource to ease debugging of NM issues */ public static final String DEBUG_NM_DELETE_DELAY_SEC = YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java index 427703413f8..b5fece7dc8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java @@ -27,7 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability; */ @InterfaceAudience.Public @InterfaceStability.Unstable -public class ResourceNotFoundException extends YarnException { +public class ResourceNotFoundException extends YarnRuntimeException { private static final long serialVersionUID = 10081982L; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index 8c0b4db7aa9..276606178f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -204,6 +204,14 @@ true + + + ${project.basedir}/src/test/resources + + + ${project.basedir}/src/test/resources/resource-types + + org.apache.rat diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 6cd824cf44f..ad1839a6dd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -25,10 +25,11 @@ import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; -import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.UnitsConversionUtil; import java.util.*; @@ -36,6 +37,7 @@ import java.util.*; @Private @Unstable public class ResourcePBImpl extends Resource { + ResourceProto proto = ResourceProto.getDefaultInstance(); ResourceProto.Builder builder = null; boolean viaProto = false; @@ -89,16 +91,11 @@ public class ResourcePBImpl extends Resource { @Override public long getMemorySize() { - try { - ResourceInformation ri = - this.getResourceInformation(ResourceInformation.MEMORY.getName()); - return (int) UnitsConversionUtil - .convert(ri.getUnits(), "M", ri.getValue()).longValue(); - } catch (YarnException ye) { - // memory should always be present - initResourcesMap(); - return 0; - } + // memory should always be present + initResourcesMap(); + ResourceInformation ri = + this.getResourceInformation(ResourceInformation.MEMORY_MB.getName()); + return UnitsConversionUtil.convert(ri.getUnits(), "M", ri.getValue()); } @Override @@ -117,14 +114,10 @@ public class ResourcePBImpl extends Resource { @Override public int getVirtualCores() { - try { - return (int) this.getResourceValue(ResourceInformation.VCORES.getName()) - .longValue(); - } catch (YarnException ye) { - // vcores should always be present - initResourcesMap(); - return 0; - } + // vcores should always be present + initResourcesMap(); + return this.getResourceValue(ResourceInformation.VCORES.getName()) + .intValue(); } @Override @@ -163,21 +156,6 @@ public class ResourcePBImpl extends Resource { } } - @Override - public void setResources(Map resources) { - maybeInitBuilder(); - if (resources == null || resources.isEmpty()) { - builder.clearResourceValueMap(); - } else { - for (Map.Entry entry : resources.entrySet()) { - if (!entry.getKey().equals(entry.getValue().getName())) { - entry.getValue().setName(entry.getKey()); - } - } - } - this.resources = resources; - } - @Override public void setResourceInformation(String resource, ResourceInformation resourceInformation) { @@ -216,8 +194,7 @@ public class ResourcePBImpl extends Resource { } @Override - public ResourceInformation getResourceInformation(String resource) - throws YarnException { + public ResourceInformation getResourceInformation(String resource) { initResources(); if (this.resources.containsKey(resource)) { return this.resources.get(resource); @@ -226,7 +203,7 @@ public class ResourcePBImpl extends Resource { } @Override - public Long getResourceValue(String resource) throws YarnException { + public Long getResourceValue(String resource) { initResources(); if (this.resources.containsKey(resource)) { return this.resources.get(resource).getValue(); @@ -237,18 +214,15 @@ public class ResourcePBImpl extends Resource { private void initResourcesMap() { if (resources == null) { resources = new HashMap<>(); - } - ResourceInformation ri; - if (!resources.containsKey(ResourceInformation.MEMORY.getName())) { - ri = ResourceInformation - .newInstance(ResourceInformation.MEMORY_MB.getName(), - ResourceInformation.MEMORY_MB.getUnits()); - this.resources.put(ResourceInformation.MEMORY.getName(), ri); - } - if (!resources.containsKey(ResourceInformation.VCORES.getName())) { - ri = - ResourceInformation.newInstance(ResourceInformation.VCORES.getName()); - this.resources.put(ResourceInformation.VCORES.getName(), ri); + Map types = ResourceUtils.getResourceTypes(); + if (types == null) { + throw new YarnRuntimeException( + "Got null return value from ResourceUtils.getResourceTypes()"); + } + for (Map.Entry entry : types.entrySet()) { + resources.put(entry.getKey(), + ResourceInformation.newInstance(entry.getValue())); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index a94e7a51ae1..0412c0fac05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.UnitsConversionUtil; -import java.util.HashSet; import java.util.Set; /** @@ -60,9 +59,7 @@ public class DominantResourceCalculator extends ResourceCalculator { private Set resourceNames; public DominantResourceCalculator() { - resourceNames = new HashSet<>(); - resourceNames.add(ResourceInformation.MEMORY.getName()); - resourceNames.add(ResourceInformation.VCORES.getName()); + resourceNames = ResourceUtils.getResourceTypes().keySet(); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java new file mode 100644 index 00000000000..0316f010e88 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util.resource; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Helper class to read the resource-types to be supported by the system. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class ResourceUtils { + + public static final String UNITS = ".units"; + public static final String TYPE = ".type"; + + private static final Set DISALLOWED_NAMES = new HashSet<>(); + static { + DISALLOWED_NAMES.add("memory"); + DISALLOWED_NAMES.add(ResourceInformation.MEMORY_MB.getName()); + DISALLOWED_NAMES.add(ResourceInformation.VCORES.getName()); + } + + private static volatile Object lock; + private static Map readOnlyResources; + + static final Log LOG = LogFactory.getLog(ResourceUtils.class); + + private ResourceUtils() { + } + + private static void checkMandatatoryResources( + Map resourceInformationMap) + throws YarnRuntimeException { + String memory = ResourceInformation.MEMORY_MB.getName(); + String vcores = ResourceInformation.VCORES.getName(); + if (resourceInformationMap.containsKey(memory)) { + ResourceInformation memInfo = resourceInformationMap.get(memory); + String memUnits = ResourceInformation.MEMORY_MB.getUnits(); + ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType(); + if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType() + .equals(memType)) { + throw new YarnRuntimeException( + "Attempt to re-define mandatory resource 'memory-mb'. It can only" + + " be of type 'COUNTABLE' and have units 'M'."); + } + } + + if (resourceInformationMap.containsKey(vcores)) { + ResourceInformation vcoreInfo = resourceInformationMap.get(vcores); + String vcoreUnits = ResourceInformation.VCORES.getUnits(); + ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType(); + if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo + .getResourceType().equals(vcoreType)) { + throw new YarnRuntimeException( + "Attempt to re-define mandatory resource 'vcores'. It can only be" + + " of type 'COUNTABLE' and have units ''(no units)."); + } + } + } + + private static void addManadtoryResources( + Map res) { + ResourceInformation ri; + if (!res.containsKey(ResourceInformation.MEMORY_MB.getName())) { + LOG.info("Adding resource type - name = " + ResourceInformation.MEMORY_MB + .getName() + ", units = " + ResourceInformation.MEMORY_MB.getUnits() + + ", type = " + ResourceTypes.COUNTABLE); + ri = ResourceInformation + .newInstance(ResourceInformation.MEMORY_MB.getName(), + ResourceInformation.MEMORY_MB.getUnits()); + res.put(ResourceInformation.MEMORY_MB.getName(), ri); + } + if (!res.containsKey(ResourceInformation.VCORES.getName())) { + LOG.info("Adding resource type - name = " + ResourceInformation.VCORES + .getName() + ", units = , type = " + ResourceTypes.COUNTABLE); + ri = + ResourceInformation.newInstance(ResourceInformation.VCORES.getName()); + res.put(ResourceInformation.VCORES.getName(), ri); + } + } + + @VisibleForTesting + static void initializeResourcesMap(Configuration conf, + Map resourceInformationMap) { + + String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES); + if (resourceNames != null && resourceNames.length != 0) { + for (String resourceName : resourceNames) { + String resourceUnits = conf.get( + YarnConfiguration.RESOURCE_TYPES + "." + resourceName + UNITS, ""); + String resourceTypeName = conf.get( + YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE, + ResourceTypes.COUNTABLE.toString()); + if (resourceName == null || resourceName.isEmpty() + || resourceUnits == null || resourceTypeName == null) { + throw new YarnRuntimeException( + "Incomplete configuration for resource type '" + resourceName + + "'. One of name, units or type is configured incorrectly."); + } + if (DISALLOWED_NAMES.contains(resourceName)) { + throw new YarnRuntimeException( + "Resource type cannot be named '" + resourceName + + "'. That name is disallowed."); + } + ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName); + LOG.info("Adding resource type - name = " + resourceName + ", units = " + + resourceUnits + ", type = " + resourceTypeName); + if (resourceInformationMap.containsKey(resourceName)) { + throw new YarnRuntimeException( + "Error in config, key '" + resourceName + "' specified twice"); + } + resourceInformationMap.put(resourceName, ResourceInformation + .newInstance(resourceName, resourceUnits, 0L, resourceType)); + } + } + checkMandatatoryResources(resourceInformationMap); + addManadtoryResources(resourceInformationMap); + readOnlyResources = Collections.unmodifiableMap(resourceInformationMap); + } + + /** + * Get the resource types to be supported by the system. + * @return A map of the resource name to a ResouceInformation object + * which contains details such as the unit. + */ + public static Map getResourceTypes() { + return getResourceTypes(null, + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + } + + private static Map getResourceTypes( + Configuration conf, String resourceFile) { + if (lock == null) { + synchronized (ResourceUtils.class) { + if (lock == null) { + synchronized (ResourceUtils.class) { + lock = new Object(); + Map resources = new HashMap<>(); + if (conf == null) { + conf = new YarnConfiguration(); + } + try { + InputStream ris = getConfInputStream(resourceFile, conf); + LOG.debug("Found " + resourceFile + ", adding to configuration"); + conf.addResource(ris); + initializeResourcesMap(conf, resources); + return resources; + } catch (FileNotFoundException fe) { + LOG.info("Unable to find '" + resourceFile + + "'. Falling back to memory and vcores as resources", fe); + initializeResourcesMap(conf, resources); + } catch (IOException ie) { + LOG.fatal( + "Exception trying to read resource types configuration '" + + resourceFile + "'.", ie); + throw new YarnRuntimeException(ie); + } catch (YarnException ye) { + LOG.fatal( + "YARN Exception trying to read resource types configuration '" + + resourceFile + "'.", ye); + throw new YarnRuntimeException(ye); + } + } + } + } + } + return readOnlyResources; + } + + static InputStream getConfInputStream(String resourceFile, Configuration conf) + throws IOException, YarnException { + + ConfigurationProvider provider = + ConfigurationProviderFactory.getConfigurationProvider(conf); + InputStream ris = provider.getConfigurationInputStream(conf, resourceFile); + if (ris == null) { + if (conf.getResource(resourceFile) == null) { + throw new FileNotFoundException("Unable to find " + resourceFile); + } + throw new IOException( + "Unable to open resource types file '" + resourceFile + + "'. Using provider " + provider); + } + return ris; + } + + @VisibleForTesting + static void resetResourceTypes() { + lock = null; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index eaebf7666cc..836f87e6c86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -119,11 +119,6 @@ public class Resources { throw new YarnException("" + resource + " not found"); } - @Override - public void setResources(Map resources) { - throw new RuntimeException(name + " cannot be modified!"); - } - @Override public void setResourceInformation(String resource, ResourceInformation resourceInformation) @@ -143,11 +138,14 @@ public class Resources { // needs to be Integer.MAX_VALUE int max = resourceValue > Integer.MAX_VALUE ? Integer.MAX_VALUE : resourceValue.intValue(); - tmp.put(ResourceInformation.MEMORY.getName(), ResourceInformation - .newInstance(ResourceInformation.MEMORY.getName(), - ResourceInformation.MEMORY_MB.getUnits(), (long) max)); - tmp.put(ResourceInformation.VCORES.getName(), ResourceInformation - .newInstance(ResourceInformation.VCORES.getName(), (long) max)); + Map types = ResourceUtils.getResourceTypes(); + if (types != null) { + for (Map.Entry entry : types.entrySet()) { + tmp.put(entry.getKey(), + ResourceInformation.newInstance(entry.getValue())); + tmp.get(entry.getKey()).setValue((long) max); + } + } return tmp; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 5c7705b7155..54a482d2902 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3479,6 +3479,16 @@ + + + yarn.resource-types + + + The resource types to be used for scheduling. Use resource-types.xml + to specify details about the individual resource types. + + + Containers launcher implementation for determining how containers diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java new file mode 100644 index 00000000000..606e89cf2a2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -0,0 +1,248 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util.resource; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.SystemClock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +public class TestResourceUtils { + + static class ResourceFileInformation { + String filename; + int resourceCount; + Map resourceNameUnitsMap; + + public ResourceFileInformation(String name, int count) { + filename = name; + resourceCount = count; + resourceNameUnitsMap = new HashMap<>(); + } + } + + @Before + public void setup() { + ResourceUtils.resetResourceTypes(); + } + + private void testMemoryAndVcores(Map res) { + String memory = ResourceInformation.MEMORY_MB.getName(); + String vcores = ResourceInformation.VCORES.getName(); + Assert.assertTrue("Resource 'memory' missing", res.containsKey(memory)); + Assert.assertEquals("'memory' units incorrect", + ResourceInformation.MEMORY_MB.getUnits(), res.get(memory).getUnits()); + Assert.assertEquals("'memory' types incorrect", + ResourceInformation.MEMORY_MB.getResourceType(), + res.get(memory).getResourceType()); + Assert.assertTrue("Resource 'vcores' missing", res.containsKey(vcores)); + Assert.assertEquals("'vcores' units incorrect", + ResourceInformation.VCORES.getUnits(), res.get(vcores).getUnits()); + Assert.assertEquals("'vcores' type incorrect", + ResourceInformation.VCORES.getResourceType(), + res.get(vcores).getResourceType()); + } + + @Test + public void testGetResourceTypes() throws Exception { + + Map res = ResourceUtils.getResourceTypes(); + Assert.assertEquals(2, res.size()); + testMemoryAndVcores(res); + } + + @Test + public void testGetResourceTypesConfigs() throws Exception { + + Configuration conf = new YarnConfiguration(); + + ResourceFileInformation testFile1 = + new ResourceFileInformation("resource-types-1.xml", 2); + ResourceFileInformation testFile2 = + new ResourceFileInformation("resource-types-2.xml", 3); + testFile2.resourceNameUnitsMap.put("resource1", "G"); + ResourceFileInformation testFile3 = + new ResourceFileInformation("resource-types-3.xml", 3); + testFile3.resourceNameUnitsMap.put("resource2", ""); + ResourceFileInformation testFile4 = + new ResourceFileInformation("resource-types-4.xml", 4); + testFile4.resourceNameUnitsMap.put("resource1", "G"); + testFile4.resourceNameUnitsMap.put("resource2", "m"); + + ResourceFileInformation[] tests = + { testFile1, testFile2, testFile3, testFile4 }; + Map res; + for (ResourceFileInformation testInformation : tests) { + ResourceUtils.resetResourceTypes(); + File source = new File( + conf.getClassLoader().getResource(testInformation.filename) + .getFile()); + File dest = new File(source.getParent(), "resource-types.xml"); + FileUtils.copyFile(source, dest); + res = ResourceUtils.getResourceTypes(); + testMemoryAndVcores(res); + Assert.assertEquals(testInformation.resourceCount, res.size()); + for (Map.Entry entry : testInformation.resourceNameUnitsMap + .entrySet()) { + String resourceName = entry.getKey(); + Assert.assertTrue("Missing key " + resourceName, + res.containsKey(resourceName)); + Assert.assertEquals(entry.getValue(), res.get(resourceName).getUnits()); + } + dest.delete(); + } + } + + @Test + public void testGetResourceTypesConfigErrors() throws Exception { + Configuration conf = new YarnConfiguration(); + + String[] resourceFiles = + { "resource-types-error-1.xml", "resource-types-error-2.xml", + "resource-types-error-3.xml", "resource-types-error-4.xml" }; + for (String resourceFile : resourceFiles) { + ResourceUtils.resetResourceTypes(); + File dest = null; + try { + File source = + new File(conf.getClassLoader().getResource(resourceFile).getFile()); + dest = new File(source.getParent(), "resource-types.xml"); + FileUtils.copyFile(source, dest); + ResourceUtils.getResourceTypes(); + Assert.fail("Expected error with file " + resourceFile); + } catch (NullPointerException ne) { + throw ne; + } catch (Exception e) { + if (dest != null) { + dest.delete(); + } + } + } + } + + @Test + public void testInitializeResourcesMap() throws Exception { + String[] empty = { "", "" }; + String[] res1 = { "resource1", "m" }; + String[] res2 = { "resource2", "G" }; + String[][] test1 = { empty }; + String[][] test2 = { res1 }; + String[][] test3 = { res2 }; + String[][] test4 = { res1, res2 }; + + String[][][] allTests = { test1, test2, test3, test4 }; + + for (String[][] test : allTests) { + + Configuration conf = new YarnConfiguration(); + String resSt = ""; + for (String[] resources : test) { + resSt += (resources[0] + ","); + } + resSt = resSt.substring(0, resSt.length() - 1); + conf.set(YarnConfiguration.RESOURCE_TYPES, resSt); + for (String[] resources : test) { + String name = + YarnConfiguration.RESOURCE_TYPES + "." + resources[0] + ".units"; + conf.set(name, resources[1]); + } + Map ret = new HashMap<>(); + ResourceUtils.initializeResourcesMap(conf, ret); + // for test1, 4 - length will be 1, 4 + // for the others, len will be 3 + int len = 3; + if (test == test1) { + len = 2; + } else if (test == test4) { + len = 4; + } + + Assert.assertEquals(len, ret.size()); + for (String[] resources : test) { + if (resources[0].length() == 0) { + continue; + } + Assert.assertTrue(ret.containsKey(resources[0])); + ResourceInformation resInfo = ret.get(resources[0]); + Assert.assertEquals(resources[1], resInfo.getUnits()); + Assert.assertEquals(ResourceTypes.COUNTABLE, resInfo.getResourceType()); + } + // we must always have memory and vcores with their fixed units + Assert.assertTrue(ret.containsKey("memory-mb")); + ResourceInformation memInfo = ret.get("memory-mb"); + Assert.assertEquals("M", memInfo.getUnits()); + Assert.assertEquals(ResourceTypes.COUNTABLE, memInfo.getResourceType()); + Assert.assertTrue(ret.containsKey("vcores")); + ResourceInformation vcoresInfo = ret.get("vcores"); + Assert.assertEquals("", vcoresInfo.getUnits()); + Assert + .assertEquals(ResourceTypes.COUNTABLE, vcoresInfo.getResourceType()); + } + } + + @Test + public void testInitializeResourcesMapErrors() throws Exception { + + String[] mem1 = { "memory-mb", "" }; + String[] vcores1 = { "vcores", "M" }; + + String[] mem2 = { "memory-mb", "m" }; + String[] vcores2 = { "vcores", "G" }; + + String[] mem3 = { "memory", "" }; + + String[][] test1 = { mem1, vcores1 }; + String[][] test2 = { mem2, vcores2 }; + String[][] test3 = { mem3 }; + + String[][][] allTests = { test1, test2, test3 }; + + for (String[][] test : allTests) { + + Configuration conf = new YarnConfiguration(); + String resSt = ""; + for (String[] resources : test) { + resSt += (resources[0] + ","); + } + resSt = resSt.substring(0, resSt.length() - 1); + conf.set(YarnConfiguration.RESOURCE_TYPES, resSt); + for (String[] resources : test) { + String name = + YarnConfiguration.RESOURCE_TYPES + "." + resources[0] + ".units"; + conf.set(name, resources[1]); + } + Map ret = new HashMap<>(); + try { + ResourceUtils.initializeResourcesMap(conf, ret); + Assert.fail("resource map initialization should fail"); + } catch (Exception e) { + // do nothing + } + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-1.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-1.xml new file mode 100644 index 00000000000..3ec106dfbb2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-1.xml @@ -0,0 +1,18 @@ + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-2.xml new file mode 100644 index 00000000000..6e5885ed7d7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-2.xml @@ -0,0 +1,29 @@ + + + + + + + + yarn.resource-types + resource1 + + + + yarn.resource-types.resource1.units + G + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-3.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-3.xml new file mode 100644 index 00000000000..8fd6fefa8f1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-3.xml @@ -0,0 +1,24 @@ + + + + + + + + yarn.resource-types + resource2 + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml new file mode 100644 index 00000000000..c84316a536e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml @@ -0,0 +1,34 @@ + + + + + + + + yarn.resource-types + resource1,resource2 + + + + yarn.resource-types.resource1.units + G + + + + yarn.resource-types.resource2.units + m + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-1.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-1.xml new file mode 100644 index 00000000000..d1942f2c97f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-1.xml @@ -0,0 +1,29 @@ + + + + + + + + yarn.resource-types + memory-mb,resource1 + + + + yarn.resource-types.resource1.calculator-units + G + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml new file mode 100644 index 00000000000..ca428ebfb32 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml @@ -0,0 +1,29 @@ + + + + + + + + yarn.resource-types + vcores,resource1 + + + + yarn.resource-types.resource1.calculator-units + G + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml new file mode 100644 index 00000000000..08b8a6dc0e9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml @@ -0,0 +1,29 @@ + + + + + + + + yarn.resource-types + vcores,resource1 + + + + yarn.resource-types.resource1.calculator-units + A + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-4.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-4.xml new file mode 100644 index 00000000000..c8eb7662097 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-4.xml @@ -0,0 +1,24 @@ + + + + + + + + yarn.resource-types + memory,resource1 + + +