From 015a55a5c63e1f28b86a211fd3d4de3138039303 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Tue, 26 Jul 2016 14:13:03 +0530 Subject: [PATCH] YARN-5242. Update DominantResourceCalculator to consider all resource types in calculations. Contributed by Varun Vasudev. (cherry picked from commit 9e4ba6aff595b7a5b53d016c97c8334e10f1f2cb) --- .../hadoop/yarn/api/records/Resource.java | 7 ++++++ .../api/records/impl/pb/ResourcePBImpl.java | 2 +- .../resource/DominantResourceCalculator.java | 23 +++++++++++++++---- .../yarn/util/resource/ResourceUtils.java | 5 ++-- .../hadoop/yarn/util/resource/Resources.java | 6 +---- 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 1de7b2fd71a..a25c8acd618 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -327,6 +327,8 @@ public abstract class Resource implements Comparable { otherResources = other.getResources(); long diff = thisResources.size() - otherResources.size(); if (diff == 0) { + // compare memory and vcores first(in that order) to preserve + // existing behaviour if (thisResources.keySet().equals(otherResources.keySet())) { diff = this.getMemorySize() - other.getMemorySize(); if (diff == 0) { @@ -335,6 +337,11 @@ public abstract class Resource implements Comparable { if (diff == 0) { for (Map.Entry entry : thisResources .entrySet()) { + if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) + || entry.getKey() + .equals(ResourceInformation.VCORES.getName())) { + continue; + } diff = entry.getValue().compareTo(otherResources.get(entry.getKey())); if (diff != 0) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index ab18f2db90b..5f326480506 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -242,7 +242,7 @@ public class ResourcePBImpl extends Resource { builder.addResourceValueMap(e); } } - builder.setMemory(this.getMemory()); + builder.setMemory(this.getMemorySize()); builder.setVirtualCores(this.getVirtualCores()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 0412c0fac05..3c4413c4e75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -397,10 +397,25 @@ public class DominantResourceCalculator extends ResourceCalculator { } @Override - public boolean fitsIn(Resource cluster, - Resource smaller, Resource bigger) { - return smaller.getMemorySize() <= bigger.getMemorySize() - && smaller.getVirtualCores() <= bigger.getVirtualCores(); + public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) { + for (String resource : resourceNames) { + try { + ResourceInformation sResourceInformation = + smaller.getResourceInformation(resource); + ResourceInformation bResourceInformation = + bigger.getResourceInformation(resource); + Long sResourceValue = UnitsConversionUtil + .convert(sResourceInformation.getUnits(), + bResourceInformation.getUnits(), + sResourceInformation.getValue()); + if(sResourceValue > bResourceInformation.getValue()) { + return false; + } + } catch (YarnException ye) { + return false; + } + } + return true; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 4f56fbb52b3..109287acb0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -178,7 +178,6 @@ public class ResourceUtils { synchronized (ResourceUtils.class) { if (lock == null) { synchronized (ResourceUtils.class) { - lock = new Object(); Map resources = new HashMap<>(); if (conf == null) { conf = new YarnConfiguration(); @@ -187,10 +186,12 @@ public class ResourceUtils { addResourcesFileToConf(resourceFile, conf); LOG.debug("Found " + resourceFile + ", adding to configuration"); initializeResourcesMap(conf, resources); + lock = new Object(); } catch (FileNotFoundException fe) { LOG.info("Unable to find '" + resourceFile + "'. Falling back to memory and vcores as resources", fe); initializeResourcesMap(conf, resources); + lock = new Object(); } } } @@ -268,12 +269,12 @@ public class ResourceUtils { synchronized (ResourceUtils.class) { if (nodeLock == null) { synchronized (ResourceUtils.class) { - nodeLock = new Object(); Map nodeResources = initializeNodeResourceInformation(conf); addManadtoryResources(nodeResources); checkMandatatoryResources(nodeResources); readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); + nodeLock = new Object(); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 836f87e6c86..73776e7dcb2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -134,16 +134,12 @@ public class Resources { private Map initResourceMap() { Map tmp = new HashMap<>(); - // Due to backwards compat, the max value for memory and vcores - // needs to be Integer.MAX_VALUE - int max = resourceValue > Integer.MAX_VALUE ? Integer.MAX_VALUE : - resourceValue.intValue(); Map types = ResourceUtils.getResourceTypes(); if (types != null) { for (Map.Entry entry : types.entrySet()) { tmp.put(entry.getKey(), ResourceInformation.newInstance(entry.getValue())); - tmp.get(entry.getKey()).setValue((long) max); + tmp.get(entry.getKey()).setValue(resourceValue); } } return tmp;