diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java deleted file mode 100644 index 6d82db77d34..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java +++ /dev/null @@ -1,104 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.hadoop.yarn.server.resourcemanager.resource; - -import org.apache.hadoop.yarn.api.records.Resource; - -public class DefaultResourceCalculator extends ResourceCalculator { - - Resource clusterResource; - - @Override - public int compare(Resource unused, Resource lhs, Resource rhs) { - // Only consider memory - return lhs.getMemory() - rhs.getMemory(); - } - - @Override - public int computeAvailableContainers(Resource available, Resource required) { - // Only consider memory - return available.getMemory() / required.getMemory(); - } - - @Override - public float divide(Resource unused, - Resource numerator, Resource denominator) { - return ratio(numerator, denominator); - } - - @Override - public float ratio(Resource a, Resource b) { - return (float)a.getMemory() / b.getMemory(); - } - - @Override - public Resource divideAndCeil(Resource numerator, int denominator) { - return Resources.createResource( - divideAndCeil(numerator.getMemory(), denominator)); - } - - @Override - public Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource, Resource stepFactor) { - int normalizedMemory = Math.min( - roundUp( - Math.max(r.getMemory(), minimumResource.getMemory()), - stepFactor.getMemory()), - maximumResource.getMemory()); - return Resources.createResource(normalizedMemory); - } - - @Override - public Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource) { - return normalize(r, minimumResource, maximumResource, minimumResource); - } - - @Override - public Resource roundUp(Resource r, Resource stepFactor) { - return Resources.createResource( - roundUp(r.getMemory(), stepFactor.getMemory()) - ); - } - - @Override - public Resource roundDown(Resource r, Resource stepFactor) { - return Resources.createResource( - roundDown(r.getMemory(), stepFactor.getMemory())); - } - - @Override - public Resource multiplyAndNormalizeUp(Resource r, double by, - Resource stepFactor) { - return Resources.createResource( - roundUp((int)(r.getMemory() * by + 0.5), stepFactor.getMemory()) - ); - } - - @Override - public Resource multiplyAndNormalizeDown(Resource r, double by, - Resource stepFactor) { - return Resources.createResource( - roundDown( - (int)(r.getMemory() * by), - stepFactor.getMemory() - ) - ); - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java deleted file mode 100644 index ef53e8dc14c..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java +++ /dev/null @@ -1,185 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.hadoop.yarn.server.resourcemanager.resource; - -import org.apache.hadoop.yarn.api.records.Resource; - -/** - * A {@link ResourceCalculator} which uses the concept of - * dominant resource to compare multi-dimensional resources. - * - * Essentially the idea is that the in a multi-resource environment, - * the resource allocation should be determined by the dominant share - * of an entity (user or queue), which is the maximum share that the - * entity has been allocated of any resource. - * - * In a nutshell, it seeks to maximize the minimum dominant share across - * all entities. - * - * For example, if user A runs CPU-heavy tasks and user B runs - * memory-heavy tasks, it attempts to equalize CPU share of user A - * with Memory-share of user B. - * - * In the single resource case, it reduces to max-min fairness for that resource. - * - * See the Dominant Resource Fairness paper for more details: - * www.cs.berkeley.edu/~matei/papers/2011/nsdi_drf.pdf - */ -public class DominantResourceCalculator extends ResourceCalculator { - - @Override - public int compare(Resource clusterResource, Resource lhs, Resource rhs) { - - if (lhs.equals(rhs)) { - return 0; - } - - float l = getResourceAsValue(clusterResource, lhs, true); - float r = getResourceAsValue(clusterResource, rhs, true); - - if (l < r) { - return -1; - } else if (l > r) { - return 1; - } else { - l = getResourceAsValue(clusterResource, lhs, false); - r = getResourceAsValue(clusterResource, rhs, false); - if (l < r) { - return -1; - } else if (l > r) { - return 1; - } - } - - return 0; - } - - /** - * Use 'dominant' for now since we only have 2 resources - gives us a slight - * performance boost. - * - * Once we add more resources, we'll need a more complicated (and slightly - * less performant algorithm). - */ - protected float getResourceAsValue( - Resource clusterResource, Resource resource, boolean dominant) { - // Just use 'dominant' resource - return (dominant) ? - Math.max( - (float)resource.getMemory() / clusterResource.getMemory(), - (float)resource.getVirtualCores() / clusterResource.getVirtualCores() - ) - : - Math.min( - (float)resource.getMemory() / clusterResource.getMemory(), - (float)resource.getVirtualCores() / clusterResource.getVirtualCores() - ); - } - - @Override - public int computeAvailableContainers(Resource available, Resource required) { - return Math.min( - available.getMemory() / required.getMemory(), - available.getVirtualCores() / required.getVirtualCores()); - } - - @Override - public float divide(Resource clusterResource, - Resource numerator, Resource denominator) { - return - getResourceAsValue(clusterResource, numerator, true) / - getResourceAsValue(clusterResource, denominator, true); - } - - @Override - public float ratio(Resource a, Resource b) { - return Math.max( - (float)a.getMemory()/b.getMemory(), - (float)a.getVirtualCores()/b.getVirtualCores() - ); - } - - @Override - public Resource divideAndCeil(Resource numerator, int denominator) { - return Resources.createResource( - divideAndCeil(numerator.getMemory(), denominator), - divideAndCeil(numerator.getVirtualCores(), denominator) - ); - } - - @Override - public Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource, Resource stepFactor) { - int normalizedMemory = Math.min( - roundUp( - Math.max(r.getMemory(), minimumResource.getMemory()), - stepFactor.getMemory()), - maximumResource.getMemory()); - int normalizedCores = Math.min( - roundUp( - Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()), - stepFactor.getVirtualCores()), - maximumResource.getVirtualCores()); - return Resources.createResource(normalizedMemory, - normalizedCores); - } - - @Override - public Resource roundUp(Resource r, Resource stepFactor) { - return Resources.createResource( - roundUp(r.getMemory(), stepFactor.getMemory()), - roundUp(r.getVirtualCores(), stepFactor.getVirtualCores()) - ); - } - - @Override - public Resource roundDown(Resource r, Resource stepFactor) { - return Resources.createResource( - roundDown(r.getMemory(), stepFactor.getMemory()), - roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()) - ); - } - - @Override - public Resource multiplyAndNormalizeUp(Resource r, double by, - Resource stepFactor) { - return Resources.createResource( - roundUp( - (int)Math.ceil(r.getMemory() * by), stepFactor.getMemory()), - roundUp( - (int)Math.ceil(r.getVirtualCores() * by), - stepFactor.getVirtualCores()) - ); - } - - @Override - public Resource multiplyAndNormalizeDown(Resource r, double by, - Resource stepFactor) { - return Resources.createResource( - roundDown( - (int)(r.getMemory() * by), - stepFactor.getMemory() - ), - roundDown( - (int)(r.getVirtualCores() * by), - stepFactor.getVirtualCores() - ) - ); - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java deleted file mode 100644 index de3777df5f6..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java +++ /dev/null @@ -1,170 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.hadoop.yarn.server.resourcemanager.resource; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.Resource; - -/** - * A set of {@link Resource} comparison and manipulation interfaces. - */ -@Private -@Unstable -public abstract class ResourceCalculator { - - private static final Log LOG = LogFactory.getLog(ResourceCalculator.class); - - public abstract int - compare(Resource clusterResource, Resource lhs, Resource rhs); - - public static int divideAndCeil(int a, int b) { - if (b == 0) { - LOG.info("divideAndCeil called with a=" + a + " b=" + b); - return 0; - } - return (a + (b - 1)) / b; - } - - public static int roundUp(int a, int b) { - return divideAndCeil(a, b) * b; - } - - public static int roundDown(int a, int b) { - return (a / b) * b; - } - - /** - * Compute the number of containers which can be allocated given - * available and required resources. - * - * @param available available resources - * @param required required resources - * @return number of containers which can be allocated - */ - public abstract int computeAvailableContainers( - Resource available, Resource required); - - /** - * Multiply resource r by factor by - * and normalize up using step-factor stepFactor. - * - * @param r resource to be multiplied - * @param by multiplier - * @param stepFactor factor by which to normalize up - * @return resulting normalized resource - */ - public abstract Resource multiplyAndNormalizeUp( - Resource r, double by, Resource stepFactor); - - /** - * Multiply resource r by factor by - * and normalize down using step-factor stepFactor. - * - * @param r resource to be multiplied - * @param by multiplier - * @param stepFactor factor by which to normalize down - * @return resulting normalized resource - */ - public abstract Resource multiplyAndNormalizeDown( - Resource r, double by, Resource stepFactor); - - /** - * Normalize resource r given the base - * minimumResource and verify against max allowed - * maximumResource - * - * @param r resource - * @param minimumResource step-factor - * @param maximumResource the upper bound of the resource to be allocated - * @return normalized resource - */ - public Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource) { - return normalize(r, minimumResource, maximumResource, minimumResource); - } - - /** - * Normalize resource r given the base - * minimumResource and verify against max allowed - * maximumResource using a step factor for hte normalization. - * - * @param r resource - * @param minimumResource minimum value - * @param maximumResource the upper bound of the resource to be allocated - * @param stepFactor the increment for resources to be allocated - * @return normalized resource - */ - public abstract Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource, - Resource stepFactor); - - - /** - * Round-up resource r given factor stepFactor. - * - * @param r resource - * @param stepFactor step-factor - * @return rounded resource - */ - public abstract Resource roundUp(Resource r, Resource stepFactor); - - /** - * Round-down resource r given factor stepFactor. - * - * @param r resource - * @param stepFactor step-factor - * @return rounded resource - */ - public abstract Resource roundDown(Resource r, Resource stepFactor); - - /** - * Divide resource numerator by resource denominator - * using specified policy (domination, average, fairness etc.); hence overall - * clusterResource is provided for context. - * - * @param clusterResource cluster resources - * @param numerator numerator - * @param denominator denominator - * @return numerator/denominator - * using specific policy - */ - public abstract float divide( - Resource clusterResource, Resource numerator, Resource denominator); - - /** - * Ratio of resource a to resource b. - * - * @param a resource - * @param b resource - * @return ratio of resource a to resource b - */ - public abstract float ratio(Resource a, Resource b); - - /** - * Divide-and-ceil numerator by denominator. - * - * @param numerator numerator resource - * @param denominator denominator - * @return resultant resource - */ - public abstract Resource divideAndCeil(Resource numerator, int denominator); - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java deleted file mode 100644 index 5e67098d4f9..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java +++ /dev/null @@ -1,221 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.server.resourcemanager.resource; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.util.Records; - -@Private -@Evolving -public class Resources { - - // Java doesn't have const :( - private static final Resource NONE = new Resource() { - - @Override - public int getMemory() { - return 0; - } - - @Override - public void setMemory(int memory) { - throw new RuntimeException("NONE cannot be modified!"); - } - - @Override - public int getVirtualCores() { - return 0; - } - - @Override - public void setVirtualCores(int cores) { - throw new RuntimeException("NONE cannot be modified!"); - } - - @Override - public int compareTo(Resource o) { - int diff = 0 - o.getMemory(); - if (diff == 0) { - diff = 0 - o.getVirtualCores(); - } - return diff; - } - - }; - - public static Resource createResource(int memory) { - return createResource(memory, (memory > 0) ? 1 : 0); - } - - public static Resource createResource(int memory, int cores) { - Resource resource = Records.newRecord(Resource.class); - resource.setMemory(memory); - resource.setVirtualCores(cores); - return resource; - } - - public static Resource none() { - return NONE; - } - - public static Resource clone(Resource res) { - return createResource(res.getMemory(), res.getVirtualCores()); - } - - public static Resource addTo(Resource lhs, Resource rhs) { - lhs.setMemory(lhs.getMemory() + rhs.getMemory()); - lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores()); - return lhs; - } - - public static Resource add(Resource lhs, Resource rhs) { - return addTo(clone(lhs), rhs); - } - - public static Resource subtractFrom(Resource lhs, Resource rhs) { - lhs.setMemory(lhs.getMemory() - rhs.getMemory()); - lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores()); - return lhs; - } - - public static Resource subtract(Resource lhs, Resource rhs) { - return subtractFrom(clone(lhs), rhs); - } - - public static Resource negate(Resource resource) { - return subtract(NONE, resource); - } - - public static Resource multiplyTo(Resource lhs, double by) { - lhs.setMemory((int)(lhs.getMemory() * by)); - lhs.setVirtualCores((int)(lhs.getVirtualCores() * by)); - return lhs; - } - - public static Resource multiply(Resource lhs, double by) { - return multiplyTo(clone(lhs), by); - } - - public static Resource multiplyAndNormalizeUp( - ResourceCalculator calculator,Resource lhs, double by, Resource factor) { - return calculator.multiplyAndNormalizeUp(lhs, by, factor); - } - - public static Resource multiplyAndNormalizeDown( - ResourceCalculator calculator,Resource lhs, double by, Resource factor) { - return calculator.multiplyAndNormalizeDown(lhs, by, factor); - } - - public static Resource multiplyAndRoundDown(Resource lhs, double by) { - Resource out = clone(lhs); - out.setMemory((int)(lhs.getMemory() * by)); - out.setVirtualCores((int)(lhs.getVirtualCores() * by)); - return out; - } - - public static Resource normalize( - ResourceCalculator calculator, Resource lhs, Resource min, - Resource max, Resource increment) { - return calculator.normalize(lhs, min, max, increment); - } - - public static Resource roundUp( - ResourceCalculator calculator, Resource lhs, Resource factor) { - return calculator.roundUp(lhs, factor); - } - - public static Resource roundDown( - ResourceCalculator calculator, Resource lhs, Resource factor) { - return calculator.roundDown(lhs, factor); - } - - public static float ratio( - ResourceCalculator resourceCalculator, Resource lhs, Resource rhs) { - return resourceCalculator.ratio(lhs, rhs); - } - - public static float divide( - ResourceCalculator resourceCalculator, - Resource clusterResource, Resource lhs, Resource rhs) { - return resourceCalculator.divide(clusterResource, lhs, rhs); - } - - public static Resource divideAndCeil( - ResourceCalculator resourceCalculator, Resource lhs, int rhs) { - return resourceCalculator.divideAndCeil(lhs, rhs); - } - - public static boolean equals(Resource lhs, Resource rhs) { - return lhs.equals(rhs); - } - - public static boolean lessThan( - ResourceCalculator resourceCalculator, - Resource clusterResource, - Resource lhs, Resource rhs) { - return (resourceCalculator.compare(clusterResource, lhs, rhs) < 0); - } - - public static boolean lessThanOrEqual( - ResourceCalculator resourceCalculator, - Resource clusterResource, - Resource lhs, Resource rhs) { - return (resourceCalculator.compare(clusterResource, lhs, rhs) <= 0); - } - - public static boolean greaterThan( - ResourceCalculator resourceCalculator, - Resource clusterResource, - Resource lhs, Resource rhs) { - return resourceCalculator.compare(clusterResource, lhs, rhs) > 0; - } - - public static boolean greaterThanOrEqual( - ResourceCalculator resourceCalculator, - Resource clusterResource, - Resource lhs, Resource rhs) { - return resourceCalculator.compare(clusterResource, lhs, rhs) >= 0; - } - - public static Resource min( - ResourceCalculator resourceCalculator, - Resource clusterResource, - Resource lhs, Resource rhs) { - return resourceCalculator.compare(clusterResource, lhs, rhs) <= 0 ? lhs : rhs; - } - - public static Resource max( - ResourceCalculator resourceCalculator, - Resource clusterResource, - Resource lhs, Resource rhs) { - return resourceCalculator.compare(clusterResource, lhs, rhs) >= 0 ? lhs : rhs; - } - - public static boolean fitsIn(Resource smaller, Resource bigger) { - return smaller.getMemory() <= bigger.getMemory() && - smaller.getVirtualCores() <= bigger.getVirtualCores(); - } - - public static Resource componentwiseMin(Resource lhs, Resource rhs) { - return createResource(Math.min(lhs.getMemory(), rhs.getMemory()), - Math.min(lhs.getVirtualCores(), rhs.getVirtualCores())); - } -}