YARN-6788. [YARN-3926] Improve performance of resource profile branch
(Contributed by Sunil Govindan via Daniel Templeton)
(cherry picked from commit 3aeaafecb8)
This commit is contained in:
Daniel Templeton
Jonathan Hung
parent
1dc66dbb3f
commit
d8f388652e
|
|
@ -615,4 +615,22 @@
|
|||
<Bug pattern="IS2_INCONSISTENT_SYNC" />
|
||||
</Match>
|
||||
|
||||
<!-- Ignore MS_EXPOSE_REP -->
|
||||
<Match>
|
||||
<Class name="org.apache.hadoop.yarn.util.resource.ResourceUtils" />
|
||||
<Method name="getResourceTypesArray" />
|
||||
<Bug pattern="MS_EXPOSE_REP" />
|
||||
</Match>
|
||||
|
||||
<Match>
|
||||
<Class name="org.apache.hadoop.yarn.util.resource.ResourceUtils" />
|
||||
<Method name="getResourceNamesArray" />
|
||||
<Bug pattern="MS_EXPOSE_REP" />
|
||||
</Match>
|
||||
|
||||
<Match>
|
||||
<Class name="org.apache.hadoop.yarn.api.records.impl.BaseResource" />
|
||||
<Method name="getResources" />
|
||||
<Bug pattern="EI_EXPOSE_REP" />
|
||||
</Match>
|
||||
</FindBugsFilter>
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@
|
|||
|
||||
package org.apache.hadoop.yarn.api.records;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||
|
|
@ -25,13 +27,10 @@ import org.apache.hadoop.classification.InterfaceStability;
|
|||
import org.apache.hadoop.classification.InterfaceStability.Evolving;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
||||
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
|
||||
import org.apache.hadoop.yarn.api.records.impl.BaseResource;
|
||||
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.util.Records;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||
|
||||
/**
|
||||
* <p><code>Resource</code> models a set of computer resources in the
|
||||
|
|
@ -60,97 +59,49 @@ import java.util.Map;
|
|||
@Stable
|
||||
public abstract class Resource implements Comparable<Resource> {
|
||||
|
||||
private static Resource tmpResource = Records.newRecord(Resource.class);
|
||||
|
||||
private static class SimpleResource extends Resource {
|
||||
private long memory;
|
||||
private long vcores;
|
||||
private Map<String, ResourceInformation> resourceInformationMap;
|
||||
|
||||
SimpleResource(long memory, long vcores) {
|
||||
this.memory = memory;
|
||||
this.vcores = vcores;
|
||||
|
||||
}
|
||||
@Override
|
||||
public int getMemory() {
|
||||
return castToIntSafely(memory);
|
||||
}
|
||||
@Override
|
||||
public void setMemory(int memory) {
|
||||
this.memory = memory;
|
||||
}
|
||||
@Override
|
||||
public long getMemorySize() {
|
||||
return memory;
|
||||
}
|
||||
@Override
|
||||
public void setMemorySize(long memory) {
|
||||
this.memory = memory;
|
||||
}
|
||||
@Override
|
||||
public int getVirtualCores() {
|
||||
return castToIntSafely(vcores);
|
||||
}
|
||||
@Override
|
||||
public void setVirtualCores(int vcores) {
|
||||
this.vcores = vcores;
|
||||
}
|
||||
@Override
|
||||
public Map<String, ResourceInformation> getResources() {
|
||||
if (resourceInformationMap == null) {
|
||||
resourceInformationMap = new HashMap<>();
|
||||
resourceInformationMap.put(ResourceInformation.MEMORY_MB.getName(),
|
||||
ResourceInformation.newInstance(ResourceInformation.MEMORY_MB));
|
||||
resourceInformationMap.put(ResourceInformation.VCORES.getName(),
|
||||
ResourceInformation.newInstance(ResourceInformation.VCORES));
|
||||
}
|
||||
resourceInformationMap.get(ResourceInformation.MEMORY_MB.getName())
|
||||
.setValue(this.memory);
|
||||
resourceInformationMap.get(ResourceInformation.VCORES.getName())
|
||||
.setValue(this.vcores);
|
||||
return Collections.unmodifiableMap(resourceInformationMap);
|
||||
}
|
||||
}
|
||||
protected static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
|
||||
protected static final String VCORES = ResourceInformation.VCORES.getName();
|
||||
|
||||
@Public
|
||||
@Stable
|
||||
public static Resource newInstance(int memory, int vCores) {
|
||||
if (tmpResource.getResources().size() > 2) {
|
||||
if (ResourceUtils.getResourceTypesArray().length > 2) {
|
||||
Resource ret = Records.newRecord(Resource.class);
|
||||
ret.setMemorySize(memory);
|
||||
ret.setVirtualCores(vCores);
|
||||
return ret;
|
||||
}
|
||||
return new SimpleResource(memory, vCores);
|
||||
return new BaseResource(memory, vCores);
|
||||
}
|
||||
|
||||
@Public
|
||||
@Stable
|
||||
public static Resource newInstance(long memory, int vCores) {
|
||||
if (tmpResource.getResources().size() > 2) {
|
||||
if (ResourceUtils.getResourceTypesArray().length > 2) {
|
||||
Resource ret = Records.newRecord(Resource.class);
|
||||
ret.setMemorySize(memory);
|
||||
ret.setVirtualCores(vCores);
|
||||
return ret;
|
||||
}
|
||||
return new SimpleResource(memory, vCores);
|
||||
return new BaseResource(memory, vCores);
|
||||
}
|
||||
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Unstable
|
||||
public static Resource newInstance(Resource resource) {
|
||||
Resource ret = Resource.newInstance(0, 0);
|
||||
Resource ret = Resource.newInstance(resource.getMemorySize(),
|
||||
resource.getVirtualCores());
|
||||
if (ResourceUtils.getResourceTypesArray().length > 2) {
|
||||
Resource.copy(resource, ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Unstable
|
||||
public static void copy(Resource source, Resource dest) {
|
||||
for (Map.Entry<String, ResourceInformation> entry : source.getResources()
|
||||
.entrySet()) {
|
||||
dest.setResourceInformation(entry.getKey(), entry.getValue());
|
||||
for (ResourceInformation entry : source.getResources()) {
|
||||
dest.setResourceInformation(entry.getName(), entry);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -251,25 +202,26 @@ public abstract class Resource implements Comparable<Resource> {
|
|||
*/
|
||||
@Public
|
||||
@Evolving
|
||||
public abstract Map<String, ResourceInformation> getResources();
|
||||
public abstract ResourceInformation[] getResources();
|
||||
|
||||
/**
|
||||
* Get ResourceInformation for a specified resource.
|
||||
*
|
||||
* @param resource name of the resource
|
||||
* @return the ResourceInformation object for the resource
|
||||
* @throws YarnException if the resource can't be found
|
||||
* @throws ResourceNotFoundException if the resource can't be found
|
||||
*/
|
||||
@Public
|
||||
@Evolving
|
||||
public ResourceInformation getResourceInformation(String resource)
|
||||
throws YarnException {
|
||||
if (getResources().containsKey(resource)) {
|
||||
return getResources().get(resource);
|
||||
throws ResourceNotFoundException {
|
||||
Integer index = ResourceUtils.getResourceTypeIndex().get(resource);
|
||||
ResourceInformation[] resources = getResources();
|
||||
if (index != null) {
|
||||
return resources[index];
|
||||
}
|
||||
throw new YarnException(
|
||||
"Unknown resource '" + resource + "'. Known resources are "
|
||||
+ getResources().keySet());
|
||||
throw new ResourceNotFoundException("Unknown resource '" + resource
|
||||
+ "'. Known resources are " + Arrays.toString(resources));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -278,17 +230,13 @@ public abstract class Resource implements Comparable<Resource> {
|
|||
*
|
||||
* @param resource name of the resource
|
||||
* @return the value for the resource
|
||||
* @throws YarnException if the resource can't be found
|
||||
* @throws ResourceNotFoundException if the resource can't be found
|
||||
*/
|
||||
@Public
|
||||
@Evolving
|
||||
public Long getResourceValue(String resource) throws YarnException {
|
||||
if (getResources().containsKey(resource)) {
|
||||
return getResources().get(resource).getValue();
|
||||
}
|
||||
throw new YarnException(
|
||||
"Unknown resource '" + resource + "'. Known resources are "
|
||||
+ getResources().keySet());
|
||||
public long getResourceValue(String resource)
|
||||
throws ResourceNotFoundException {
|
||||
return getResourceInformation(resource).getValue();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -301,23 +249,18 @@ public abstract class Resource implements Comparable<Resource> {
|
|||
@Public
|
||||
@Evolving
|
||||
public void setResourceInformation(String resource,
|
||||
ResourceInformation resourceInformation) throws ResourceNotFoundException {
|
||||
if (resource.equals(ResourceInformation.MEMORY_MB.getName())) {
|
||||
ResourceInformation resourceInformation)
|
||||
throws ResourceNotFoundException {
|
||||
if (resource.equals(MEMORY)) {
|
||||
this.setMemorySize(resourceInformation.getValue());
|
||||
return;
|
||||
}
|
||||
if (resource.equals(ResourceInformation.VCORES.getName())) {
|
||||
if (resource.equals(VCORES)) {
|
||||
this.setVirtualCores((int) resourceInformation.getValue());
|
||||
return;
|
||||
}
|
||||
if (getResources().containsKey(resource)) {
|
||||
ResourceInformation
|
||||
.copy(resourceInformation, getResources().get(resource));
|
||||
return;
|
||||
}
|
||||
throw new ResourceNotFoundException(
|
||||
"Unknown resource '" + resource + "'. Known resources are "
|
||||
+ getResources().keySet());
|
||||
ResourceInformation storedResourceInfo = getResourceInformation(resource);
|
||||
ResourceInformation.copy(resourceInformation, storedResourceInfo);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -332,21 +275,17 @@ public abstract class Resource implements Comparable<Resource> {
|
|||
@Evolving
|
||||
public void setResourceValue(String resource, Long value)
|
||||
throws ResourceNotFoundException {
|
||||
if (resource.equals(ResourceInformation.MEMORY_MB.getName())) {
|
||||
if (resource.equals(MEMORY)) {
|
||||
this.setMemorySize(value);
|
||||
return;
|
||||
}
|
||||
if (resource.equals(ResourceInformation.VCORES.getName())) {
|
||||
if (resource.equals(VCORES)) {
|
||||
this.setVirtualCores(value.intValue());
|
||||
return;
|
||||
}
|
||||
if (getResources().containsKey(resource)) {
|
||||
getResources().get(resource).setValue(value);
|
||||
return;
|
||||
}
|
||||
throw new ResourceNotFoundException(
|
||||
"Unknown resource '" + resource + "'. Known resources are "
|
||||
+ getResources().keySet());
|
||||
|
||||
ResourceInformation storedResourceInfo = getResourceInformation(resource);
|
||||
storedResourceInfo.setValue(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -356,13 +295,10 @@ public abstract class Resource implements Comparable<Resource> {
|
|||
int result = (int) (939769357
|
||||
+ getMemorySize()); // prime * result = 939769357 initially
|
||||
result = prime * result + getVirtualCores();
|
||||
for (Map.Entry<String, ResourceInformation> entry : getResources()
|
||||
.entrySet()) {
|
||||
if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
|
||||
|| entry.getKey().equals(ResourceInformation.VCORES.getName())) {
|
||||
continue;
|
||||
for (ResourceInformation entry : getResources()) {
|
||||
if (!entry.getName().equals(MEMORY) && !entry.getName().equals(VCORES)) {
|
||||
result = prime * result + entry.hashCode();
|
||||
}
|
||||
result = prime * result + entry.getValue().hashCode();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
|
@ -379,11 +315,26 @@ public abstract class Resource implements Comparable<Resource> {
|
|||
return false;
|
||||
}
|
||||
Resource other = (Resource) obj;
|
||||
if (getMemorySize() != other.getMemorySize() || getVirtualCores() != other
|
||||
.getVirtualCores()) {
|
||||
if (getMemorySize() != other.getMemorySize()
|
||||
|| getVirtualCores() != other.getVirtualCores()) {
|
||||
return false;
|
||||
}
|
||||
return this.getResources().equals(other.getResources());
|
||||
|
||||
ResourceInformation[] myVectors = getResources();
|
||||
ResourceInformation[] otherVectors = other.getResources();
|
||||
|
||||
if (myVectors.length != otherVectors.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (int i = 0; i < myVectors.length; i++) {
|
||||
ResourceInformation a = myVectors[i];
|
||||
ResourceInformation b = otherVectors[i];
|
||||
if ((a != b) && ((a == null) || !a.equals(b))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -391,21 +342,20 @@ public abstract class Resource implements Comparable<Resource> {
|
|||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("<memory:").append(getMemorySize()).append(", vCores:")
|
||||
.append(getVirtualCores());
|
||||
for (Map.Entry<String, ResourceInformation> entry : getResources()
|
||||
.entrySet()) {
|
||||
if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
|
||||
&& entry.getValue().getUnits()
|
||||
for (ResourceInformation entry : getResources()) {
|
||||
if (entry.getName().equals(MEMORY)
|
||||
&& entry.getUnits()
|
||||
.equals(ResourceInformation.MEMORY_MB.getUnits())) {
|
||||
continue;
|
||||
}
|
||||
if (entry.getKey().equals(ResourceInformation.VCORES.getName())
|
||||
&& entry.getValue().getUnits()
|
||||
if (entry.getName().equals(VCORES)
|
||||
&& entry.getUnits()
|
||||
.equals(ResourceInformation.VCORES.getUnits())) {
|
||||
continue;
|
||||
}
|
||||
sb.append(", ").append(entry.getKey()).append(": ")
|
||||
.append(entry.getValue().getValue())
|
||||
.append(entry.getValue().getUnits());
|
||||
sb.append(", ").append(entry.getName()).append(": ")
|
||||
.append(entry.getValue())
|
||||
.append(entry.getUnits());
|
||||
}
|
||||
sb.append(">");
|
||||
return sb.toString();
|
||||
|
|
@ -413,28 +363,30 @@ public abstract class Resource implements Comparable<Resource> {
|
|||
|
||||
@Override
|
||||
public int compareTo(Resource other) {
|
||||
Map<String, ResourceInformation> thisResources, otherResources;
|
||||
thisResources = this.getResources();
|
||||
otherResources = other.getResources();
|
||||
long diff = thisResources.size() - otherResources.size();
|
||||
if (diff == 0) {
|
||||
ResourceInformation[] thisResources = this.getResources();
|
||||
ResourceInformation[] otherResources = other.getResources();
|
||||
|
||||
// compare memory and vcores first(in that order) to preserve
|
||||
// existing behaviour
|
||||
if (thisResources.keySet().equals(otherResources.keySet())) {
|
||||
diff = this.getMemorySize() - other.getMemorySize();
|
||||
long diff = this.getMemorySize() - other.getMemorySize();
|
||||
if (diff == 0) {
|
||||
diff = this.getVirtualCores() - other.getVirtualCores();
|
||||
}
|
||||
if (diff == 0) {
|
||||
for (Map.Entry<String, ResourceInformation> entry : thisResources
|
||||
.entrySet()) {
|
||||
if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
|
||||
|| entry.getKey()
|
||||
.equals(ResourceInformation.VCORES.getName())) {
|
||||
diff = thisResources.length - otherResources.length;
|
||||
if (diff == 0) {
|
||||
int maxLength = ResourceUtils.getResourceTypesArray().length;
|
||||
for (int i = 0; i < maxLength; i++) {
|
||||
// For memory and vcores, we can skip the loop as it's already
|
||||
// compared.
|
||||
if (i < 2) {
|
||||
continue;
|
||||
}
|
||||
diff =
|
||||
entry.getValue().compareTo(otherResources.get(entry.getKey()));
|
||||
|
||||
ResourceInformation entry = thisResources[i];
|
||||
ResourceInformation otherEntry = otherResources[i];
|
||||
if (entry.getName().equals(otherEntry.getName())) {
|
||||
diff = entry.compareTo(otherEntry);
|
||||
if (diff != 0) {
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -242,10 +242,15 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
return false;
|
||||
}
|
||||
ResourceInformation r = (ResourceInformation) obj;
|
||||
int cmp =
|
||||
UnitsConversionUtil.compare(this.units, this.value, r.units, r.value);
|
||||
return this.name.equals(r.getName()) && this.resourceType
|
||||
.equals(r.getResourceType()) && (cmp == 0);
|
||||
if (!this.name.equals(r.getName())
|
||||
|| !this.resourceType.equals(r.getResourceType())) {
|
||||
return false;
|
||||
}
|
||||
if (this.units.equals(r.units)) {
|
||||
return this.value == r.value;
|
||||
}
|
||||
return (UnitsConversionUtil.compare(this.units, this.value, r.units,
|
||||
r.value) == 0);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -0,0 +1,133 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.api.records.impl;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* <code>BaseResource</code> extends Resource to handle base resources such
|
||||
* as memory and CPU.
|
||||
* TODO: We have a long term plan to use AbstractResource when additional
|
||||
* resource types are to be handled as well.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Currently it models both <em>memory</em> and <em>CPU</em>.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The unit for memory is megabytes. CPU is modeled with virtual cores (vcores),
|
||||
* a unit for expressing parallelism. A node's capacity should be configured
|
||||
* with virtual cores equal to its number of physical cores. A container should
|
||||
* be requested with the number of cores it can saturate, i.e. the average
|
||||
* number of threads it expects to have runnable at a time.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Virtual cores take integer values and thus currently CPU-scheduling is very
|
||||
* coarse. A complementary axis for CPU requests that represents processing
|
||||
* power will likely be added in the future to enable finer-grained resource
|
||||
* configuration.
|
||||
* </p>
|
||||
*
|
||||
* @see Resource
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
public class BaseResource extends Resource {
|
||||
|
||||
private ResourceInformation memoryResInfo;
|
||||
private ResourceInformation vcoresResInfo;
|
||||
protected ResourceInformation[] resources = null;
|
||||
protected ResourceInformation[] readOnlyResources = null;
|
||||
|
||||
protected enum MandatoryResources {
|
||||
MEMORY(0), VCORES(1);
|
||||
|
||||
private final int id;
|
||||
|
||||
MandatoryResources(int id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public int getId() {
|
||||
return this.id;
|
||||
}
|
||||
}
|
||||
|
||||
public BaseResource() {
|
||||
// Base constructor.
|
||||
}
|
||||
|
||||
public BaseResource(long memory, long vcores) {
|
||||
this.memoryResInfo = ResourceInformation.newInstance(MEMORY,
|
||||
ResourceInformation.MEMORY_MB.getUnits(), memory);
|
||||
this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores);
|
||||
|
||||
resources = new ResourceInformation[MandatoryResources.values().length];
|
||||
readOnlyResources = new ResourceInformation[MandatoryResources
|
||||
.values().length];
|
||||
resources[MandatoryResources.MEMORY.id] = memoryResInfo;
|
||||
resources[MandatoryResources.VCORES.id] = vcoresResInfo;
|
||||
readOnlyResources = Arrays.copyOf(resources, resources.length);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("deprecation")
|
||||
public int getMemory() {
|
||||
return (int) memoryResInfo.getValue();
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("deprecation")
|
||||
public void setMemory(int memory) {
|
||||
this.memoryResInfo.setValue(memory);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySize() {
|
||||
return memoryResInfo.getValue();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setMemorySize(long memory) {
|
||||
this.memoryResInfo.setValue(memory);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getVirtualCores() {
|
||||
return (int) vcoresResInfo.getValue();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setVirtualCores(int vcores) {
|
||||
this.vcoresResInfo.setValue(vcores);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResourceInformation[] getResources() {
|
||||
return readOnlyResources;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
@InterfaceAudience.Public
|
||||
@InterfaceStability.Unstable
|
||||
package org.apache.hadoop.yarn.api.records.impl;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
|
|
@ -186,11 +186,11 @@ public class UnitsConversionUtil {
|
|||
if (!KNOWN_UNITS.contains(unitB)) {
|
||||
throw new IllegalArgumentException("Unknown unit '" + unitB + "'");
|
||||
}
|
||||
if (unitA.equals(unitB)) {
|
||||
return Long.compare(valueA, valueB);
|
||||
}
|
||||
Converter unitAC = getConverter(unitA);
|
||||
Converter unitBC = getConverter(unitB);
|
||||
if (unitA.equals(unitB)) {
|
||||
return Long.valueOf(valueA).compareTo(valueB);
|
||||
}
|
||||
int unitAPos = SORTED_UNITS.indexOf(unitA);
|
||||
int unitBPos = SORTED_UNITS.indexOf(unitB);
|
||||
try {
|
||||
|
|
@ -201,7 +201,7 @@ public class UnitsConversionUtil {
|
|||
} else {
|
||||
tmpA = convert(unitA, unitB, valueA);
|
||||
}
|
||||
return Long.valueOf(tmpA).compareTo(tmpB);
|
||||
return Long.compare(tmpA, tmpB);
|
||||
} catch (IllegalArgumentException ie) {
|
||||
BigInteger tmpA = BigInteger.valueOf(valueA);
|
||||
BigInteger tmpB = BigInteger.valueOf(valueB);
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ import java.util.HashMap;
|
|||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* Helper class to read the resource-types to be supported by the system.
|
||||
|
|
@ -65,11 +66,14 @@ public class ResourceUtils {
|
|||
DISALLOWED_NAMES.add(VCORES);
|
||||
}
|
||||
|
||||
private static volatile Object lock;
|
||||
private static Map<String, ResourceInformation> readOnlyResources;
|
||||
private static volatile Object nodeLock;
|
||||
private static Map<String, ResourceInformation> readOnlyNodeResources;
|
||||
|
||||
private static volatile boolean initializedResources = false;
|
||||
private static final Map<String, Integer> RESOURCE_NAME_TO_INDEX =
|
||||
new ConcurrentHashMap<String, Integer>();
|
||||
private static volatile Map<String, ResourceInformation> resourceTypes;
|
||||
private static volatile String[] resourceNamesArray;
|
||||
private static volatile ResourceInformation[] resourceTypesArray;
|
||||
private static volatile boolean initializedNodeResources = false;
|
||||
private static volatile Map<String, ResourceInformation> readOnlyNodeResources;
|
||||
|
||||
static final Log LOG = LogFactory.getLog(ResourceUtils.class);
|
||||
|
||||
|
|
@ -127,21 +131,17 @@ public class ResourceUtils {
|
|||
|
||||
private static void setMinimumAllocationForMandatoryResources(
|
||||
Map<String, ResourceInformation> res, Configuration conf) {
|
||||
String[][] resourceTypesKeys =
|
||||
{
|
||||
String[][] resourceTypesKeys = {
|
||||
{ResourceInformation.MEMORY_MB.getName(),
|
||||
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
|
||||
String.valueOf(
|
||||
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
|
||||
ResourceInformation.MEMORY_MB.getName()
|
||||
},
|
||||
ResourceInformation.MEMORY_MB.getName()},
|
||||
{ResourceInformation.VCORES.getName(),
|
||||
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
|
||||
String.valueOf(
|
||||
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES),
|
||||
ResourceInformation.VCORES.getName()
|
||||
}
|
||||
};
|
||||
ResourceInformation.VCORES.getName()}};
|
||||
for (String[] arr : resourceTypesKeys) {
|
||||
String resourceTypesKey =
|
||||
YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION;
|
||||
|
|
@ -166,23 +166,17 @@ public class ResourceUtils {
|
|||
|
||||
private static void setMaximumAllocationForMandatoryResources(
|
||||
Map<String, ResourceInformation> res, Configuration conf) {
|
||||
String[][] resourceTypesKeys =
|
||||
{
|
||||
{
|
||||
ResourceInformation.MEMORY_MB.getName(),
|
||||
String[][] resourceTypesKeys = {
|
||||
{ResourceInformation.MEMORY_MB.getName(),
|
||||
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
|
||||
String.valueOf(
|
||||
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
|
||||
ResourceInformation.MEMORY_MB.getName()
|
||||
},
|
||||
{
|
||||
ResourceInformation.VCORES.getName(),
|
||||
ResourceInformation.MEMORY_MB.getName()},
|
||||
{ResourceInformation.VCORES.getName(),
|
||||
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
|
||||
String.valueOf(
|
||||
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
|
||||
ResourceInformation.VCORES.getName()
|
||||
}
|
||||
};
|
||||
ResourceInformation.VCORES.getName()}};
|
||||
for (String[] arr : resourceTypesKeys) {
|
||||
String resourceTypesKey =
|
||||
YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION;
|
||||
|
|
@ -251,7 +245,50 @@ public class ResourceUtils {
|
|||
addManadtoryResources(resourceInformationMap);
|
||||
setMinimumAllocationForMandatoryResources(resourceInformationMap, conf);
|
||||
setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
|
||||
readOnlyResources = Collections.unmodifiableMap(resourceInformationMap);
|
||||
resourceTypes = Collections.unmodifiableMap(resourceInformationMap);
|
||||
updateKnownResources();
|
||||
updateResourceTypeIndex();
|
||||
}
|
||||
|
||||
private static void updateKnownResources() {
|
||||
// Update resource names.
|
||||
resourceNamesArray = new String[resourceTypes.size()];
|
||||
resourceTypesArray = new ResourceInformation[resourceTypes.size()];
|
||||
|
||||
int index = 2;
|
||||
for (ResourceInformation resInfo : resourceTypes.values()) {
|
||||
if (resInfo.getName().equals(MEMORY)) {
|
||||
resourceTypesArray[0] = ResourceInformation
|
||||
.newInstance(resourceTypes.get(MEMORY));
|
||||
resourceNamesArray[0] = MEMORY;
|
||||
} else if (resInfo.getName().equals(VCORES)) {
|
||||
resourceTypesArray[1] = ResourceInformation
|
||||
.newInstance(resourceTypes.get(VCORES));
|
||||
resourceNamesArray[1] = VCORES;
|
||||
} else {
|
||||
resourceTypesArray[index] = ResourceInformation.newInstance(resInfo);
|
||||
resourceNamesArray[index] = resInfo.getName();
|
||||
index++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void updateResourceTypeIndex() {
|
||||
RESOURCE_NAME_TO_INDEX.clear();
|
||||
|
||||
for (int index = 0; index < resourceTypesArray.length; index++) {
|
||||
ResourceInformation resInfo = resourceTypesArray[index];
|
||||
RESOURCE_NAME_TO_INDEX.put(resInfo.getName(), index);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get associate index of resource types such memory, cpu etc.
|
||||
* This could help to access each resource types in a resource faster.
|
||||
* @return Index map for all Resource Types.
|
||||
*/
|
||||
public static Map<String, Integer> getResourceTypeIndex() {
|
||||
return RESOURCE_NAME_TO_INDEX;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -264,6 +301,22 @@ public class ResourceUtils {
|
|||
YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get resource names array, this is mostly for performance perspective. Never
|
||||
* modify returned array.
|
||||
*
|
||||
* @return resourceNamesArray
|
||||
*/
|
||||
public static String[] getResourceNamesArray() {
|
||||
getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
|
||||
return resourceNamesArray;
|
||||
}
|
||||
|
||||
public static ResourceInformation[] getResourceTypesArray() {
|
||||
getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
|
||||
return resourceTypesArray;
|
||||
}
|
||||
|
||||
private static Map<String, ResourceInformation> getResourceTypes(
|
||||
Configuration conf) {
|
||||
return getResourceTypes(conf,
|
||||
|
|
@ -272,10 +325,9 @@ public class ResourceUtils {
|
|||
|
||||
private static Map<String, ResourceInformation> getResourceTypes(
|
||||
Configuration conf, String resourceFile) {
|
||||
if (lock == null) {
|
||||
synchronized (ResourceUtils.class) {
|
||||
if (lock == null) {
|
||||
if (!initializedResources) {
|
||||
synchronized (ResourceUtils.class) {
|
||||
if (!initializedResources) {
|
||||
Map<String, ResourceInformation> resources = new HashMap<>();
|
||||
if (conf == null) {
|
||||
conf = new YarnConfiguration();
|
||||
|
|
@ -284,18 +336,17 @@ public class ResourceUtils {
|
|||
addResourcesFileToConf(resourceFile, conf);
|
||||
LOG.debug("Found " + resourceFile + ", adding to configuration");
|
||||
initializeResourcesMap(conf, resources);
|
||||
lock = new Object();
|
||||
initializedResources = true;
|
||||
} catch (FileNotFoundException fe) {
|
||||
LOG.info("Unable to find '" + resourceFile
|
||||
+ "'. Falling back to memory and vcores as resources", fe);
|
||||
initializeResourcesMap(conf, resources);
|
||||
lock = new Object();
|
||||
initializedResources = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return readOnlyResources;
|
||||
return resourceTypes;
|
||||
}
|
||||
|
||||
private static InputStream getConfInputStream(String resourceFile,
|
||||
|
|
@ -341,13 +392,15 @@ public class ResourceUtils {
|
|||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static void resetResourceTypes() {
|
||||
lock = null;
|
||||
synchronized static void resetResourceTypes() {
|
||||
initializedResources = false;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static void resetResourceTypes(Configuration conf) {
|
||||
lock = null;
|
||||
synchronized (ResourceUtils.class) {
|
||||
initializedResources = false;
|
||||
}
|
||||
getResourceTypes(conf);
|
||||
}
|
||||
|
||||
|
|
@ -375,17 +428,17 @@ public class ResourceUtils {
|
|||
*/
|
||||
public static Map<String, ResourceInformation> getNodeResourceInformation(
|
||||
Configuration conf) {
|
||||
if (nodeLock == null) {
|
||||
if (!initializedNodeResources) {
|
||||
synchronized (ResourceUtils.class) {
|
||||
if (nodeLock == null) {
|
||||
synchronized (ResourceUtils.class) {
|
||||
Map<String, ResourceInformation> nodeResources =
|
||||
initializeNodeResourceInformation(conf);
|
||||
if (!initializedNodeResources) {
|
||||
Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
|
||||
conf);
|
||||
addManadtoryResources(nodeResources);
|
||||
checkMandatatoryResources(nodeResources);
|
||||
setMinimumAllocationForMandatoryResources(nodeResources, conf);
|
||||
setMaximumAllocationForMandatoryResources(nodeResources, conf);
|
||||
readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
|
||||
nodeLock = new Object();
|
||||
}
|
||||
initializedNodeResources = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -433,28 +486,24 @@ public class ResourceUtils {
|
|||
|
||||
@VisibleForTesting
|
||||
synchronized public static void resetNodeResources() {
|
||||
nodeLock = null;
|
||||
initializedNodeResources = false;
|
||||
}
|
||||
|
||||
public static Resource getResourceTypesMinimumAllocation() {
|
||||
Map<String, ResourceInformation> resourceTypes = getResourceTypes();
|
||||
Resource ret = Resource.newInstance(0, 0);
|
||||
for (Map.Entry<String, ResourceInformation> entry : resourceTypes
|
||||
.entrySet()) {
|
||||
String name = entry.getKey();
|
||||
for (ResourceInformation entry : resourceTypesArray) {
|
||||
String name = entry.getName();
|
||||
if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
|
||||
ret.setMemorySize(entry.getValue().getMinimumAllocation());
|
||||
continue;
|
||||
}
|
||||
if (name.equals(ResourceInformation.VCORES.getName())) {
|
||||
Long tmp = entry.getValue().getMinimumAllocation();
|
||||
ret.setMemorySize(entry.getMinimumAllocation());
|
||||
} else if (name.equals(ResourceInformation.VCORES.getName())) {
|
||||
Long tmp = entry.getMinimumAllocation();
|
||||
if (tmp > Integer.MAX_VALUE) {
|
||||
tmp = (long) Integer.MAX_VALUE;
|
||||
}
|
||||
ret.setVirtualCores(tmp.intValue());
|
||||
continue;
|
||||
} else {
|
||||
ret.setResourceValue(name, entry.getMinimumAllocation());
|
||||
}
|
||||
ret.setResourceValue(name, entry.getValue().getMinimumAllocation());
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -464,24 +513,21 @@ public class ResourceUtils {
|
|||
* @return a Resource object with the maximum allocation for the scheduler
|
||||
*/
|
||||
public static Resource getResourceTypesMaximumAllocation() {
|
||||
Map<String, ResourceInformation> resourceTypes = getResourceTypes();
|
||||
Resource ret = Resource.newInstance(0, 0);
|
||||
for (Map.Entry<String, ResourceInformation> entry : resourceTypes
|
||||
.entrySet()) {
|
||||
String name = entry.getKey();
|
||||
for (ResourceInformation entry : resourceTypesArray) {
|
||||
String name = entry.getName();
|
||||
if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
|
||||
ret.setMemorySize(entry.getValue().getMaximumAllocation());
|
||||
continue;
|
||||
}
|
||||
if (name.equals(ResourceInformation.VCORES.getName())) {
|
||||
Long tmp = entry.getValue().getMaximumAllocation();
|
||||
ret.setMemorySize(entry.getMaximumAllocation());
|
||||
} else if (name.equals(ResourceInformation.VCORES.getName())) {
|
||||
Long tmp = entry.getMaximumAllocation();
|
||||
if (tmp > Integer.MAX_VALUE) {
|
||||
tmp = (long) Integer.MAX_VALUE;
|
||||
}
|
||||
ret.setVirtualCores(tmp.intValue());
|
||||
continue;
|
||||
} else {
|
||||
ret.setResourceValue(name, entry.getMaximumAllocation());
|
||||
}
|
||||
ret.setResourceValue(name, entry.getValue().getMaximumAllocation());
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
@InterfaceAudience.Public
|
||||
@InterfaceStability.Unstable
|
||||
package org.apache.hadoop.yarn.util.resource;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
|
|
@ -472,9 +472,8 @@ public class ProtoUtils {
|
|||
List<YarnProtos.StringLongMapProto> pList) {
|
||||
Resource tmp = Resource.newInstance(0, 0);
|
||||
Map<String, Long> ret = new HashMap<>();
|
||||
for (Map.Entry<String, ResourceInformation> entry : tmp.getResources()
|
||||
.entrySet()) {
|
||||
ret.put(entry.getKey(), 0L);
|
||||
for (ResourceInformation entry : tmp.getResources()) {
|
||||
ret.put(entry.getName(), 0L);
|
||||
}
|
||||
if (pList != null) {
|
||||
for (YarnProtos.StringLongMapProto p : pList) {
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
|||
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||
import org.apache.hadoop.yarn.api.records.impl.BaseResource;
|
||||
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
||||
|
|
@ -33,14 +34,13 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto;
|
|||
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Arrays;
|
||||
import java.util.Map;
|
||||
import java.util.Collections;
|
||||
|
||||
|
||||
@Private
|
||||
@Unstable
|
||||
public class ResourcePBImpl extends Resource {
|
||||
public class ResourcePBImpl extends BaseResource {
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(ResourcePBImpl.class);
|
||||
|
||||
|
|
@ -48,10 +48,6 @@ public class ResourcePBImpl extends Resource {
|
|||
ResourceProto.Builder builder = null;
|
||||
boolean viaProto = false;
|
||||
|
||||
private Map<String, ResourceInformation> resources;
|
||||
private Map<String, ResourceInformation> readOnlyResources;
|
||||
|
||||
|
||||
// call via ProtoUtils.convertToProtoFormat(Resource)
|
||||
static ResourceProto getProto(Resource r) {
|
||||
final ResourcePBImpl pb;
|
||||
|
|
@ -72,8 +68,6 @@ public class ResourcePBImpl extends Resource {
|
|||
public ResourcePBImpl(ResourceProto proto) {
|
||||
this.proto = proto;
|
||||
viaProto = true;
|
||||
this.readOnlyResources = null;
|
||||
this.resources = null;
|
||||
initResources();
|
||||
}
|
||||
|
||||
|
|
@ -101,11 +95,13 @@ public class ResourcePBImpl extends Resource {
|
|||
public long getMemorySize() {
|
||||
// memory should always be present
|
||||
initResources();
|
||||
ResourceInformation ri =
|
||||
this.getResourceInformation(ResourceInformation.MEMORY_MB.getName());
|
||||
return UnitsConversionUtil
|
||||
.convert(ri.getUnits(), ResourceInformation.MEMORY_MB.getUnits(),
|
||||
ri.getValue());
|
||||
ResourceInformation ri = resources[MandatoryResources.MEMORY.getId()];
|
||||
|
||||
if (ri.getUnits().isEmpty()) {
|
||||
return ri.getValue();
|
||||
}
|
||||
return UnitsConversionUtil.convert(ri.getUnits(),
|
||||
ResourceInformation.MEMORY_MB.getUnits(), ri.getValue());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -117,23 +113,20 @@ public class ResourcePBImpl extends Resource {
|
|||
@Override
|
||||
public void setMemorySize(long memory) {
|
||||
maybeInitBuilder();
|
||||
getResourceInformation(ResourceInformation.MEMORY_MB.getName())
|
||||
.setValue(memory);
|
||||
getResourceInformation(MEMORY).setValue(memory);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getVirtualCores() {
|
||||
// vcores should always be present
|
||||
initResources();
|
||||
return this.getResourceValue(ResourceInformation.VCORES.getName())
|
||||
.intValue();
|
||||
return (int) resources[MandatoryResources.VCORES.getId()].getValue();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setVirtualCores(int vCores) {
|
||||
maybeInitBuilder();
|
||||
getResourceInformation(ResourceInformation.VCORES.getName())
|
||||
.setValue(vCores);
|
||||
getResourceInformation(VCORES).setValue(vCores);
|
||||
}
|
||||
|
||||
private void initResources() {
|
||||
|
|
@ -142,6 +135,7 @@ public class ResourcePBImpl extends Resource {
|
|||
}
|
||||
ResourceProtoOrBuilder p = viaProto ? proto : builder;
|
||||
initResourcesMap();
|
||||
Map<String, Integer> indexMap = ResourceUtils.getResourceTypeIndex();
|
||||
for (ResourceInformationProto entry : p.getResourceValueMapList()) {
|
||||
ResourceTypes type =
|
||||
entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) :
|
||||
|
|
@ -150,14 +144,16 @@ public class ResourcePBImpl extends Resource {
|
|||
long value = entry.hasValue() ? entry.getValue() : 0L;
|
||||
ResourceInformation ri = ResourceInformation
|
||||
.newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE);
|
||||
if (resources.containsKey(ri.getName())) {
|
||||
resources.get(ri.getName()).setResourceType(ri.getResourceType());
|
||||
resources.get(ri.getName()).setUnits(ri.getUnits());
|
||||
resources.get(ri.getName()).setValue(value);
|
||||
} else {
|
||||
Integer index = indexMap.get(entry.getKey());
|
||||
if (index == null) {
|
||||
LOG.warn("Got unknown resource type: " + ri.getName() + "; skipping");
|
||||
} else {
|
||||
resources[index].setResourceType(ri.getResourceType());
|
||||
resources[index].setUnits(ri.getUnits());
|
||||
resources[index].setValue(value);
|
||||
}
|
||||
}
|
||||
readOnlyResources = Arrays.copyOf(resources, resources.length);
|
||||
this.setMemorySize(p.getMemory());
|
||||
this.setVirtualCores(p.getVirtualCores());
|
||||
}
|
||||
|
|
@ -173,79 +169,67 @@ public class ResourcePBImpl extends Resource {
|
|||
if (!resource.equals(resourceInformation.getName())) {
|
||||
resourceInformation.setName(resource);
|
||||
}
|
||||
initResources();
|
||||
if (resources.containsKey(resource)) {
|
||||
ResourceInformation.copy(resourceInformation, resources.get(resource));
|
||||
}
|
||||
ResourceInformation storedResourceInfo = getResourceInformation(resource);
|
||||
ResourceInformation.copy(resourceInformation, storedResourceInfo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setResourceValue(String resource, Long value)
|
||||
throws ResourceNotFoundException {
|
||||
maybeInitBuilder();
|
||||
initResources();
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("resource type object cannot be null");
|
||||
}
|
||||
if (resources == null || (!resources.containsKey(resource))) {
|
||||
throw new ResourceNotFoundException(
|
||||
"Resource " + resource + " not found");
|
||||
}
|
||||
resources.get(resource).setValue(value);
|
||||
getResourceInformation(resource).setValue(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, ResourceInformation> getResources() {
|
||||
public ResourceInformation[] getResources() {
|
||||
initResources();
|
||||
return readOnlyResources;
|
||||
return super.getResources();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResourceInformation getResourceInformation(String resource) {
|
||||
public ResourceInformation getResourceInformation(String resource)
|
||||
throws ResourceNotFoundException {
|
||||
initResources();
|
||||
if (this.resources.containsKey(resource)) {
|
||||
return this.resources.get(resource);
|
||||
}
|
||||
throw new ResourceNotFoundException("Could not find entry for " + resource);
|
||||
return super.getResourceInformation(resource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long getResourceValue(String resource) {
|
||||
public long getResourceValue(String resource)
|
||||
throws ResourceNotFoundException {
|
||||
initResources();
|
||||
if (this.resources.containsKey(resource)) {
|
||||
return this.resources.get(resource).getValue();
|
||||
}
|
||||
throw new ResourceNotFoundException("Could not find entry for " + resource);
|
||||
return super.getResourceValue(resource);
|
||||
}
|
||||
|
||||
private void initResourcesMap() {
|
||||
if (resources == null) {
|
||||
resources = new HashMap<>();
|
||||
Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
|
||||
ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
|
||||
if (types == null) {
|
||||
throw new YarnRuntimeException(
|
||||
"Got null return value from ResourceUtils.getResourceTypes()");
|
||||
}
|
||||
for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
|
||||
resources.put(entry.getKey(),
|
||||
ResourceInformation.newInstance(entry.getValue()));
|
||||
|
||||
resources = new ResourceInformation[types.length];
|
||||
readOnlyResources = new ResourceInformation[types.length];
|
||||
for (ResourceInformation entry : types) {
|
||||
int index = ResourceUtils.getResourceTypeIndex().get(entry.getName());
|
||||
resources[index] = ResourceInformation.newInstance(entry);
|
||||
}
|
||||
readOnlyResources = Collections.unmodifiableMap(resources);
|
||||
}
|
||||
}
|
||||
|
||||
synchronized private void mergeLocalToBuilder() {
|
||||
builder.clearResourceValueMap();
|
||||
if (resources != null && !resources.isEmpty()) {
|
||||
for (Map.Entry<String, ResourceInformation> entry :
|
||||
resources.entrySet()) {
|
||||
ResourceInformationProto.Builder e =
|
||||
ResourceInformationProto.newBuilder();
|
||||
e.setKey(entry.getKey());
|
||||
e.setUnits(entry.getValue().getUnits());
|
||||
e.setType(
|
||||
ProtoUtils.converToProtoFormat(entry.getValue().getResourceType()));
|
||||
e.setValue(entry.getValue().getValue());
|
||||
if(resources != null && resources.length != 0) {
|
||||
for (ResourceInformation resInfo : resources) {
|
||||
ResourceInformationProto.Builder e = ResourceInformationProto
|
||||
.newBuilder();
|
||||
e.setKey(resInfo.getName());
|
||||
e.setUnits(resInfo.getUnits());
|
||||
e.setType(ProtoUtils.converToProtoFormat(resInfo.getResourceType()));
|
||||
e.setValue(resInfo.getValue());
|
||||
builder.addResourceValueMap(e);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,10 +23,9 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
|
|||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
|
||||
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* A {@link ResourceCalculator} which uses the concept of
|
||||
|
|
@ -56,10 +55,10 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
LogFactory.getLog(DominantResourceCalculator.class);
|
||||
|
||||
|
||||
private Set<String> resourceNames;
|
||||
private String[] resourceNames;
|
||||
|
||||
public DominantResourceCalculator() {
|
||||
resourceNames = ResourceUtils.getResourceTypes().keySet();
|
||||
resourceNames = ResourceUtils.getResourceNamesArray();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -88,7 +87,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
} else if (diff <= -1) {
|
||||
rhsGreater = true;
|
||||
}
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource information for " + rName, ye);
|
||||
}
|
||||
|
|
@ -163,7 +162,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
.getValue();
|
||||
min = min < tmp ? min : tmp;
|
||||
max = max > tmp ? max : tmp;
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource information for " + resource, ye);
|
||||
}
|
||||
|
|
@ -187,7 +186,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
long tmp = availableResource.getValue() / requiredResourceValue;
|
||||
min = min < tmp ? min : tmp;
|
||||
}
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource information for " + resource, ye);
|
||||
}
|
||||
|
|
@ -206,15 +205,10 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
|
||||
@Override
|
||||
public boolean isInvalidDivisor(Resource r) {
|
||||
for (String resource : resourceNames) {
|
||||
try {
|
||||
if (r.getResourceValue(resource).equals(0L)) {
|
||||
for (ResourceInformation res : r.getResources()) {
|
||||
if (res.getValue() == 0L) {
|
||||
return true;
|
||||
}
|
||||
} catch (YarnException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource value for " + resource, ye);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
@ -235,7 +229,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
float tmp =
|
||||
(float) aResourceInformation.getValue() / (float) bResourceValue;
|
||||
ratio = ratio > tmp ? ratio : tmp;
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource information for " + resource, ye);
|
||||
}
|
||||
|
|
@ -256,7 +250,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
ret.getResourceInformation(resource);
|
||||
resourceInformation.setValue(
|
||||
divideAndCeil(resourceInformation.getValue(), denominator));
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource information for " + resource, ye);
|
||||
}
|
||||
|
|
@ -307,7 +301,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
}
|
||||
tmp.setValue(Math.min(value, maximumValue));
|
||||
ret.setResourceInformation(resource, tmp);
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource information for " + resource, ye);
|
||||
}
|
||||
|
|
@ -347,7 +341,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
ResourceInformation
|
||||
.copy(rResourceInformation, ret.getResourceInformation(resource));
|
||||
ret.getResourceInformation(resource).setValue(value);
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource information for " + resource, ye);
|
||||
}
|
||||
|
|
@ -372,28 +366,29 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
Resource ret = Resource.newInstance(r);
|
||||
for (String resource : resourceNames) {
|
||||
try {
|
||||
ResourceInformation rResourceInformation =
|
||||
r.getResourceInformation(resource);
|
||||
ResourceInformation stepFactorResourceInformation =
|
||||
stepFactor.getResourceInformation(resource);
|
||||
ResourceInformation rResourceInformation = r
|
||||
.getResourceInformation(resource);
|
||||
ResourceInformation stepFactorResourceInformation = stepFactor
|
||||
.getResourceInformation(resource);
|
||||
ResourceInformation tmp = ret.getResourceInformation(resource);
|
||||
|
||||
Long rValue = rResourceInformation.getValue();
|
||||
Long stepFactorValue = UnitsConversionUtil
|
||||
.convert(stepFactorResourceInformation.getUnits(),
|
||||
long rValue = rResourceInformation.getValue();
|
||||
long stepFactorValue = UnitsConversionUtil.convert(
|
||||
stepFactorResourceInformation.getUnits(),
|
||||
rResourceInformation.getUnits(),
|
||||
stepFactorResourceInformation.getValue());
|
||||
Long value;
|
||||
long value;
|
||||
if (stepFactorValue != 0) {
|
||||
value = roundUp ?
|
||||
roundUp((long) Math.ceil(rValue * by), stepFactorValue) :
|
||||
roundDown((long) (rValue * by), stepFactorValue);
|
||||
value = roundUp
|
||||
? roundUp((long) Math.ceil(rValue * by), stepFactorValue)
|
||||
: roundDown((long) (rValue * by), stepFactorValue);
|
||||
} else {
|
||||
value =
|
||||
roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by);
|
||||
value = roundUp
|
||||
? (long) Math.ceil(rValue * by)
|
||||
: (long) (rValue * by);
|
||||
}
|
||||
tmp.setValue(value);
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource information for " + resource, ye);
|
||||
}
|
||||
|
|
@ -416,7 +411,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
if(sResourceValue > bResourceInformation.getValue()) {
|
||||
return false;
|
||||
}
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,30 +18,31 @@
|
|||
|
||||
package org.apache.hadoop.yarn.util.resource;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||
import org.apache.hadoop.yarn.api.records.impl.BaseResource;
|
||||
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.util.Records;
|
||||
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Arrays;
|
||||
|
||||
@InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" })
|
||||
@Unstable
|
||||
public class Resources {
|
||||
|
||||
private static final Log LOG =
|
||||
LogFactory.getLog(Resources.class);
|
||||
|
||||
/**
|
||||
* Helper class to create a resource with a fixed value for all resource
|
||||
* types. For example, a NONE resource which returns 0 for any resource type.
|
||||
*/
|
||||
static class FixedValueResource extends Resource {
|
||||
static class FixedValueResource extends BaseResource {
|
||||
|
||||
private Map<String, ResourceInformation> resources;
|
||||
private long resourceValue;
|
||||
private String name;
|
||||
|
||||
|
|
@ -53,7 +54,7 @@ public class Resources {
|
|||
FixedValueResource(String rName, long value) {
|
||||
this.resourceValue = value;
|
||||
this.name = rName;
|
||||
resources = initResourceMap();
|
||||
initResourceMap();
|
||||
}
|
||||
|
||||
private int resourceValueToInt() {
|
||||
|
|
@ -95,31 +96,6 @@ public class Resources {
|
|||
throw new RuntimeException(name + " cannot be modified!");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, ResourceInformation> getResources() {
|
||||
return Collections.unmodifiableMap(this.resources);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResourceInformation getResourceInformation(String resource)
|
||||
throws YarnException {
|
||||
if (resources.containsKey(resource)) {
|
||||
ResourceInformation value = this.resources.get(resource);
|
||||
ResourceInformation ret = ResourceInformation.newInstance(value);
|
||||
ret.setValue(resourceValue);
|
||||
return ret;
|
||||
}
|
||||
throw new YarnException("" + resource + " not found");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long getResourceValue(String resource) throws YarnException {
|
||||
if (resources.containsKey(resource)) {
|
||||
return resourceValue;
|
||||
}
|
||||
throw new YarnException("" + resource + " not found");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setResourceInformation(String resource,
|
||||
ResourceInformation resourceInformation)
|
||||
|
|
@ -133,24 +109,24 @@ public class Resources {
|
|||
throw new RuntimeException(name + " cannot be modified!");
|
||||
}
|
||||
|
||||
private Map<String, ResourceInformation> initResourceMap() {
|
||||
Map<String, ResourceInformation> tmp = new HashMap<>();
|
||||
Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
|
||||
private void initResourceMap() {
|
||||
ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
|
||||
if (types != null) {
|
||||
for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
|
||||
tmp.put(entry.getKey(),
|
||||
ResourceInformation.newInstance(entry.getValue()));
|
||||
tmp.get(entry.getKey()).setValue(resourceValue);
|
||||
}
|
||||
}
|
||||
// this is a fix for getVirtualCores returning an int
|
||||
if (resourceValue > Integer.MAX_VALUE) {
|
||||
tmp.get(ResourceInformation.VCORES.getName())
|
||||
.setValue((long) Integer.MAX_VALUE);
|
||||
}
|
||||
return tmp;
|
||||
}
|
||||
resources = new ResourceInformation[types.length];
|
||||
readOnlyResources = new ResourceInformation[types.length];
|
||||
for (int index = 0; index < types.length; index++) {
|
||||
resources[index] = ResourceInformation.newInstance(types[index]);
|
||||
resources[index].setValue(resourceValue);
|
||||
|
||||
// this is a fix for getVirtualCores returning an int
|
||||
if (resourceValue > Integer.MAX_VALUE && ResourceInformation.VCORES
|
||||
.getName().equals(resources[index].getName())) {
|
||||
resources[index].setValue((long) Integer.MAX_VALUE);
|
||||
}
|
||||
}
|
||||
}
|
||||
readOnlyResources = Arrays.copyOf(resources, resources.length);
|
||||
}
|
||||
}
|
||||
|
||||
public static Resource createResource(int memory) {
|
||||
|
|
@ -197,17 +173,19 @@ public class Resources {
|
|||
}
|
||||
|
||||
public static Resource addTo(Resource lhs, Resource rhs) {
|
||||
for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
|
||||
.entrySet()) {
|
||||
String name = entry.getKey();
|
||||
for (ResourceInformation entry : lhs.getResources()) {
|
||||
String name = entry.getName();
|
||||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(name);
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
long convertedRhs = UnitsConversionUtil
|
||||
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
|
||||
rhsValue.getValue());
|
||||
ResourceInformation lhsValue = entry;
|
||||
|
||||
long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
|
||||
? rhsValue.getValue()
|
||||
: UnitsConversionUtil.convert(rhsValue.getUnits(),
|
||||
lhsValue.getUnits(), rhsValue.getValue());
|
||||
lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
LOG.warn("Resource is missing:" + ye.getMessage());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
|
@ -219,17 +197,19 @@ public class Resources {
|
|||
}
|
||||
|
||||
public static Resource subtractFrom(Resource lhs, Resource rhs) {
|
||||
for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
|
||||
.entrySet()) {
|
||||
String name = entry.getKey();
|
||||
for (ResourceInformation entry : lhs.getResources()) {
|
||||
String name = entry.getName();
|
||||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(name);
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
long convertedRhs = UnitsConversionUtil
|
||||
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
|
||||
rhsValue.getValue());
|
||||
ResourceInformation lhsValue = entry;
|
||||
|
||||
long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
|
||||
? rhsValue.getValue()
|
||||
: UnitsConversionUtil.convert(rhsValue.getUnits(),
|
||||
lhsValue.getUnits(), rhsValue.getValue());
|
||||
lhs.setResourceValue(name, lhsValue.getValue() - convertedRhs);
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
LOG.warn("Resource is missing:" + ye.getMessage());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
|
@ -263,10 +243,9 @@ public class Resources {
|
|||
}
|
||||
|
||||
public static Resource multiplyTo(Resource lhs, double by) {
|
||||
for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
|
||||
.entrySet()) {
|
||||
String name = entry.getKey();
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
for (ResourceInformation entry : lhs.getResources()) {
|
||||
String name = entry.getName();
|
||||
ResourceInformation lhsValue = entry;
|
||||
lhs.setResourceValue(name, (long) (lhsValue.getValue() * by));
|
||||
}
|
||||
return lhs;
|
||||
|
|
@ -282,17 +261,21 @@ public class Resources {
|
|||
*/
|
||||
public static Resource multiplyAndAddTo(
|
||||
Resource lhs, Resource rhs, double by) {
|
||||
for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
|
||||
.entrySet()) {
|
||||
String name = entry.getKey();
|
||||
for (ResourceInformation entry : lhs.getResources()) {
|
||||
String name = entry.getName();
|
||||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(name);
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
long convertedRhs = (long) (UnitsConversionUtil
|
||||
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
|
||||
rhsValue.getValue()) * by);
|
||||
ResourceInformation lhsValue = entry;
|
||||
|
||||
long convertedRhs = (long) (((rhsValue.getUnits()
|
||||
.equals(lhsValue.getUnits()))
|
||||
? rhsValue.getValue()
|
||||
: UnitsConversionUtil.convert(rhsValue.getUnits(),
|
||||
lhsValue.getUnits(), rhsValue.getValue()))
|
||||
* by);
|
||||
lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
LOG.warn("Resource is missing:" + ye.getMessage());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
|
@ -311,10 +294,9 @@ public class Resources {
|
|||
|
||||
public static Resource multiplyAndRoundDown(Resource lhs, double by) {
|
||||
Resource out = clone(lhs);
|
||||
for (Map.Entry<String, ResourceInformation> entry : out.getResources()
|
||||
.entrySet()) {
|
||||
String name = entry.getKey();
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
for (ResourceInformation entry : out.getResources()) {
|
||||
String name = entry.getName();
|
||||
ResourceInformation lhsValue = entry;
|
||||
out.setResourceValue(name, (long) (lhsValue.getValue() * by));
|
||||
}
|
||||
return out;
|
||||
|
|
@ -416,19 +398,21 @@ public class Resources {
|
|||
}
|
||||
|
||||
public static boolean fitsIn(Resource smaller, Resource bigger) {
|
||||
for (Map.Entry<String, ResourceInformation> entry : smaller.getResources()
|
||||
.entrySet()) {
|
||||
String name = entry.getKey();
|
||||
for (ResourceInformation entry : smaller.getResources()) {
|
||||
String name = entry.getName();
|
||||
try {
|
||||
ResourceInformation rhsValue = bigger.getResourceInformation(name);
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
long convertedRhs = UnitsConversionUtil
|
||||
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
|
||||
rhsValue.getValue());
|
||||
ResourceInformation lhsValue = entry;
|
||||
|
||||
long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
|
||||
? rhsValue.getValue()
|
||||
: UnitsConversionUtil.convert(rhsValue.getUnits(),
|
||||
lhsValue.getUnits(), rhsValue.getValue());
|
||||
if(lhsValue.getValue() > convertedRhs) {
|
||||
return false;
|
||||
}
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
LOG.warn("Resource is missing:" + ye.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -442,19 +426,21 @@ public class Resources {
|
|||
|
||||
public static Resource componentwiseMin(Resource lhs, Resource rhs) {
|
||||
Resource ret = createResource(0);
|
||||
for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
|
||||
.entrySet()) {
|
||||
String name = entry.getKey();
|
||||
for (ResourceInformation entry : lhs.getResources()) {
|
||||
String name = entry.getName();
|
||||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(name);
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
long convertedRhs = UnitsConversionUtil
|
||||
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
|
||||
rhsValue.getValue());
|
||||
ResourceInformation lhsValue = entry;
|
||||
|
||||
long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
|
||||
? rhsValue.getValue()
|
||||
: UnitsConversionUtil.convert(rhsValue.getUnits(),
|
||||
lhsValue.getUnits(), rhsValue.getValue());
|
||||
ResourceInformation outInfo =
|
||||
lhsValue.getValue() < convertedRhs ? lhsValue : rhsValue;
|
||||
ret.setResourceInformation(name, outInfo);
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
LOG.warn("Resource is missing:" + ye.getMessage());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
|
@ -463,19 +449,21 @@ public class Resources {
|
|||
|
||||
public static Resource componentwiseMax(Resource lhs, Resource rhs) {
|
||||
Resource ret = createResource(0);
|
||||
for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
|
||||
.entrySet()) {
|
||||
String name = entry.getKey();
|
||||
for (ResourceInformation entry : lhs.getResources()) {
|
||||
String name = entry.getName();
|
||||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(name);
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
long convertedRhs = UnitsConversionUtil
|
||||
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
|
||||
rhsValue.getValue());
|
||||
ResourceInformation lhsValue = entry;
|
||||
|
||||
long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
|
||||
? rhsValue.getValue()
|
||||
: UnitsConversionUtil.convert(rhsValue.getUnits(),
|
||||
lhsValue.getUnits(), rhsValue.getValue());
|
||||
ResourceInformation outInfo =
|
||||
lhsValue.getValue() > convertedRhs ? lhsValue : rhsValue;
|
||||
ret.setResourceInformation(name, outInfo);
|
||||
} catch (YarnException ye) {
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
LOG.warn("Resource is missing:" + ye.getMessage());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -276,13 +276,17 @@ public class TestResourceUtils {
|
|||
String resourceFile = entry.getKey();
|
||||
ResourceUtils.resetNodeResources();
|
||||
File dest;
|
||||
File source =
|
||||
new File(conf.getClassLoader().getResource(resourceFile).getFile());
|
||||
File source = new File(
|
||||
conf.getClassLoader().getResource(resourceFile).getFile());
|
||||
dest = new File(source.getParent(), "node-resources.xml");
|
||||
FileUtils.copyFile(source, dest);
|
||||
Map<String, ResourceInformation> actual =
|
||||
ResourceUtils.getNodeResourceInformation(conf);
|
||||
Assert.assertEquals(entry.getValue().getResources(), actual);
|
||||
Map<String, ResourceInformation> actual = ResourceUtils
|
||||
.getNodeResourceInformation(conf);
|
||||
Assert.assertEquals(actual.size(),
|
||||
entry.getValue().getResources().length);
|
||||
for (ResourceInformation resInfo : entry.getValue().getResources()) {
|
||||
Assert.assertEquals(resInfo, actual.get(resInfo.getName()));
|
||||
}
|
||||
dest.delete();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -114,7 +114,6 @@ public class TestResources {
|
|||
assertTrue(Resources.none().compareTo(createResource(1, 0, 0)) < 0);
|
||||
assertTrue(Resources.none().compareTo(createResource(0, 1, 0)) < 0);
|
||||
assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0);
|
||||
assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0);
|
||||
}
|
||||
|
||||
@Test(timeout=10000)
|
||||
|
|
@ -246,7 +245,9 @@ public class TestResources {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testMultiplyAndAddTo() {
|
||||
public void testMultiplyAndAddTo() throws Exception {
|
||||
unsetExtraResourceType();
|
||||
setupExtraResourceType();
|
||||
assertEquals(createResource(6, 4),
|
||||
multiplyAndAddTo(createResource(3, 1), createResource(2, 2), 1.5));
|
||||
assertEquals(createResource(6, 4, 0),
|
||||
|
|
|
|||
|
|
@ -178,16 +178,15 @@ public class RMAppAttemptMetrics {
|
|||
|
||||
private void updateUsageMap(Resource allocated, long deltaUsedMillis,
|
||||
Map<String, AtomicLong> targetMap) {
|
||||
for (Map.Entry<String, ResourceInformation> entry : allocated.getResources()
|
||||
.entrySet()) {
|
||||
for (ResourceInformation entry : allocated.getResources()) {
|
||||
AtomicLong resourceUsed;
|
||||
if (!targetMap.containsKey(entry.getKey())) {
|
||||
if (!targetMap.containsKey(entry.getName())) {
|
||||
resourceUsed = new AtomicLong(0);
|
||||
targetMap.put(entry.getKey(), resourceUsed);
|
||||
targetMap.put(entry.getName(), resourceUsed);
|
||||
|
||||
}
|
||||
resourceUsed = targetMap.get(entry.getKey());
|
||||
resourceUsed.addAndGet((entry.getValue().getValue() * deltaUsedMillis)
|
||||
resourceUsed = targetMap.get(entry.getName());
|
||||
resourceUsed.addAndGet((entry.getValue() * deltaUsedMillis)
|
||||
/ DateUtils.MILLIS_PER_SECOND);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1006,13 +1006,12 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
|
|||
for (RMContainer rmContainer : this.liveContainers.values()) {
|
||||
long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
|
||||
Resource resource = rmContainer.getContainer().getResource();
|
||||
for (Map.Entry<String, ResourceInformation> entry : resource
|
||||
.getResources().entrySet()) {
|
||||
for (ResourceInformation entry : resource.getResources()) {
|
||||
long value = RMServerUtils
|
||||
.getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
|
||||
value += entry.getValue().getValue() * usedMillis
|
||||
.getOrDefault(resourceSecondsMap, entry.getName(), 0L);
|
||||
value += entry.getValue() * usedMillis
|
||||
/ DateUtils.MILLIS_PER_SECOND;
|
||||
resourceSecondsMap.put(entry.getKey(), value);
|
||||
resourceSecondsMap.put(entry.getName(), value);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.EnumSet;
|
||||
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
|
|
@ -73,7 +74,7 @@ public class SchedulerInfo {
|
|||
}
|
||||
|
||||
public String getSchedulerResourceTypes() {
|
||||
return minAllocResource.getResource().getResources().keySet().toString();
|
||||
return Arrays.toString(minAllocResource.getResource().getResources());
|
||||
}
|
||||
|
||||
public int getMaxClusterLevelAppPriority() {
|
||||
|
|
|
|||
|
|
@ -249,6 +249,7 @@ public class TestAppManager{
|
|||
ResourceScheduler scheduler = mockResourceScheduler();
|
||||
((RMContextImpl)rmContext).setScheduler(scheduler);
|
||||
Configuration conf = new Configuration();
|
||||
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
|
||||
((RMContextImpl) rmContext).setYarnConfiguration(conf);
|
||||
ApplicationMasterService masterService =
|
||||
new ApplicationMasterService(rmContext, scheduler);
|
||||
|
|
|
|||
Loading…
Reference in New Issue