diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 014557dc8bc..cdfff3dc4e8 100755 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -135,7 +135,6 @@ import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.util.Clock; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.RackResolver; import com.google.common.annotations.VisibleForTesting; @@ -664,7 +663,7 @@ public abstract class TaskAttemptImpl implements //TODO:create the resource reqt for this Task attempt this.resourceCapability = recordFactory.newRecordInstance(Resource.class); - this.resourceCapability.setMemory( + this.resourceCapability.setMemorySize( getMemoryRequired(conf, taskId.getTaskType())); this.resourceCapability.setVirtualCores( getCpuRequired(conf, taskId.getTaskType())); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index e4384b3bcd8..93905b6c344 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -382,7 +382,7 @@ public class RMContainerAllocator extends RMContainerRequestor } } // set the resources - reqEvent.getCapability().setMemory(mapResourceRequest.getMemorySize()); + reqEvent.getCapability().setMemorySize(mapResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( mapResourceRequest.getVirtualCores()); scheduledRequests.addMap(reqEvent);//maps are immediately scheduled @@ -410,7 +410,7 @@ public class RMContainerAllocator extends RMContainerRequestor } } // set the resources - reqEvent.getCapability().setMemory(reduceResourceRequest.getMemorySize()); + reqEvent.getCapability().setMemorySize(reduceResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( reduceResourceRequest.getVirtualCores()); if (reqEvent.getEarlierAttemptFailed()) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index 6ff243de724..d6a303cc684 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -253,7 +253,7 @@ public class MRApp extends MRAppMaster { // the job can reaches the final state when MRAppMaster shuts down. this.successfullyUnregistered.set(unregistered); this.assignedQueue = assignedQueue; - this.resource = Resource.newInstance(1234L, 2L); + this.resource = Resource.newInstance(1234L, 2); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java index 2027d01b70f..629a2467653 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java @@ -89,7 +89,7 @@ public class TestTypeConverter { ApplicationResourceUsageReport appUsageRpt = Records .newRecord(ApplicationResourceUsageReport.class); Resource r = Records.newRecord(Resource.class); - r.setMemory(2048); + r.setMemorySize(2048); appUsageRpt.setNeededResources(r); appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); @@ -128,7 +128,7 @@ public class TestTypeConverter { ApplicationResourceUsageReport appUsageRpt = Records .newRecord(ApplicationResourceUsageReport.class); Resource r = Records.newRecord(Resource.class); - r.setMemory(2048); + r.setMemorySize(2048); appUsageRpt.setNeededResources(r); appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index b30641ebb22..4c6f0f3139d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -337,7 +337,7 @@ public class YARNRunner implements ClientProtocol { // Setup resource requirements Resource capability = recordFactory.newRecordInstance(Resource.class); - capability.setMemory( + capability.setMemorySize( conf.getInt( MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB ) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index 6bee74dd4fc..a5a84292aa9 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -309,7 +309,7 @@ public class SLSRunner { if (jsonTask.containsKey("container.memory")) { int containerMemory = Integer.parseInt( jsonTask.get("container.memory").toString()); - res.setMemory(containerMemory); + res.setMemorySize(containerMemory); } if (jsonTask.containsKey("container.vcores")) { diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index 1a936abb468..f9e5facc35b 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -543,11 +543,11 @@ public class ResourceSchedulerWrapper } ); metrics.register("variable.cluster.allocated.vcores", - new Gauge() { + new Gauge() { @Override - public Long getValue() { + public Integer getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0L; + return 0; } else { return scheduler.getRootQueueMetrics().getAllocatedVirtualCores(); } @@ -567,11 +567,11 @@ public class ResourceSchedulerWrapper } ); metrics.register("variable.cluster.available.vcores", - new Gauge() { + new Gauge() { @Override - public Long getValue() { + public Integer getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0L; + return 0; } else { return scheduler.getRootQueueMetrics().getAvailableVirtualCores(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index cf3fbde4e5c..0d31b6fd7d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -19,14 +19,13 @@ package org.apache.hadoop.yarn.api.records; import org.apache.commons.lang.NotImplementedException; -import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.util.Records; + /** *

Resource models a set of computer resources in the * cluster.

@@ -56,9 +55,18 @@ public abstract class Resource implements Comparable { @Public @Stable - public static Resource newInstance(long memory, long vCores) { + public static Resource newInstance(int memory, int vCores) { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(memory); + resource.setMemorySize(memory); + resource.setVirtualCores(vCores); + return resource; + } + + @Public + @Stable + public static Resource newInstance(long memory, int vCores) { + Resource resource = Records.newRecord(Resource.class); + resource.setMemorySize(memory); resource.setVirtualCores(vCores); return resource; } @@ -78,19 +86,31 @@ public abstract class Resource implements Comparable { * Get memory of the resource. * @return memory of the resource */ - @Private - @Unstable + @Public + @Stable public long getMemorySize() { - throw new NotImplementedException("getVirtualCoresSize is not implemented"); + throw new NotImplementedException( + "This method is implemented by ResourcePBImpl"); } - + + /** + * Set memory of the resource. + * @param memory memory of the resource + */ + @Public + @Deprecated + public abstract void setMemory(int memory); + /** * Set memory of the resource. * @param memory memory of the resource */ @Public @Stable - public abstract void setMemory(long memory); + public void setMemorySize(long memory) { + throw new NotImplementedException( + "This method is implemented by ResourcePBImpl"); + } /** @@ -106,12 +126,6 @@ public abstract class Resource implements Comparable { @Public @Evolving public abstract int getVirtualCores(); - - @Public - @Unstable - public long getVirtualCoresSize() { - throw new NotImplementedException("getVirtualCoresSize is not implemented"); - } /** * Set number of virtual cpu cores of the resource. @@ -125,9 +139,7 @@ public abstract class Resource implements Comparable { */ @Public @Evolving - public void setVirtualCores(long vCores) { - throw new NotImplementedException("getVirtualCoresSize is not implemented"); - } + public abstract void setVirtualCores(int vCores); @Override public int hashCode() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 172cea3d9a9..c3d121a14d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -55,7 +55,7 @@ message ContainerIdProto { message ResourceProto { optional int64 memory = 1; - optional int64 virtual_cores = 2; + optional int32 virtual_cores = 2; } message ResourceUtilizationProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java index 8e00554ad6c..c9fa91513af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java @@ -171,7 +171,7 @@ public class TestApplicationClientProtocolOnHA extends ProtocolHATestBase { Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(10); + capability.setMemorySize(10); capability.setVirtualCores(1); appContext.setResource(capability); ApplicationId appId = client.submitApplication(appContext); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 928dc82a418..c569a37da95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -67,24 +67,25 @@ public class ResourcePBImpl extends Resource { } @Override - public void setMemory(long memory) { + @SuppressWarnings("deprecation") + public void setMemory(int memory) { + setMemorySize(memory); + } + + @Override + public void setMemorySize(long memory) { maybeInitBuilder(); builder.setMemory(memory); } @Override public int getVirtualCores() { - return (int) getVirtualCoresSize(); - } - - @Override - public long getVirtualCoresSize() { ResourceProtoOrBuilder p = viaProto ? proto : builder; return p.getVirtualCores(); } @Override - public void setVirtualCores(long vCores) { + public void setVirtualCores(int vCores) { maybeInitBuilder(); builder.setVirtualCores(vCores); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java index b98cf0c518c..fe78de2d972 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java @@ -56,7 +56,7 @@ public class DefaultResourceCalculator extends ResourceCalculator { } @Override - public Resource divideAndCeil(Resource numerator, long denominator) { + public Resource divideAndCeil(Resource numerator, int denominator) { return Resources.createResource( divideAndCeil(numerator.getMemorySize(), denominator)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 8f97fab0323..2695811fc18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -142,7 +142,7 @@ public class DominantResourceCalculator extends ResourceCalculator { } @Override - public Resource divideAndCeil(Resource numerator, long denominator) { + public Resource divideAndCeil(Resource numerator, int denominator) { return Resources.createResource( divideAndCeil(numerator.getMemorySize(), denominator), divideAndCeil(numerator.getVirtualCores(), denominator) @@ -157,7 +157,7 @@ public class DominantResourceCalculator extends ResourceCalculator { Math.max(r.getMemorySize(), minimumResource.getMemorySize()), stepFactor.getMemorySize()), maximumResource.getMemorySize()); - long normalizedCores = Math.min( + int normalizedCores = Math.min( roundUp( Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()), stepFactor.getVirtualCores()), @@ -189,7 +189,7 @@ public class DominantResourceCalculator extends ResourceCalculator { roundUp( (int)Math.ceil(r.getMemorySize() * by), stepFactor.getMemorySize()), roundUp( - (int)Math.ceil(r.getVirtualCores() * by), + (int)Math.ceil(r.getVirtualCores() * by), stepFactor.getVirtualCores()) ); } @@ -203,7 +203,7 @@ public class DominantResourceCalculator extends ResourceCalculator { stepFactor.getMemorySize() ), roundDown( - (int)(r.getVirtualCores() * by), + (int)(r.getVirtualCores() * by), stepFactor.getVirtualCores() ) ); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java index 3fe8adecb8b..a7df9c53c24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java @@ -30,6 +30,13 @@ public abstract class ResourceCalculator { public abstract int compare(Resource clusterResource, Resource lhs, Resource rhs); + + public static int divideAndCeil(int a, int b) { + if (b == 0) { + return 0; + } + return (a + (b - 1)) / b; + } public static long divideAndCeil(long a, long b) { if (b == 0) { @@ -38,6 +45,10 @@ public abstract class ResourceCalculator { return (a + (b - 1)) / b; } + public static int roundUp(int a, int b) { + return divideAndCeil(a, b) * b; + } + public static long roundUp(long a, long b) { return divideAndCeil(a, b) * b; } @@ -46,6 +57,10 @@ public abstract class ResourceCalculator { return (a / b) * b; } + public static int roundDown(int a, int b) { + return (a / b) * b; + } + /** * Compute the number of containers which can be allocated given * available and required resources. @@ -169,7 +184,7 @@ public abstract class ResourceCalculator { * @param denominator denominator * @return resultant resource */ - public abstract Resource divideAndCeil(Resource numerator, long denominator); + public abstract Resource divideAndCeil(Resource numerator, int denominator); /** * Check if a smaller resource can be contained by bigger resource. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 4645373ef5b..06586ae64e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -42,7 +42,13 @@ public class Resources { } @Override - public void setMemory(long memory) { + public void setMemorySize(long memory) { + throw new RuntimeException("NONE cannot be modified!"); + } + + @Override + @SuppressWarnings("deprecation") + public void setMemory(int memory) { throw new RuntimeException("NONE cannot be modified!"); } @@ -52,12 +58,7 @@ public class Resources { } @Override - public long getVirtualCoresSize() { - return 0; - } - - @Override - public void setVirtualCores(long cores) { + public void setVirtualCores(int cores) { throw new RuntimeException("NONE cannot be modified!"); } @@ -86,7 +87,13 @@ public class Resources { } @Override - public void setMemory(long memory) { + @SuppressWarnings("deprecation") + public void setMemory(int memory) { + throw new RuntimeException("UNBOUNDED cannot be modified!"); + } + + @Override + public void setMemorySize(long memory) { throw new RuntimeException("UNBOUNDED cannot be modified!"); } @@ -96,12 +103,7 @@ public class Resources { } @Override - public long getVirtualCoresSize() { - return Long.MAX_VALUE; - } - - @Override - public void setVirtualCores(long cores) { + public void setVirtualCores(int cores) { throw new RuntimeException("UNBOUNDED cannot be modified!"); } @@ -109,20 +111,31 @@ public class Resources { public int compareTo(Resource o) { long diff = Long.MAX_VALUE - o.getMemorySize(); if (diff == 0) { - diff = Long.MAX_VALUE - o.getVirtualCoresSize(); + diff = Integer.MAX_VALUE - o.getVirtualCores(); } return Long.signum(diff); } }; + public static Resource createResource(int memory) { + return createResource(memory, (memory > 0) ? 1 : 0); + } + + public static Resource createResource(int memory, int cores) { + Resource resource = Records.newRecord(Resource.class); + resource.setMemorySize(memory); + resource.setVirtualCores(cores); + return resource; + } + public static Resource createResource(long memory) { return createResource(memory, (memory > 0) ? 1 : 0); } - public static Resource createResource(long memory, long cores) { + public static Resource createResource(long memory, int cores) { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(cores); return resource; } @@ -140,7 +153,7 @@ public class Resources { } public static Resource addTo(Resource lhs, Resource rhs) { - lhs.setMemory(lhs.getMemorySize() + rhs.getMemorySize()); + lhs.setMemorySize(lhs.getMemorySize() + rhs.getMemorySize()); lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores()); return lhs; } @@ -150,7 +163,7 @@ public class Resources { } public static Resource subtractFrom(Resource lhs, Resource rhs) { - lhs.setMemory(lhs.getMemorySize() - rhs.getMemorySize()); + lhs.setMemorySize(lhs.getMemorySize() - rhs.getMemorySize()); lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores()); return lhs; } @@ -164,7 +177,7 @@ public class Resources { } public static Resource multiplyTo(Resource lhs, double by) { - lhs.setMemory((int)(lhs.getMemorySize() * by)); + lhs.setMemorySize((long)(lhs.getMemorySize() * by)); lhs.setVirtualCores((int)(lhs.getVirtualCores() * by)); return lhs; } @@ -179,7 +192,7 @@ public class Resources { */ public static Resource multiplyAndAddTo( Resource lhs, Resource rhs, double by) { - lhs.setMemory(lhs.getMemorySize() + (int)(rhs.getMemorySize() * by)); + lhs.setMemorySize(lhs.getMemorySize() + (long)(rhs.getMemorySize() * by)); lhs.setVirtualCores(lhs.getVirtualCores() + (int)(rhs.getVirtualCores() * by)); return lhs; @@ -197,7 +210,7 @@ public class Resources { public static Resource multiplyAndRoundDown(Resource lhs, double by) { Resource out = clone(lhs); - out.setMemory((int)(lhs.getMemorySize() * by)); + out.setMemorySize((long)(lhs.getMemorySize() * by)); out.setVirtualCores((int)(lhs.getVirtualCores() * by)); return out; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index 7396adc2a32..057214badae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -24,18 +24,18 @@ import static org.junit.Assert.assertTrue; public class TestResources { - public Resource createResource(long memory, long vCores) { + public Resource createResource(long memory, int vCores) { return Resource.newInstance(memory, vCores); } @Test(timeout=1000) public void testCompareToWithUnboundedResource() { assertTrue(Resources.unbounded().compareTo( - createResource(Long.MAX_VALUE, Long.MAX_VALUE)) == 0); + createResource(Long.MAX_VALUE, Integer.MAX_VALUE)) == 0); assertTrue(Resources.unbounded().compareTo( createResource(Long.MAX_VALUE, 0)) > 0); assertTrue(Resources.unbounded().compareTo( - createResource(0, Long.MAX_VALUE)) > 0); + createResource(0, Integer.MAX_VALUE)) > 0); } @Test(timeout=1000) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 134f22de331..a049fb0d3e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; -import org.apache.hadoop.yarn.util.ConverterUtils; import com.google.common.annotations.VisibleForTesting; @@ -442,9 +441,9 @@ public class BuilderUtils { return report; } - public static Resource newResource(long memory, long vCores) { + public static Resource newResource(long memory, int vCores) { Resource resource = recordFactory.newRecordInstance(Resource.class); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(vCores); return resource; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java index e25547d0d57..59278ef4f37 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java @@ -193,7 +193,7 @@ public class TestYarnServerApiClasses { original.setHttpPort(8080); original.setNodeId(getNodeId()); Resource resource = recordFactory.newRecordInstance(Resource.class); - resource.setMemory(10000); + resource.setMemorySize(10000); resource.setVirtualCores(2); original.setResource(resource); RegisterNodeManagerRequestPBImpl copy = new RegisterNodeManagerRequestPBImpl( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java index 6ff74c9f53e..4f726d44922 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java @@ -39,7 +39,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; @@ -464,7 +463,7 @@ public abstract class BaseAMRMProxyTest { pri.setPriority(priority); req.setPriority(pri); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(memory); + capability.setMemorySize(memory); capability.setVirtualCores(vCores); req.setCapability(capability); if (labelExpression != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java index c0210d5a0b3..a08ee82e759 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java @@ -33,16 +33,16 @@ public class TestNodeManagerMetrics { DefaultMetricsSystem.initialize("NodeManager"); NodeManagerMetrics metrics = NodeManagerMetrics.create(); Resource total = Records.newRecord(Resource.class); - total.setMemory(8*GiB); + total.setMemorySize(8*GiB); total.setVirtualCores(16); Resource resource = Records.newRecord(Resource.class); - resource.setMemory(512); //512MiB + resource.setMemorySize(512); //512MiB resource.setVirtualCores(2); Resource largerResource = Records.newRecord(Resource.class); - largerResource.setMemory(1024); + largerResource.setMemorySize(1024); largerResource.setVirtualCores(2); Resource smallerResource = Records.newRecord(Resource.class); - smallerResource.setMemory(256); + smallerResource.setMemorySize(256); smallerResource.setVirtualCores(1); metrics.addResource(total); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java index feb071fca83..413bbdf7db1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java @@ -57,7 +57,7 @@ public class ClusterNodeTracker { // Max allocation private long maxNodeMemory = -1; - private long maxNodeVCores = -1; + private int maxNodeVCores = -1; private Resource configuredMaxAllocation; private boolean forceConfiguredMaxAllocation = true; private long configuredMaxAllocationWaitTime; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 69c3e8b8773..4e364f70f52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -61,7 +61,7 @@ public class QueueMetrics implements MetricsSource { @Metric("# of apps failed") MutableCounterInt appsFailed; @Metric("Allocated memory in MB") MutableGaugeLong allocatedMB; - @Metric("Allocated CPU in virtual cores") MutableGaugeLong allocatedVCores; + @Metric("Allocated CPU in virtual cores") MutableGaugeInt allocatedVCores; @Metric("# of allocated containers") MutableGaugeInt allocatedContainers; @Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated; @Metric("Aggregate # of allocated node-local containers") @@ -72,12 +72,12 @@ public class QueueMetrics implements MetricsSource { MutableCounterLong aggregateOffSwitchContainersAllocated; @Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased; @Metric("Available memory in MB") MutableGaugeLong availableMB; - @Metric("Available CPU in virtual cores") MutableGaugeLong availableVCores; + @Metric("Available CPU in virtual cores") MutableGaugeInt availableVCores; @Metric("Pending memory allocation in MB") MutableGaugeLong pendingMB; - @Metric("Pending CPU allocation in virtual cores") MutableGaugeLong pendingVCores; + @Metric("Pending CPU allocation in virtual cores") MutableGaugeInt pendingVCores; @Metric("# of pending containers") MutableGaugeInt pendingContainers; @Metric("# of reserved memory in MB") MutableGaugeLong reservedMB; - @Metric("Reserved CPU in virtual cores") MutableGaugeLong reservedVCores; + @Metric("Reserved CPU in virtual cores") MutableGaugeInt reservedVCores; @Metric("# of reserved containers") MutableGaugeInt reservedContainers; @Metric("# of active users") MutableGaugeInt activeUsers; @Metric("# of active applications") MutableGaugeInt activeApplications; @@ -364,7 +364,7 @@ public class QueueMetrics implements MetricsSource { private void _incrPendingResources(int containers, Resource res) { pendingContainers.incr(containers); pendingMB.incr(res.getMemorySize() * containers); - pendingVCores.incr(res.getVirtualCoresSize() * containers); + pendingVCores.incr(res.getVirtualCores() * containers); } public void decrPendingResources(String user, int containers, Resource res) { @@ -381,7 +381,7 @@ public class QueueMetrics implements MetricsSource { private void _decrPendingResources(int containers, Resource res) { pendingContainers.decr(containers); pendingMB.decr(res.getMemorySize() * containers); - pendingVCores.decr(res.getVirtualCoresSize() * containers); + pendingVCores.decr(res.getVirtualCores() * containers); } public void incrNodeTypeAggregations(String user, NodeType type) { @@ -409,7 +409,7 @@ public class QueueMetrics implements MetricsSource { aggregateContainersAllocated.incr(containers); allocatedMB.incr(res.getMemorySize() * containers); - allocatedVCores.incr(res.getVirtualCoresSize() * containers); + allocatedVCores.incr(res.getVirtualCores() * containers); if (decrPending) { _decrPendingResources(containers, res); } @@ -448,7 +448,7 @@ public class QueueMetrics implements MetricsSource { allocatedContainers.decr(containers); aggregateContainersReleased.incr(containers); allocatedMB.decr(res.getMemorySize() * containers); - allocatedVCores.decr(res.getVirtualCoresSize() * containers); + allocatedVCores.decr(res.getVirtualCores() * containers); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.releaseResources(user, containers, res); @@ -561,14 +561,15 @@ public class QueueMetrics implements MetricsSource { } public Resource getAllocatedResources() { - return BuilderUtils.newResource(allocatedMB.value(), allocatedVCores.value()); + return BuilderUtils.newResource(allocatedMB.value(), + (int) allocatedVCores.value()); } public long getAllocatedMB() { return allocatedMB.value(); } - public long getAllocatedVirtualCores() { + public int getAllocatedVirtualCores() { return allocatedVCores.value(); } @@ -580,7 +581,7 @@ public class QueueMetrics implements MetricsSource { return availableMB.value(); } - public long getAvailableVirtualCores() { + public int getAvailableVirtualCores() { return availableVCores.value(); } @@ -588,7 +589,7 @@ public class QueueMetrics implements MetricsSource { return pendingMB.value(); } - public long getPendingVirtualCores() { + public int getPendingVirtualCores() { return pendingVCores.value(); } @@ -600,7 +601,7 @@ public class QueueMetrics implements MetricsSource { return reservedMB.value(); } - public long getReservedVirtualCores() { + public int getReservedVirtualCores() { return reservedVCores.value(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index aae07708e60..b4a2639a1bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -465,7 +465,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { public synchronized Resource getHeadroom() { // Corner case to deal with applications being slightly over-limit if (resourceLimit.getMemorySize() < 0) { - resourceLimit.setMemory(0); + resourceLimit.setMemorySize(0); } return resourceLimit; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java index 49892f73528..5605f18b773 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java @@ -66,7 +66,7 @@ public class CapacityHeadroomProvider { } // Corner case to deal with applications being slightly over-limit if (headroom.getMemorySize() < 0) { - headroom.setMemory(0); + headroom.setMemorySize(0); } return headroom; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 634f667cd5b..1eead9a11c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -194,7 +194,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt node.getUnallocatedResource()); } if (availableResources.getMemorySize() < 0) { - availableResources.setMemory(0); + availableResources.setMemorySize(0); } if (availableResources.getVirtualCores() < 0) { availableResources.setVirtualCores(0); @@ -460,7 +460,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt } public void clearPreemptedResources() { - preemptedResources.setMemory(0); + preemptedResources.setMemorySize(0); preemptedResources.setVirtualCores(0); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 6723fd9e749..7c9b11edbe4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -496,15 +496,15 @@ public class FSLeafQueue extends FSQueue { // maxAMResource Resource maxResource = Resources.clone(getFairShare()); if (maxResource.getMemorySize() == 0) { - maxResource.setMemory( + maxResource.setMemorySize( Math.min(scheduler.getRootQueueMetrics().getAvailableMB(), getMaxShare().getMemorySize())); } - if (maxResource.getVirtualCoresSize() == 0) { + if (maxResource.getVirtualCores() == 0) { maxResource.setVirtualCores(Math.min( scheduler.getRootQueueMetrics().getAvailableVirtualCores(), - getMaxShare().getVirtualCoresSize())); + getMaxShare().getVirtualCores())); } Resource maxAMResource = Resources.multiply(maxResource, maxAMShare); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java index 5f500cd9495..440c73cefdd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java @@ -266,10 +266,10 @@ public class ComputeFairShares { private static void setResourceValue(long val, Resource resource, ResourceType type) { switch (type) { case MEMORY: - resource.setMemory(val); + resource.setMemorySize(val); break; case CPU: - resource.setVirtualCores(val); + resource.setVirtualCores((int)val); break; default: throw new IllegalArgumentException("Invalid resource"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index 76d43e0c4fe..d198bb97afd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.api.records.Resource; @XmlAccessorType(XmlAccessType.FIELD) public class ResourceInfo { long memory; - long vCores; + int vCores; public ResourceInfo() { } @@ -42,7 +42,7 @@ public class ResourceInfo { return memory; } - public long getvCores() { + public int getvCores() { return vCores; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index f2b4ea7a4d9..dc17f5f0f1f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.util.Records; import org.apache.log4j.Logger; -import org.junit.Assert; public class MockAM { @@ -207,7 +206,7 @@ public class MockAM { pri.setPriority(priority); req.setPriority(pri); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(memory); + capability.setMemorySize(memory); req.setCapability(capability); if (labelExpression != null) { req.setNodeLabelExpression(labelExpression); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 14872f8acf8..e64d6bbc9e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -82,19 +82,19 @@ public class MockNodes { public static Resource newResource(int mem) { Resource rs = recordFactory.newRecordInstance(Resource.class); - rs.setMemory(mem); + rs.setMemorySize(mem); return rs; } public static Resource newUsedResource(Resource total) { Resource rs = recordFactory.newRecordInstance(Resource.class); - rs.setMemory((int)(Math.random() * total.getMemorySize())); + rs.setMemorySize((int)(Math.random() * total.getMemorySize())); return rs; } public static Resource newAvailResource(Resource total, Resource used) { Resource rs = recordFactory.newRecordInstance(Resource.class); - rs.setMemory(total.getMemorySize() - used.getMemorySize()); + rs.setMemorySize(total.getMemorySize() - used.getMemorySize()); return rs; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index f1f3d51943c..84fe7c0ea6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -455,7 +455,7 @@ public class MockRM extends ResourceManager { Map acls, String queue, String amLabel) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, name, user, acls, false, queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, @@ -508,7 +508,7 @@ public class MockRM extends ResourceManager { int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); return submitApp(resource, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, false, null, 0, null, true, Priority.newInstance(0)); @@ -517,7 +517,7 @@ public class MockRM extends ResourceManager { public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, @@ -532,7 +532,7 @@ public class MockRM extends ResourceManager { boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, ApplicationId applicationId) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, @@ -542,7 +542,7 @@ public class MockRM extends ResourceManager { public RMApp submitApp(int masterMemory, LogAggregationContext logAggregationContext) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 73bef0c7b71..eb0d5f13399 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -821,21 +821,21 @@ public class TestResourceTrackerService extends NodeLabelTestBase { resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction()); - capability.setMemory(2048); + capability.setMemorySize(2048); capability.setVirtualCores(1); req.setResource(capability); RegisterNodeManagerResponse response2 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction()); - capability.setMemory(1024); + capability.setMemorySize(1024); capability.setVirtualCores(4); req.setResource(capability); RegisterNodeManagerResponse response3 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction()); - capability.setMemory(2048); + capability.setMemorySize(2048); capability.setVirtualCores(4); req.setResource(capability); RegisterNodeManagerResponse response4 = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 3a60e0234f2..a9f1f634be3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -1345,7 +1345,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase // submit app with keepContainersAcrossApplicationAttempts true Resource resource = Records.newRecord(Resource.class); - resource.setMemory(200); + resource.setMemorySize(200); RMApp app0 = rm1.submitApp(resource, "", UserGroupInformation .getCurrentUser().getShortUserName(), null, false, null, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java index 4cb235fa323..d383bac2189 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java @@ -127,7 +127,7 @@ public class TestReservationSystemUtil { public Resource createResource(int memory, int vCores) { Resource resource = new ResourcePBImpl(); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(vCores); return resource; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index 62b1082f534..cb1104bffb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -402,9 +402,9 @@ public class TestQueueMetrics { } public static void checkResources(MetricsSource source, long allocatedMB, - long allocatedCores, int allocCtnrs, long aggreAllocCtnrs, - long aggreReleasedCtnrs, long availableMB, long availableCores, long pendingMB, - long pendingCores, int pendingCtnrs, long reservedMB, long reservedCores, + int allocatedCores, int allocCtnrs, long aggreAllocCtnrs, + long aggreReleasedCtnrs, long availableMB, int availableCores, long pendingMB, + int pendingCores, int pendingCtnrs, long reservedMB, int reservedCores, int reservedCtnrs) { MetricsRecordBuilder rb = getMetrics(source); assertGauge("AllocatedMB", allocatedMB, rb); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 2c4a1966639..0ee9b138bc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -1143,7 +1143,7 @@ public class TestDelegationTokenRenewer { // submit app1 with a token, set cancelTokenWhenComplete to false; Resource resource = Records.newRecord(Resource.class); - resource.setMemory(200); + resource.setMemorySize(200); RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, false, null); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); @@ -1205,7 +1205,7 @@ public class TestDelegationTokenRenewer { Assert.assertFalse(Renewer.cancelled); Resource resource = Records.newRecord(Resource.class); - resource.setMemory(200); + resource.setMemorySize(200); RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, true, null);