From 23eb3c7cebe9971b0a50a553eaca4fe2c1de632a Mon Sep 17 00:00:00 2001 From: Jian He Date: Mon, 11 Jul 2016 22:38:35 -0700 Subject: [PATCH] YARN-5270. Solve miscellaneous issues caused by YARN-4844. Contributed by Wangda Tan --- .../v2/app/job/impl/TaskAttemptImpl.java | 3 +- .../v2/app/rm/RMContainerAllocator.java | 4 +- .../hadoop/mapreduce/TestTypeConverter.java | 4 +- .../org/apache/hadoop/mapred/YARNRunner.java | 2 +- .../scheduler/ResourceSchedulerWrapper.java | 12 ++-- .../hadoop/yarn/api/records/Resource.java | 50 +++++++++------- .../src/main/proto/yarn_protos.proto | 2 +- .../TestApplicationClientProtocolOnHA.java | 2 +- .../api/records/impl/pb/ResourcePBImpl.java | 15 ++--- .../resource/DefaultResourceCalculator.java | 2 +- .../resource/DominantResourceCalculator.java | 4 +- .../util/resource/ResourceCalculator.java | 17 +++++- .../hadoop/yarn/util/resource/Resources.java | 57 ++++++++++++------- .../yarn/util/resource/TestResources.java | 6 +- .../yarn/server/utils/BuilderUtils.java | 5 +- .../hadoop/yarn/TestYarnServerApiClasses.java | 2 +- .../amrmproxy/BaseAMRMProxyTest.java | 2 +- .../metrics/TestNodeManagerMetrics.java | 8 +-- .../ResourceTrackerService.java | 2 +- .../scheduler/AbstractYarnScheduler.java | 6 +- .../scheduler/QueueMetrics.java | 27 ++++----- .../SchedulerApplicationAttempt.java | 2 +- .../capacity/CapacityHeadroomProvider.java | 2 +- .../scheduler/fair/FSAppAttempt.java | 2 +- .../fair/policies/ComputeFairShares.java | 4 +- .../webapp/dao/ResourceInfo.java | 4 +- .../yarn/server/resourcemanager/MockAM.java | 2 +- .../server/resourcemanager/MockNodes.java | 6 +- .../yarn/server/resourcemanager/MockRM.java | 10 ++-- .../TestResourceTrackerService.java | 6 +- .../TestWorkPreservingRMRestart.java | 2 +- .../TestReservationSystemUtil.java | 2 +- .../scheduler/TestQueueMetrics.java | 6 +- .../security/TestDelegationTokenRenewer.java | 4 +- 34 files changed, 162 insertions(+), 122 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 4e2b13d975a..379b828923a 100755 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -135,7 +135,6 @@ import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.util.Clock; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.RackResolver; import com.google.common.annotations.VisibleForTesting; @@ -664,7 +663,7 @@ public abstract class TaskAttemptImpl implements //TODO:create the resource reqt for this Task attempt this.resourceCapability = recordFactory.newRecordInstance(Resource.class); - this.resourceCapability.setMemory( + this.resourceCapability.setMemorySize( getMemoryRequired(conf, taskId.getTaskType())); this.resourceCapability.setVirtualCores( getCpuRequired(conf, taskId.getTaskType())); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index 9b244fdfb54..4e28ae9a030 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -374,7 +374,7 @@ public class RMContainerAllocator extends RMContainerRequestor } } // set the resources - reqEvent.getCapability().setMemory(mapResourceRequest.getMemorySize()); + reqEvent.getCapability().setMemorySize(mapResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( mapResourceRequest.getVirtualCores()); scheduledRequests.addMap(reqEvent);//maps are immediately scheduled @@ -402,7 +402,7 @@ public class RMContainerAllocator extends RMContainerRequestor } } // set the resources - reqEvent.getCapability().setMemory(reduceResourceRequest.getMemorySize()); + reqEvent.getCapability().setMemorySize(reduceResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( reduceResourceRequest.getVirtualCores()); if (reqEvent.getEarlierAttemptFailed()) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java index 2027d01b70f..629a2467653 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java @@ -89,7 +89,7 @@ public class TestTypeConverter { ApplicationResourceUsageReport appUsageRpt = Records .newRecord(ApplicationResourceUsageReport.class); Resource r = Records.newRecord(Resource.class); - r.setMemory(2048); + r.setMemorySize(2048); appUsageRpt.setNeededResources(r); appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); @@ -128,7 +128,7 @@ public class TestTypeConverter { ApplicationResourceUsageReport appUsageRpt = Records .newRecord(ApplicationResourceUsageReport.class); Resource r = Records.newRecord(Resource.class); - r.setMemory(2048); + r.setMemorySize(2048); appUsageRpt.setNeededResources(r); appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index b30641ebb22..4c6f0f3139d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -337,7 +337,7 @@ public class YARNRunner implements ClientProtocol { // Setup resource requirements Resource capability = recordFactory.newRecordInstance(Resource.class); - capability.setMemory( + capability.setMemorySize( conf.getInt( MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB ) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index 1a936abb468..f9e5facc35b 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -543,11 +543,11 @@ public class ResourceSchedulerWrapper } ); metrics.register("variable.cluster.allocated.vcores", - new Gauge() { + new Gauge() { @Override - public Long getValue() { + public Integer getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0L; + return 0; } else { return scheduler.getRootQueueMetrics().getAllocatedVirtualCores(); } @@ -567,11 +567,11 @@ public class ResourceSchedulerWrapper } ); metrics.register("variable.cluster.available.vcores", - new Gauge() { + new Gauge() { @Override - public Long getValue() { + public Integer getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0L; + return 0; } else { return scheduler.getRootQueueMetrics().getAvailableVirtualCores(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index cf3fbde4e5c..35428ec792c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -19,14 +19,13 @@ package org.apache.hadoop.yarn.api.records; import org.apache.commons.lang.NotImplementedException; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.util.Records; + /** *

Resource models a set of computer resources in the * cluster.

@@ -56,9 +55,18 @@ public abstract class Resource implements Comparable { @Public @Stable - public static Resource newInstance(long memory, long vCores) { + public static Resource newInstance(int memory, int vCores) { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(memory); + resource.setMemorySize(memory); + resource.setVirtualCores(vCores); + return resource; + } + + @Public + @Stable + public static Resource newInstance(long memory, int vCores) { + Resource resource = Records.newRecord(Resource.class); + resource.setMemorySize(memory); resource.setVirtualCores(vCores); return resource; } @@ -78,19 +86,31 @@ public abstract class Resource implements Comparable { * Get memory of the resource. * @return memory of the resource */ - @Private - @Unstable + @Public + @Stable public long getMemorySize() { - throw new NotImplementedException("getVirtualCoresSize is not implemented"); + throw new NotImplementedException( + "This method is implemented by ResourcePBImpl"); } - + + /** + * Set memory of the resource. + * @param memory memory of the resource + */ + @Public + @Deprecated + public abstract void setMemory(int memory); + /** * Set memory of the resource. * @param memory memory of the resource */ @Public @Stable - public abstract void setMemory(long memory); + public void setMemorySize(long memory) { + throw new NotImplementedException( + "This method is implemented by ResourcePBImpl"); + } /** @@ -106,12 +126,6 @@ public abstract class Resource implements Comparable { @Public @Evolving public abstract int getVirtualCores(); - - @Public - @Unstable - public long getVirtualCoresSize() { - throw new NotImplementedException("getVirtualCoresSize is not implemented"); - } /** * Set number of virtual cpu cores of the resource. @@ -125,9 +139,7 @@ public abstract class Resource implements Comparable { */ @Public @Evolving - public void setVirtualCores(long vCores) { - throw new NotImplementedException("getVirtualCoresSize is not implemented"); - } + public abstract void setVirtualCores(int vCores); @Override public int hashCode() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index a3741f11251..0e0923ba7da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -55,7 +55,7 @@ message ContainerIdProto { message ResourceProto { optional int64 memory = 1; - optional int64 virtual_cores = 2; + optional int32 virtual_cores = 2; } message ResourceUtilizationProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java index 8e00554ad6c..c9fa91513af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java @@ -171,7 +171,7 @@ public class TestApplicationClientProtocolOnHA extends ProtocolHATestBase { Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(10); + capability.setMemorySize(10); capability.setVirtualCores(1); appContext.setResource(capability); ApplicationId appId = client.submitApplication(appContext); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 928dc82a418..c569a37da95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -67,24 +67,25 @@ public class ResourcePBImpl extends Resource { } @Override - public void setMemory(long memory) { + @SuppressWarnings("deprecation") + public void setMemory(int memory) { + setMemorySize(memory); + } + + @Override + public void setMemorySize(long memory) { maybeInitBuilder(); builder.setMemory(memory); } @Override public int getVirtualCores() { - return (int) getVirtualCoresSize(); - } - - @Override - public long getVirtualCoresSize() { ResourceProtoOrBuilder p = viaProto ? proto : builder; return p.getVirtualCores(); } @Override - public void setVirtualCores(long vCores) { + public void setVirtualCores(int vCores) { maybeInitBuilder(); builder.setVirtualCores(vCores); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java index b98cf0c518c..fe78de2d972 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java @@ -56,7 +56,7 @@ public class DefaultResourceCalculator extends ResourceCalculator { } @Override - public Resource divideAndCeil(Resource numerator, long denominator) { + public Resource divideAndCeil(Resource numerator, int denominator) { return Resources.createResource( divideAndCeil(numerator.getMemorySize(), denominator)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 8f97fab0323..c73a81f5afd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -142,7 +142,7 @@ public class DominantResourceCalculator extends ResourceCalculator { } @Override - public Resource divideAndCeil(Resource numerator, long denominator) { + public Resource divideAndCeil(Resource numerator, int denominator) { return Resources.createResource( divideAndCeil(numerator.getMemorySize(), denominator), divideAndCeil(numerator.getVirtualCores(), denominator) @@ -157,7 +157,7 @@ public class DominantResourceCalculator extends ResourceCalculator { Math.max(r.getMemorySize(), minimumResource.getMemorySize()), stepFactor.getMemorySize()), maximumResource.getMemorySize()); - long normalizedCores = Math.min( + int normalizedCores = Math.min( roundUp( Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()), stepFactor.getVirtualCores()), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java index 3fe8adecb8b..a7df9c53c24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java @@ -30,6 +30,13 @@ public abstract class ResourceCalculator { public abstract int compare(Resource clusterResource, Resource lhs, Resource rhs); + + public static int divideAndCeil(int a, int b) { + if (b == 0) { + return 0; + } + return (a + (b - 1)) / b; + } public static long divideAndCeil(long a, long b) { if (b == 0) { @@ -38,6 +45,10 @@ public abstract class ResourceCalculator { return (a + (b - 1)) / b; } + public static int roundUp(int a, int b) { + return divideAndCeil(a, b) * b; + } + public static long roundUp(long a, long b) { return divideAndCeil(a, b) * b; } @@ -46,6 +57,10 @@ public abstract class ResourceCalculator { return (a / b) * b; } + public static int roundDown(int a, int b) { + return (a / b) * b; + } + /** * Compute the number of containers which can be allocated given * available and required resources. @@ -169,7 +184,7 @@ public abstract class ResourceCalculator { * @param denominator denominator * @return resultant resource */ - public abstract Resource divideAndCeil(Resource numerator, long denominator); + public abstract Resource divideAndCeil(Resource numerator, int denominator); /** * Check if a smaller resource can be contained by bigger resource. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 93932987dc8..fdd9ec767aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -42,7 +42,13 @@ public class Resources { } @Override - public void setMemory(long memory) { + public void setMemorySize(long memory) { + throw new RuntimeException("NONE cannot be modified!"); + } + + @Override + @SuppressWarnings("deprecation") + public void setMemory(int memory) { throw new RuntimeException("NONE cannot be modified!"); } @@ -52,12 +58,7 @@ public class Resources { } @Override - public long getVirtualCoresSize() { - return 0; - } - - @Override - public void setVirtualCores(long cores) { + public void setVirtualCores(int cores) { throw new RuntimeException("NONE cannot be modified!"); } @@ -86,7 +87,13 @@ public class Resources { } @Override - public void setMemory(long memory) { + @SuppressWarnings("deprecation") + public void setMemory(int memory) { + throw new RuntimeException("UNBOUNDED cannot be modified!"); + } + + @Override + public void setMemorySize(long memory) { throw new RuntimeException("UNBOUNDED cannot be modified!"); } @@ -96,12 +103,7 @@ public class Resources { } @Override - public long getVirtualCoresSize() { - return Long.MAX_VALUE; - } - - @Override - public void setVirtualCores(long cores) { + public void setVirtualCores(int cores) { throw new RuntimeException("UNBOUNDED cannot be modified!"); } @@ -109,20 +111,31 @@ public class Resources { public int compareTo(Resource o) { long diff = Long.MAX_VALUE - o.getMemorySize(); if (diff == 0) { - diff = Long.MAX_VALUE - o.getVirtualCoresSize(); + diff = Integer.MAX_VALUE - o.getVirtualCores(); } return Long.signum(diff); } }; + public static Resource createResource(int memory) { + return createResource(memory, (memory > 0) ? 1 : 0); + } + + public static Resource createResource(int memory, int cores) { + Resource resource = Records.newRecord(Resource.class); + resource.setMemorySize(memory); + resource.setVirtualCores(cores); + return resource; + } + public static Resource createResource(long memory) { return createResource(memory, (memory > 0) ? 1 : 0); } - public static Resource createResource(long memory, long cores) { + public static Resource createResource(long memory, int cores) { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(cores); return resource; } @@ -140,7 +153,7 @@ public class Resources { } public static Resource addTo(Resource lhs, Resource rhs) { - lhs.setMemory(lhs.getMemorySize() + rhs.getMemorySize()); + lhs.setMemorySize(lhs.getMemorySize() + rhs.getMemorySize()); lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores()); return lhs; } @@ -150,7 +163,7 @@ public class Resources { } public static Resource subtractFrom(Resource lhs, Resource rhs) { - lhs.setMemory(lhs.getMemorySize() - rhs.getMemorySize()); + lhs.setMemorySize(lhs.getMemorySize() - rhs.getMemorySize()); lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores()); return lhs; } @@ -164,7 +177,7 @@ public class Resources { } public static Resource multiplyTo(Resource lhs, double by) { - lhs.setMemory((int)(lhs.getMemorySize() * by)); + lhs.setMemorySize((long)(lhs.getMemorySize() * by)); lhs.setVirtualCores((int)(lhs.getVirtualCores() * by)); return lhs; } @@ -179,7 +192,7 @@ public class Resources { */ public static Resource multiplyAndAddTo( Resource lhs, Resource rhs, double by) { - lhs.setMemory(lhs.getMemorySize() + (int)(rhs.getMemorySize() * by)); + lhs.setMemorySize(lhs.getMemorySize() + (long)(rhs.getMemorySize() * by)); lhs.setVirtualCores(lhs.getVirtualCores() + (int)(rhs.getVirtualCores() * by)); return lhs; @@ -197,7 +210,7 @@ public class Resources { public static Resource multiplyAndRoundDown(Resource lhs, double by) { Resource out = clone(lhs); - out.setMemory((int)(lhs.getMemorySize() * by)); + out.setMemorySize((long)(lhs.getMemorySize() * by)); out.setVirtualCores((int)(lhs.getVirtualCores() * by)); return out; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index 7396adc2a32..057214badae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -24,18 +24,18 @@ import static org.junit.Assert.assertTrue; public class TestResources { - public Resource createResource(long memory, long vCores) { + public Resource createResource(long memory, int vCores) { return Resource.newInstance(memory, vCores); } @Test(timeout=1000) public void testCompareToWithUnboundedResource() { assertTrue(Resources.unbounded().compareTo( - createResource(Long.MAX_VALUE, Long.MAX_VALUE)) == 0); + createResource(Long.MAX_VALUE, Integer.MAX_VALUE)) == 0); assertTrue(Resources.unbounded().compareTo( createResource(Long.MAX_VALUE, 0)) > 0); assertTrue(Resources.unbounded().compareTo( - createResource(0, Long.MAX_VALUE)) > 0); + createResource(0, Integer.MAX_VALUE)) > 0); } @Test(timeout=1000) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index d3e76f4fb8b..9ad1d4d6a9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -63,7 +63,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; -import org.apache.hadoop.yarn.util.ConverterUtils; import com.google.common.annotations.VisibleForTesting; @@ -425,9 +424,9 @@ public class BuilderUtils { return report; } - public static Resource newResource(long memory, long vCores) { + public static Resource newResource(long memory, int vCores) { Resource resource = recordFactory.newRecordInstance(Resource.class); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(vCores); return resource; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java index e25547d0d57..59278ef4f37 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java @@ -193,7 +193,7 @@ public class TestYarnServerApiClasses { original.setHttpPort(8080); original.setNodeId(getNodeId()); Resource resource = recordFactory.newRecordInstance(Resource.class); - resource.setMemory(10000); + resource.setMemorySize(10000); resource.setVirtualCores(2); original.setResource(resource); RegisterNodeManagerRequestPBImpl copy = new RegisterNodeManagerRequestPBImpl( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java index 1aea9d27635..ea0e3307130 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java @@ -462,7 +462,7 @@ public abstract class BaseAMRMProxyTest { pri.setPriority(priority); req.setPriority(pri); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(memory); + capability.setMemorySize(memory); capability.setVirtualCores(vCores); req.setCapability(capability); if (labelExpression != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java index c0210d5a0b3..a08ee82e759 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java @@ -33,16 +33,16 @@ public class TestNodeManagerMetrics { DefaultMetricsSystem.initialize("NodeManager"); NodeManagerMetrics metrics = NodeManagerMetrics.create(); Resource total = Records.newRecord(Resource.class); - total.setMemory(8*GiB); + total.setMemorySize(8*GiB); total.setVirtualCores(16); Resource resource = Records.newRecord(Resource.class); - resource.setMemory(512); //512MiB + resource.setMemorySize(512); //512MiB resource.setVirtualCores(2); Resource largerResource = Records.newRecord(Resource.class); - largerResource.setMemory(1024); + largerResource.setMemorySize(1024); largerResource.setVirtualCores(2); Resource smallerResource = Records.newRecord(Resource.class); - smallerResource.setMemory(256); + smallerResource.setMemorySize(256); smallerResource.setVirtualCores(1); metrics.addResource(total); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 98a5e472338..0d5662b3cef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -331,7 +331,7 @@ public class ResourceTrackerService extends AbstractService implements String nid = nodeId.toString(); if (nodes != null && Arrays.asList(nodes).contains(nid)) { - capability.setMemory(this.drConf.getMemoryPerNode(nid)); + capability.setMemorySize(this.drConf.getMemoryPerNode(nid)); capability.setVirtualCores(this.drConf.getVcoresPerNode(nid)); if (LOG.isDebugEnabled()) { LOG.debug("Resource for node: " + nid + " is adjusted to " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index d708a468744..cfe2a7a9ce0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -706,7 +706,7 @@ public abstract class AbstractYarnScheduler int nodeMemory = (int)totalResource.getMemorySize(); if (nodeMemory > maxNodeMemory) { maxNodeMemory = nodeMemory; - maximumAllocation.setMemory(Math.min( + maximumAllocation.setMemorySize(Math.min( configuredMaximumAllocation.getMemorySize(), maxNodeMemory)); } int nodeVCores = totalResource.getVirtualCores(); @@ -738,9 +738,9 @@ public abstract class AbstractYarnScheduler } } if (maxNodeMemory == -1) { // no nodes - maximumAllocation.setMemory(configuredMaximumAllocation.getMemorySize()); + maximumAllocation.setMemorySize(configuredMaximumAllocation.getMemorySize()); } else { - maximumAllocation.setMemory( + maximumAllocation.setMemorySize( Math.min(configuredMaximumAllocation.getMemorySize(), maxNodeMemory)); } if (maxNodeVCores == -1) { // no nodes diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 69c3e8b8773..4e364f70f52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -61,7 +61,7 @@ public class QueueMetrics implements MetricsSource { @Metric("# of apps failed") MutableCounterInt appsFailed; @Metric("Allocated memory in MB") MutableGaugeLong allocatedMB; - @Metric("Allocated CPU in virtual cores") MutableGaugeLong allocatedVCores; + @Metric("Allocated CPU in virtual cores") MutableGaugeInt allocatedVCores; @Metric("# of allocated containers") MutableGaugeInt allocatedContainers; @Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated; @Metric("Aggregate # of allocated node-local containers") @@ -72,12 +72,12 @@ public class QueueMetrics implements MetricsSource { MutableCounterLong aggregateOffSwitchContainersAllocated; @Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased; @Metric("Available memory in MB") MutableGaugeLong availableMB; - @Metric("Available CPU in virtual cores") MutableGaugeLong availableVCores; + @Metric("Available CPU in virtual cores") MutableGaugeInt availableVCores; @Metric("Pending memory allocation in MB") MutableGaugeLong pendingMB; - @Metric("Pending CPU allocation in virtual cores") MutableGaugeLong pendingVCores; + @Metric("Pending CPU allocation in virtual cores") MutableGaugeInt pendingVCores; @Metric("# of pending containers") MutableGaugeInt pendingContainers; @Metric("# of reserved memory in MB") MutableGaugeLong reservedMB; - @Metric("Reserved CPU in virtual cores") MutableGaugeLong reservedVCores; + @Metric("Reserved CPU in virtual cores") MutableGaugeInt reservedVCores; @Metric("# of reserved containers") MutableGaugeInt reservedContainers; @Metric("# of active users") MutableGaugeInt activeUsers; @Metric("# of active applications") MutableGaugeInt activeApplications; @@ -364,7 +364,7 @@ public class QueueMetrics implements MetricsSource { private void _incrPendingResources(int containers, Resource res) { pendingContainers.incr(containers); pendingMB.incr(res.getMemorySize() * containers); - pendingVCores.incr(res.getVirtualCoresSize() * containers); + pendingVCores.incr(res.getVirtualCores() * containers); } public void decrPendingResources(String user, int containers, Resource res) { @@ -381,7 +381,7 @@ public class QueueMetrics implements MetricsSource { private void _decrPendingResources(int containers, Resource res) { pendingContainers.decr(containers); pendingMB.decr(res.getMemorySize() * containers); - pendingVCores.decr(res.getVirtualCoresSize() * containers); + pendingVCores.decr(res.getVirtualCores() * containers); } public void incrNodeTypeAggregations(String user, NodeType type) { @@ -409,7 +409,7 @@ public class QueueMetrics implements MetricsSource { aggregateContainersAllocated.incr(containers); allocatedMB.incr(res.getMemorySize() * containers); - allocatedVCores.incr(res.getVirtualCoresSize() * containers); + allocatedVCores.incr(res.getVirtualCores() * containers); if (decrPending) { _decrPendingResources(containers, res); } @@ -448,7 +448,7 @@ public class QueueMetrics implements MetricsSource { allocatedContainers.decr(containers); aggregateContainersReleased.incr(containers); allocatedMB.decr(res.getMemorySize() * containers); - allocatedVCores.decr(res.getVirtualCoresSize() * containers); + allocatedVCores.decr(res.getVirtualCores() * containers); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.releaseResources(user, containers, res); @@ -561,14 +561,15 @@ public class QueueMetrics implements MetricsSource { } public Resource getAllocatedResources() { - return BuilderUtils.newResource(allocatedMB.value(), allocatedVCores.value()); + return BuilderUtils.newResource(allocatedMB.value(), + (int) allocatedVCores.value()); } public long getAllocatedMB() { return allocatedMB.value(); } - public long getAllocatedVirtualCores() { + public int getAllocatedVirtualCores() { return allocatedVCores.value(); } @@ -580,7 +581,7 @@ public class QueueMetrics implements MetricsSource { return availableMB.value(); } - public long getAvailableVirtualCores() { + public int getAvailableVirtualCores() { return availableVCores.value(); } @@ -588,7 +589,7 @@ public class QueueMetrics implements MetricsSource { return pendingMB.value(); } - public long getPendingVirtualCores() { + public int getPendingVirtualCores() { return pendingVCores.value(); } @@ -600,7 +601,7 @@ public class QueueMetrics implements MetricsSource { return reservedMB.value(); } - public long getReservedVirtualCores() { + public int getReservedVirtualCores() { return reservedVCores.value(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 8929512df47..9b68d96d607 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -443,7 +443,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { public synchronized Resource getHeadroom() { // Corner case to deal with applications being slightly over-limit if (resourceLimit.getMemorySize() < 0) { - resourceLimit.setMemory(0); + resourceLimit.setMemorySize(0); } return resourceLimit; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java index 49892f73528..5605f18b773 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java @@ -66,7 +66,7 @@ public class CapacityHeadroomProvider { } // Corner case to deal with applications being slightly over-limit if (headroom.getMemorySize() < 0) { - headroom.setMemory(0); + headroom.setMemorySize(0); } return headroom; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 1daea0a54e2..25f40952a04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -429,7 +429,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt } public void clearPreemptedResources() { - preemptedResources.setMemory(0); + preemptedResources.setMemorySize(0); preemptedResources.setVirtualCores(0); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java index 5f500cd9495..440c73cefdd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java @@ -266,10 +266,10 @@ public class ComputeFairShares { private static void setResourceValue(long val, Resource resource, ResourceType type) { switch (type) { case MEMORY: - resource.setMemory(val); + resource.setMemorySize(val); break; case CPU: - resource.setVirtualCores(val); + resource.setVirtualCores((int)val); break; default: throw new IllegalArgumentException("Invalid resource"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index 76d43e0c4fe..d198bb97afd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.api.records.Resource; @XmlAccessorType(XmlAccessType.FIELD) public class ResourceInfo { long memory; - long vCores; + int vCores; public ResourceInfo() { } @@ -42,7 +42,7 @@ public class ResourceInfo { return memory; } - public long getvCores() { + public int getvCores() { return vCores; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index c325a657020..0bd8df6d448 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -220,7 +220,7 @@ public class MockAM { pri.setPriority(priority); req.setPriority(pri); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(memory); + capability.setMemorySize(memory); req.setCapability(capability); if (labelExpression != null) { req.setNodeLabelExpression(labelExpression); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 5ef24be01ed..83e901db943 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -83,19 +83,19 @@ public class MockNodes { public static Resource newResource(int mem) { Resource rs = recordFactory.newRecordInstance(Resource.class); - rs.setMemory(mem); + rs.setMemorySize(mem); return rs; } public static Resource newUsedResource(Resource total) { Resource rs = recordFactory.newRecordInstance(Resource.class); - rs.setMemory((int)(Math.random() * total.getMemorySize())); + rs.setMemorySize((int)(Math.random() * total.getMemorySize())); return rs; } public static Resource newAvailResource(Resource total, Resource used) { Resource rs = recordFactory.newRecordInstance(Resource.class); - rs.setMemory(total.getMemorySize() - used.getMemorySize()); + rs.setMemorySize(total.getMemorySize() - used.getMemorySize()); return rs; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index ddc75a24720..a5d62a6e5f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -382,7 +382,7 @@ public class MockRM extends ResourceManager { Map acls, String queue, String amLabel) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, name, user, acls, false, queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, @@ -435,7 +435,7 @@ public class MockRM extends ResourceManager { int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); return submitApp(resource, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, false, null, 0, null, true, Priority.newInstance(0)); @@ -444,7 +444,7 @@ public class MockRM extends ResourceManager { public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, @@ -459,7 +459,7 @@ public class MockRM extends ResourceManager { boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, ApplicationId applicationId) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, @@ -469,7 +469,7 @@ public class MockRM extends ResourceManager { public RMApp submitApp(int masterMemory, LogAggregationContext logAggregationContext) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index f3a7ba52400..87e268b84f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -820,21 +820,21 @@ public class TestResourceTrackerService extends NodeLabelTestBase { resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction()); - capability.setMemory(2048); + capability.setMemorySize(2048); capability.setVirtualCores(1); req.setResource(capability); RegisterNodeManagerResponse response2 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction()); - capability.setMemory(1024); + capability.setMemorySize(1024); capability.setVirtualCores(4); req.setResource(capability); RegisterNodeManagerResponse response3 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction()); - capability.setMemory(2048); + capability.setMemorySize(2048); capability.setVirtualCores(4); req.setResource(capability); RegisterNodeManagerResponse response4 = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index dbf2163fee1..63070873919 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -1318,7 +1318,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase // submit app with keepContainersAcrossApplicationAttempts true Resource resource = Records.newRecord(Resource.class); - resource.setMemory(200); + resource.setMemorySize(200); RMApp app0 = rm1.submitApp(resource, "", UserGroupInformation .getCurrentUser().getShortUserName(), null, false, null, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java index 4cb235fa323..d383bac2189 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java @@ -127,7 +127,7 @@ public class TestReservationSystemUtil { public Resource createResource(int memory, int vCores) { Resource resource = new ResourcePBImpl(); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(vCores); return resource; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index 62b1082f534..cb1104bffb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -402,9 +402,9 @@ public class TestQueueMetrics { } public static void checkResources(MetricsSource source, long allocatedMB, - long allocatedCores, int allocCtnrs, long aggreAllocCtnrs, - long aggreReleasedCtnrs, long availableMB, long availableCores, long pendingMB, - long pendingCores, int pendingCtnrs, long reservedMB, long reservedCores, + int allocatedCores, int allocCtnrs, long aggreAllocCtnrs, + long aggreReleasedCtnrs, long availableMB, int availableCores, long pendingMB, + int pendingCores, int pendingCtnrs, long reservedMB, int reservedCores, int reservedCtnrs) { MetricsRecordBuilder rb = getMetrics(source); assertGauge("AllocatedMB", allocatedMB, rb); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index a811ca3ac51..afb04d22443 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -1143,7 +1143,7 @@ public class TestDelegationTokenRenewer { // submit app1 with a token, set cancelTokenWhenComplete to false; Resource resource = Records.newRecord(Resource.class); - resource.setMemory(200); + resource.setMemorySize(200); RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, false, null); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); @@ -1208,7 +1208,7 @@ public class TestDelegationTokenRenewer { Assert.assertFalse(Renewer.cancelled); Resource resource = Records.newRecord(Resource.class); - resource.setMemory(200); + resource.setMemorySize(200); RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, true, null);