Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-382. SchedulerUtils improve way normalizeRequest sets the resource capabilities (Zhijie Shen via bikas) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1463653 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Bikas Saha 2013-04-02 18:07:07 +00:00
parent 9c26c28c87
commit d40b154d27
5 changed files with 18 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -176,6 +176,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-516. Fix failure in TestContainerLocalizer caused by HADOOP-9357.
(Andrew Wang via vinodkv)
YARN-382. SchedulerUtils improve way normalizeRequest sets the resource
capabilities. (Zhijie Shen via bikas)
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -771,6 +771,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
// Set the masterContainer
appAttempt.setMasterContainer(amContainerAllocation.getContainers().get(
0));
// Updating CLC's resource is no longer necessary once YARN-486 is
// completed, because nothing from Container to CLC will be copied into
// CLC then.
appAttempt.getSubmissionContext().getAMContainerSpec().setResource(
appAttempt.getMasterContainer().getResource());
RMStateStore store = appAttempt.rmContext.getStateStore();
appAttempt.storeAttempt(store);

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
View File

@ -108,8 +108,7 @@ public class SchedulerUtils {
Resource normalized =
Resources.normalize(
resourceCalculator, ask.getCapability(), minimumResource);
ask.getCapability().setMemory(normalized.getMemory());
ask.getCapability().setVirtualCores(normalized.getVirtualCores());
ask.setCapability(normalized);
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
View File

@ -202,9 +202,10 @@ public class TestRMAppAttemptTransitions {
submissionContext = mock(ApplicationSubmissionContext.class);
when(submissionContext.getUser()).thenReturn(user);
when(submissionContext.getQueue()).thenReturn(queue);
ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);
Resource resource = mock(Resource.class);
when(amContainerSpec.getResource()).thenReturn(resource);
Resource resource = BuilderUtils.newResource(1536, 1);
ContainerLaunchContext amContainerSpec =
BuilderUtils.newContainerLaunchContext(null, user, resource, null, null,
null, null, null, null);
when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
unmanagedAM = false;
@ -469,8 +470,10 @@ public class TestRMAppAttemptTransitions {
// Mock the allocation of AM container
Container container = mock(Container.class);
Resource resource = BuilderUtils.newResource(2048, 1);
when(container.getId()).thenReturn(
BuilderUtils.newContainerId(applicationAttempt.getAppAttemptId(), 1));
when(container.getResource()).thenReturn(resource);
Allocation allocation = mock(Allocation.class);
when(allocation.getContainers()).
thenReturn(Collections.singletonList(container));
@ -491,6 +494,9 @@ public class TestRMAppAttemptTransitions {
applicationAttempt.handle(
new RMAppAttemptStoredEvent(
applicationAttempt.getAppAttemptId(), null));
assertEquals(resource,
applicationAttempt.getSubmissionContext()
.getAMContainerSpec().getResource());
testAppAttemptAllocatedState(container);

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
View File

@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@ -43,35 +42,23 @@ public class TestSchedulerUtils {
// case negative memory
ask.setCapability(Resources.createResource(-1024));
Resource before = ask.getCapability();
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource);
Resource after = ask.getCapability();
assertEquals(minMemory, ask.getCapability().getMemory());
assertTrue(before == after);
// case zero memory
ask.setCapability(Resources.createResource(0));
before = ask.getCapability();
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource);
after = ask.getCapability();
assertEquals(minMemory, ask.getCapability().getMemory());
assertTrue(before == after);
// case memory is a multiple of minMemory
ask.setCapability(Resources.createResource(2 * minMemory));
before = ask.getCapability();
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource);
after = ask.getCapability();
assertEquals(2 * minMemory, ask.getCapability().getMemory());
assertTrue(before == after);
// case memory is not a multiple of minMemory
ask.setCapability(Resources.createResource(minMemory + 10));
before = ask.getCapability();
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource);
after = ask.getCapability();
assertEquals(2 * minMemory, ask.getCapability().getMemory());
assertTrue(before == after);
}
@ -86,33 +73,24 @@ public class TestSchedulerUtils {
// case negative memory/vcores
ask.setCapability(Resources.createResource(-1024, -1));
Resource before = ask.getCapability();
SchedulerUtils.normalizeRequest(
ask, resourceCalculator, clusterResource, minResource);
Resource after = ask.getCapability();
assertEquals(minResource, ask.getCapability());
assertTrue(before == after);
// case zero memory/vcores
ask.setCapability(Resources.createResource(0, 0));
before = ask.getCapability();
SchedulerUtils.normalizeRequest(
ask, resourceCalculator, clusterResource, minResource);
after = ask.getCapability();
assertEquals(minResource, ask.getCapability());
assertEquals(1, ask.getCapability().getVirtualCores());
assertEquals(1024, ask.getCapability().getMemory());
assertTrue(before == after);
// case non-zero memory & zero cores
ask.setCapability(Resources.createResource(1536, 0));
before = ask.getCapability();
SchedulerUtils.normalizeRequest(
ask, resourceCalculator, clusterResource, minResource);
after = ask.getCapability();
assertEquals(Resources.createResource(2048, 1), ask.getCapability());
assertEquals(1, ask.getCapability().getVirtualCores());
assertEquals(2048, ask.getCapability().getMemory());
assertTrue(before == after);
}
}