Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-5310. MRAM should not normalize allocation request capabilities. (tucu) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1493207 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2013-06-14 19:06:50 +00:00
parent 334de8d211
commit 079ed1c9e1
4 changed files with 3 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -557,6 +557,9 @@ Release 2.1.0-beta - UNRELEASED
MAPREDUCE-5319. Set user.name in job.xml. (Xuan Gong via acmurthy) MAPREDUCE-5319. Set user.name in job.xml. (Xuan Gong via acmurthy)
MAPREDUCE-5310. MRAM should not normalize allocation request capabilities.
(tucu)
Release 2.0.5-alpha - 06/06/2013 Release 2.0.5-alpha - 06/06/2013
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

9
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
View File

@ -72,7 +72,6 @@ public abstract class RMCommunicator extends AbstractService
protected AMRMProtocol scheduler; protected AMRMProtocol scheduler;
private final ClientService clientService; private final ClientService clientService;
protected int lastResponseID; protected int lastResponseID;
private Resource minContainerCapability;
private Resource maxContainerCapability; private Resource maxContainerCapability;
protected Map<ApplicationAccessType, String> applicationACLs; protected Map<ApplicationAccessType, String> applicationACLs;
private volatile long lastHeartbeatTime; private volatile long lastHeartbeatTime;
@ -152,14 +151,10 @@ public abstract class RMCommunicator extends AbstractService
} }
RegisterApplicationMasterResponse response = RegisterApplicationMasterResponse response =
scheduler.registerApplicationMaster(request); scheduler.registerApplicationMaster(request);
minContainerCapability = response.getMinimumResourceCapability();
maxContainerCapability = response.getMaximumResourceCapability(); maxContainerCapability = response.getMaximumResourceCapability();
this.context.getClusterInfo().setMinContainerCapability(
minContainerCapability);
this.context.getClusterInfo().setMaxContainerCapability( this.context.getClusterInfo().setMaxContainerCapability(
maxContainerCapability); maxContainerCapability);
this.applicationACLs = response.getApplicationACLs(); this.applicationACLs = response.getApplicationACLs();
LOG.info("minContainerCapability: " + minContainerCapability.getMemory());
LOG.info("maxContainerCapability: " + maxContainerCapability.getMemory()); LOG.info("maxContainerCapability: " + maxContainerCapability.getMemory());
} catch (Exception are) { } catch (Exception are) {
LOG.error("Exception while registering", are); LOG.error("Exception while registering", are);
@ -202,10 +197,6 @@ public abstract class RMCommunicator extends AbstractService
} }
} }
protected Resource getMinContainerCapability() {
return minContainerCapability;
}
protected Resource getMaxContainerCapability() { protected Resource getMaxContainerCapability() {
return maxContainerCapability; return maxContainerCapability;
} }

7
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
View File

@ -291,9 +291,6 @@ public class RMContainerAllocator extends RMContainerRequestor
if (reqEvent.getAttemptID().getTaskId().getTaskType().equals(TaskType.MAP)) { if (reqEvent.getAttemptID().getTaskId().getTaskType().equals(TaskType.MAP)) {
if (mapResourceReqt == 0) { if (mapResourceReqt == 0) {
mapResourceReqt = reqEvent.getCapability().getMemory(); mapResourceReqt = reqEvent.getCapability().getMemory();
int minSlotMemSize = getMinContainerCapability().getMemory();
mapResourceReqt = (int) Math.ceil((float) mapResourceReqt/minSlotMemSize)
* minSlotMemSize;
eventHandler.handle(new JobHistoryEvent(jobId, eventHandler.handle(new JobHistoryEvent(jobId,
new NormalizedResourceEvent(org.apache.hadoop.mapreduce.TaskType.MAP, new NormalizedResourceEvent(org.apache.hadoop.mapreduce.TaskType.MAP,
mapResourceReqt))); mapResourceReqt)));
@ -314,10 +311,6 @@ public class RMContainerAllocator extends RMContainerRequestor
} else { } else {
if (reduceResourceReqt == 0) { if (reduceResourceReqt == 0) {
reduceResourceReqt = reqEvent.getCapability().getMemory(); reduceResourceReqt = reqEvent.getCapability().getMemory();
int minSlotMemSize = getMinContainerCapability().getMemory();
//round off on slotsize
reduceResourceReqt = (int) Math.ceil((float)
reduceResourceReqt/minSlotMemSize) * minSlotMemSize;
eventHandler.handle(new JobHistoryEvent(jobId, eventHandler.handle(new JobHistoryEvent(jobId,
new NormalizedResourceEvent( new NormalizedResourceEvent(
org.apache.hadoop.mapreduce.TaskType.REDUCE, org.apache.hadoop.mapreduce.TaskType.REDUCE,

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
View File

@ -1401,11 +1401,6 @@ public class TestRMContainerAllocator {
protected void unregister() { protected void unregister() {
} }
@Override
protected Resource getMinContainerCapability() {
return Resource.newInstance(1024, 1);
}
@Override @Override
protected Resource getMaxContainerCapability() { protected Resource getMaxContainerCapability() {
return Resource.newInstance(10240, 1); return Resource.newInstance(10240, 1);