Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-5280. Bring back removed constructor and a method in mapreduce ClusterMetrics for binary compatibility with 1.x APIs. Contributed by Mayank Bansal. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1488458 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-06-01 03:57:40 +00:00
parent b8e9f06774
commit a77030abdc
5 changed files with 70 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/CHANGES.txt
View File

@ -263,6 +263,10 @@ Release 2.0.5-beta - UNRELEASED
mapreduce CombineFileRecordReader for binary compatibility with 1.x APIs.
(Mayank Bansal via vinodkv)
MAPREDUCE-5280. Bring back removed constructor and a method in mapreduce
ClusterMetrics for binary compatibility with 1.x APIs. (Mayank Bansal via
vinodkv)
OPTIMIZATIONS
MAPREDUCE-4974. Optimising the LineRecordReader initialize() method

33
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java
View File

@ -188,6 +188,7 @@ public class ClusterStatus implements Writable {
private JobTrackerStatus status;
private Collection<BlackListInfo> blacklistedTrackersInfo =
new ArrayList<BlackListInfo>();
private int grayListedTrackers;
ClusterStatus() {}
@ -223,9 +224,30 @@ public class ClusterStatus implements Writable {
* @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
* @param numDecommissionedNodes number of decommission trackers
*/
ClusterStatus(int trackers, int blacklists, long ttExpiryInterval,
int maps, int reduces, int maxMaps, int maxReduces,
JobTrackerStatus status, int numDecommissionedNodes) {
ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, int maps,
int reduces, int maxMaps, int maxReduces, JobTrackerStatus status,
int numDecommissionedNodes) {
this(trackers, blacklists, ttExpiryInterval, maps, reduces, maxMaps,
maxReduces, status, numDecommissionedNodes, 0);
}
/**
* Construct a new cluster status.
*
* @param trackers no. of tasktrackers in the cluster
* @param blacklists no of blacklisted task trackers in the cluster
* @param ttExpiryInterval the tasktracker expiry interval
* @param maps no. of currently running map-tasks in the cluster
* @param reduces no. of currently running reduce-tasks in the cluster
* @param maxMaps the maximum no. of map tasks in the cluster
* @param maxReduces the maximum no. of reduce tasks in the cluster
* @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
* @param numDecommissionedNodes number of decommission trackers
* @param numGrayListedTrackers number of graylisted trackers
*/
ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, int maps,
int reduces, int maxMaps, int maxReduces, JobTrackerStatus status,
int numDecommissionedNodes, int numGrayListedTrackers) {
numActiveTrackers = trackers;
numBlacklistedTrackers = blacklists;
this.numExcludedNodes = numDecommissionedNodes;
@ -235,6 +257,7 @@ public class ClusterStatus implements Writable {
max_map_tasks = maxMaps;
max_reduce_tasks = maxReduces;
this.status = status;
this.grayListedTrackers = numGrayListedTrackers;
}
/**
@ -339,7 +362,7 @@ public class ClusterStatus implements Writable {
*/
@Deprecated
public int getGraylistedTrackers() {
return 0;
return grayListedTrackers;
}
/**
@ -480,6 +503,7 @@ public class ClusterStatus implements Writable {
out.writeInt(max_map_tasks);
out.writeInt(max_reduce_tasks);
WritableUtils.writeEnum(out, status);
out.writeInt(grayListedTrackers);
}
public void readFields(DataInput in) throws IOException {
@ -507,5 +531,6 @@ public class ClusterStatus implements Writable {
max_map_tasks = in.readInt();
max_reduce_tasks = in.readInt();
status = WritableUtils.readEnum(in, JobTrackerStatus.class);
grayListedTrackers = in.readInt();
}
}

19
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
View File

@ -711,19 +711,18 @@ public class JobClient extends CLI {
public ClusterStatus getClusterStatus() throws IOException {
try {
return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
public ClusterStatus run() throws IOException, InterruptedException {
public ClusterStatus run() throws IOException, InterruptedException {
ClusterMetrics metrics = cluster.getClusterStatus();
return new ClusterStatus(metrics.getTaskTrackerCount(),
metrics.getBlackListedTaskTrackerCount(), cluster.getTaskTrackerExpiryInterval(),
metrics.getOccupiedMapSlots(),
metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
metrics.getReduceSlotCapacity(),
cluster.getJobTrackerStatus(),
metrics.getDecommissionedTaskTrackerCount());
return new ClusterStatus(metrics.getTaskTrackerCount(), metrics
.getBlackListedTaskTrackerCount(), cluster
.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
metrics.getReduceSlotCapacity(), cluster.getJobTrackerStatus(),
metrics.getDecommissionedTaskTrackerCount(), metrics
.getGrayListedTaskTrackerCount());
}
});
}
catch (InterruptedException ie) {
} catch (InterruptedException ie) {
throw new IOException(ie);
}
}

32
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/ClusterMetrics.java
View File

@ -69,18 +69,28 @@ public class ClusterMetrics implements Writable {
private int totalJobSubmissions;
private int numTrackers;
private int numBlacklistedTrackers;
private int numGraylistedTrackers;
private int numDecommissionedTrackers;
public ClusterMetrics() {
}
public ClusterMetrics(int runningMaps, int runningReduces,
int occupiedMapSlots, int occupiedReduceSlots,
int reservedMapSlots, int reservedReduceSlots,
int mapSlots, int reduceSlots,
int totalJobSubmissions,
int numTrackers, int numBlacklistedTrackers,
int occupiedMapSlots, int occupiedReduceSlots, int reservedMapSlots,
int reservedReduceSlots, int mapSlots, int reduceSlots,
int totalJobSubmissions, int numTrackers, int numBlacklistedTrackers,
int numDecommissionedNodes) {
this(runningMaps, runningReduces, occupiedMapSlots, occupiedReduceSlots,
reservedMapSlots, reservedReduceSlots, mapSlots, reduceSlots,
totalJobSubmissions, numTrackers, numBlacklistedTrackers, 0,
numDecommissionedNodes);
}
public ClusterMetrics(int runningMaps, int runningReduces,
int occupiedMapSlots, int occupiedReduceSlots, int reservedMapSlots,
int reservedReduceSlots, int mapSlots, int reduceSlots,
int totalJobSubmissions, int numTrackers, int numBlacklistedTrackers,
int numGraylistedTrackers, int numDecommissionedNodes) {
this.runningMaps = runningMaps;
this.runningReduces = runningReduces;
this.occupiedMapSlots = occupiedMapSlots;
@ -92,6 +102,7 @@ public class ClusterMetrics implements Writable {
this.totalJobSubmissions = totalJobSubmissions;
this.numTrackers = numTrackers;
this.numBlacklistedTrackers = numBlacklistedTrackers;
this.numGraylistedTrackers = numGraylistedTrackers;
this.numDecommissionedTrackers = numDecommissionedNodes;
}
@ -194,6 +205,15 @@ public class ClusterMetrics implements Writable {
return numBlacklistedTrackers;
}
/**
* Get the number of graylisted trackers in the cluster.
*
* @return graylisted tracker count
*/
public int getGrayListedTaskTrackerCount() {
return numGraylistedTrackers;
}
/**
* Get the number of decommissioned trackers in the cluster.
*
@ -216,6 +236,7 @@ public class ClusterMetrics implements Writable {
totalJobSubmissions = in.readInt();
numTrackers = in.readInt();
numBlacklistedTrackers = in.readInt();
numGraylistedTrackers = in.readInt();
numDecommissionedTrackers = in.readInt();
}
@ -232,6 +253,7 @@ public class ClusterMetrics implements Writable {
out.writeInt(totalJobSubmissions);
out.writeInt(numTrackers);
out.writeInt(numBlacklistedTrackers);
out.writeInt(numGraylistedTrackers);
out.writeInt(numDecommissionedTrackers);
}

1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java
View File

@ -206,6 +206,7 @@ public class TestNetworkedJob {
assertEquals(status.getTaskTrackers(), 2);
assertEquals(status.getTTExpiryInterval(), 0);
assertEquals(status.getJobTrackerStatus(), JobTrackerStatus.RUNNING);
assertEquals(status.getGraylistedTrackers(), 0);
// test read and write
ByteArrayOutputStream dataOut = new ByteArrayOutputStream();