Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-6068. Illegal progress value warnings in map tasks. Contributed by Binglin Chang.

This commit is contained in:
Junping Du 2016-01-11 06:12:19 -08:00
parent 76e7270851
commit 8c1adeaa26
2 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-mapreduce-project/CHANGES.txt
View File

@ -677,8 +677,8 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6550. archive-logs tool changes log ownership to the Yarn MAPREDUCE-6550. archive-logs tool changes log ownership to the Yarn
user when using DefaultContainerExecutor (rkanter) user when using DefaultContainerExecutor (rkanter)
MAPREDUCE-6419. JobHistoryServer doesn't sort properly based on Job ID when Job id's MAPREDUCE-6419. JobHistoryServer doesn't sort properly based on Job ID when Job id's
exceed 9999. (Mohammad Shahid Khan via rohithsharmaks) exceed 9999. (Mohammad Shahid Khan via rohithsharmaks)
MAPREDUCE-6589. TestTaskLog outputs a log under directory other than MAPREDUCE-6589. TestTaskLog outputs a log under directory other than
target/test-dir. (aajisaka) target/test-dir. (aajisaka)
@ -686,6 +686,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6593. TestJobHistoryEventHandler.testTimelineEventHandling fails MAPREDUCE-6593. TestJobHistoryEventHandler.testTimelineEventHandling fails
on trunk because of NPE. (Naganarasimha G R via aajisaka) on trunk because of NPE. (Naganarasimha G R via aajisaka)
MAPREDUCE-6068. Illegal progress value warnings in map tasks. (Binglin Chang
via junping_du)
Release 2.7.3 - UNRELEASED Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java
View File

@ -519,7 +519,7 @@ private void adjustPriorityQueue(Segment<K, V> reader) throws IOException{
boolean hasNext = reader.nextRawKey(); boolean hasNext = reader.nextRawKey();
long endPos = reader.getReader().bytesRead; long endPos = reader.getReader().bytesRead;
totalBytesProcessed += endPos - startPos; totalBytesProcessed += endPos - startPos;
mergeProgress.set(totalBytesProcessed * progPerByte); mergeProgress.set(Math.min(1.0f, totalBytesProcessed * progPerByte));
if (hasNext) { if (hasNext) {
adjustTop(); adjustTop();
} else { } else {
@ -571,7 +571,7 @@ public boolean next() throws IOException {
} }
long endPos = minSegment.getReader().bytesRead; long endPos = minSegment.getReader().bytesRead;
totalBytesProcessed += endPos - startPos; totalBytesProcessed += endPos - startPos;
mergeProgress.set(totalBytesProcessed * progPerByte); mergeProgress.set(Math.min(1.0f, totalBytesProcessed * progPerByte));
return true; return true;
} }
@ -698,7 +698,7 @@ RawKeyValueIterator merge(Class<K> keyClass, Class<V> valueClass,
totalBytesProcessed += startBytes; totalBytesProcessed += startBytes;
if (totalBytes != 0) if (totalBytes != 0)
mergeProgress.set(totalBytesProcessed * progPerByte); mergeProgress.set(Math.min(1.0f, totalBytesProcessed * progPerByte));
else else
mergeProgress.set(1.0f); // Last pass and no segments left - we're done mergeProgress.set(1.0f); // Last pass and no segments left - we're done