Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-6923. Optimize MapReduce Shuffle I/O for small partitions. Contributed by Robert Schmidtke. 

(cherry picked from commit ac7d0604bc)
This commit is contained in:
Ravi Prakash 2017-08-09 15:39:52 -07:00
parent 306abf4980
commit c556cdaaf4
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
View File

@ -111,7 +111,10 @@ public class FadvisedFileRegion extends DefaultFileRegion {
long trans = actualCount;
int readSize;
ByteBuffer byteBuffer = ByteBuffer.allocate(this.shuffleBufferSize);
ByteBuffer byteBuffer = ByteBuffer.allocate(
Math.min(
this.shuffleBufferSize,
trans > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) trans));
while(trans > 0L &&
(readSize = fileChannel.read(byteBuffer, this.position+position)) > 0) {