diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index c90960ec839..8e766887c9a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -58,6 +58,9 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-4716. TestHsWebServicesJobsQuery.testJobsQueryStateInvalid fails with jdk7. (tgraves via tucu) + MAPREDUCE-5028. Maps fail when io.sort.mb is set to high value. + (kkambatl via tucu) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java index cd4ba51c31c..496ca2e6565 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java @@ -1165,8 +1165,9 @@ public class MapTask extends Task { equator = pos; // set index prior to first entry, aligned at meta boundary final int aligned = pos - (pos % METASIZE); - kvindex = - ((aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4; + // Cast one of the operands to long to ensure large values don't cause int + // overflow + kvindex = (int) (((long) aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4; if (LOG.isInfoEnabled()) { LOG.info("(EQUATOR) " + pos + " kvi " + kvindex + "(" + (kvindex * 4) + ")"); @@ -1183,8 +1184,9 @@ public class MapTask extends Task { bufstart = bufend = e; final int aligned = e - (e % METASIZE); // set start/end to point to first meta record - kvstart = kvend = - ((aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4; + // Cast one of the operands to long to ensure large values don't cause int + // overflow + kvstart = kvend = (int) (((long) aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4; if (LOG.isInfoEnabled()) { LOG.info("(RESET) equator " + e + " kv " + kvstart + "(" + (kvstart * 4) + ")" + " kvi " + kvindex + "(" + (kvindex * 4) + ")"); @@ -1748,7 +1750,7 @@ public class MapTask extends Task { this.start = 0; } - super.reset(this.buffer, this.start, this.length); + super.reset(this.buffer, this.start, this.length - this.start); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/ReduceContextImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/ReduceContextImpl.java index 7ad08e956d9..8736de52f8c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/ReduceContextImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/ReduceContextImpl.java @@ -205,7 +205,8 @@ public class ReduceContextImpl if (backupStore.hasNext()) { backupStore.next(); DataInputBuffer next = backupStore.nextValue(); - buffer.reset(next.getData(), next.getPosition(), next.getLength()); + buffer.reset(next.getData(), next.getPosition(), next.getLength() + - next.getPosition()); value = valueDeserializer.deserialize(value); return value; } else { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java index 543ff3f9cc7..6def88b97b4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java @@ -49,14 +49,14 @@ public class InMemoryReader extends Reader { buffer = data; bufferSize = (int)fileLength; - memDataIn.reset(buffer, start, length); + memDataIn.reset(buffer, start, length - start); this.start = start; this.length = length; } @Override public void reset(int offset) { - memDataIn.reset(buffer, start + offset, length); + memDataIn.reset(buffer, start + offset, length - start - offset); bytesRead = offset; eof = false; }