diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 08c7ae8f30c..08a8bf4d796 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -978,6 +978,9 @@ Release 2.8.0 - UNRELEASED HADOOP-10555. Add offset support to MurmurHash. (Sergey Shelukhin via wheat9) + HADOOP-10465. Fix use of generics within SortedMapWritable. + (Bertrand Dechoux via wheat9) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java index 3b51fc25ffc..3ed2a1dff7d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java @@ -36,15 +36,15 @@ import org.apache.hadoop.util.ReflectionUtils; */ @InterfaceAudience.Public @InterfaceStability.Stable -public class SortedMapWritable extends AbstractMapWritable - implements SortedMap { +public class SortedMapWritable> extends AbstractMapWritable + implements SortedMap { - private SortedMap instance; + private SortedMap instance; /** default constructor. */ public SortedMapWritable() { super(); - this.instance = new TreeMap(); + this.instance = new TreeMap(); } /** @@ -52,45 +52,39 @@ public class SortedMapWritable extends AbstractMapWritable * * @param other the map to copy from */ - public SortedMapWritable(SortedMapWritable other) { + public SortedMapWritable(SortedMapWritable other) { this(); copy(other); } @Override - public Comparator comparator() { + public Comparator comparator() { // Returning null means we use the natural ordering of the keys return null; } @Override - public WritableComparable firstKey() { + public K firstKey() { return instance.firstKey(); } @Override - public SortedMap - headMap(WritableComparable toKey) { - + public SortedMap headMap(K toKey) { return instance.headMap(toKey); } @Override - public WritableComparable lastKey() { + public K lastKey() { return instance.lastKey(); } @Override - public SortedMap - subMap(WritableComparable fromKey, WritableComparable toKey) { - + public SortedMap subMap(K fromKey, K toKey) { return instance.subMap(fromKey, toKey); } @Override - public SortedMap - tailMap(WritableComparable fromKey) { - + public SortedMap tailMap(K fromKey) { return instance.tailMap(fromKey); } @@ -110,7 +104,7 @@ public class SortedMapWritable extends AbstractMapWritable } @Override - public Set> entrySet() { + public Set> entrySet() { return instance.entrySet(); } @@ -125,22 +119,21 @@ public class SortedMapWritable extends AbstractMapWritable } @Override - public Set keySet() { + public Set keySet() { return instance.keySet(); } @Override - public Writable put(WritableComparable key, Writable value) { + public Writable put(K key, Writable value) { addToMap(key.getClass()); addToMap(value.getClass()); return instance.put(key, value); } @Override - public void putAll(Map t) { - for (Map.Entry e: + public void putAll(Map t) { + for (Map.Entry e: t.entrySet()) { - put(e.getKey(), e.getValue()); } } @@ -172,8 +165,8 @@ public class SortedMapWritable extends AbstractMapWritable // Then read each key/value pair for (int i = 0; i < entries; i++) { - WritableComparable key = - (WritableComparable) ReflectionUtils.newInstance(getClass( + K key = + (K) ReflectionUtils.newInstance(getClass( in.readByte()), getConf()); key.readFields(in); @@ -196,7 +189,7 @@ public class SortedMapWritable extends AbstractMapWritable // Then write out each key/value pair - for (Map.Entry e: instance.entrySet()) { + for (Map.Entry e: instance.entrySet()) { out.writeByte(getId(e.getKey().getClass())); e.getKey().write(out); out.writeByte(getId(e.getValue().getClass())); @@ -211,7 +204,7 @@ public class SortedMapWritable extends AbstractMapWritable } if (obj instanceof SortedMapWritable) { - Map map = (Map) obj; + Map map = (Map) obj; if (size() != map.size()) { return false; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java index 5ed1db25ffb..8c988083848 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java @@ -45,7 +45,7 @@ public class TestSortedMapWritable { new BytesWritable("value3".getBytes()) }; - SortedMapWritable inMap = new SortedMapWritable(); + SortedMapWritable inMap = new SortedMapWritable(); for (int i = 0; i < keys.length; i++) { inMap.put(keys[i], values[i]); } @@ -53,13 +53,14 @@ public class TestSortedMapWritable { assertEquals(0, inMap.firstKey().compareTo(keys[0])); assertEquals(0, inMap.lastKey().compareTo(keys[2])); - SortedMapWritable outMap = new SortedMapWritable(inMap); + SortedMapWritable outMap = new SortedMapWritable(inMap); assertEquals(inMap.size(), outMap.size()); - for (Map.Entry e: inMap.entrySet()) { + for (Map.Entry e: inMap.entrySet()) { assertTrue(outMap.containsKey(e.getKey())); - assertEquals(0, ((WritableComparable) outMap.get(e.getKey())).compareTo( - e.getValue())); + WritableComparable> aValue = (WritableComparable>) outMap.get(e.getKey()); + WritableComparable> bValue = (WritableComparable>) e.getValue(); + assertEquals(0, aValue.compareTo(bValue)); } // Now for something a little harder... @@ -69,24 +70,24 @@ public class TestSortedMapWritable { new Text("map2") }; - SortedMapWritable mapOfMaps = new SortedMapWritable(); + SortedMapWritable mapOfMaps = new SortedMapWritable(); mapOfMaps.put(maps[0], inMap); mapOfMaps.put(maps[1], outMap); - SortedMapWritable copyOfMapOfMaps = new SortedMapWritable(mapOfMaps); + SortedMapWritable copyOfMapOfMaps = new SortedMapWritable(mapOfMaps); for (int i = 0; i < maps.length; i++) { assertTrue(copyOfMapOfMaps.containsKey(maps[i])); - SortedMapWritable a = (SortedMapWritable) mapOfMaps.get(maps[i]); - SortedMapWritable b = (SortedMapWritable) copyOfMapOfMaps.get(maps[i]); + SortedMapWritable a = (SortedMapWritable) mapOfMaps.get(maps[i]); + SortedMapWritable b = (SortedMapWritable) copyOfMapOfMaps.get(maps[i]); assertEquals(a.size(), b.size()); for (Writable key: a.keySet()) { assertTrue(b.containsKey(key)); // This will work because we know what we put into each set - WritableComparable aValue = (WritableComparable) a.get(key); - WritableComparable bValue = (WritableComparable) b.get(key); + WritableComparable> aValue = (WritableComparable>) a.get(key); + WritableComparable> bValue = (WritableComparable>) b.get(key); assertEquals(0, aValue.compareTo(bValue)); } } @@ -98,11 +99,11 @@ public class TestSortedMapWritable { @Test @SuppressWarnings("deprecation") public void testForeignClass() { - SortedMapWritable inMap = new SortedMapWritable(); + SortedMapWritable inMap = new SortedMapWritable(); inMap.put(new Text("key"), new UTF8("value")); inMap.put(new Text("key2"), new UTF8("value2")); - SortedMapWritable outMap = new SortedMapWritable(inMap); - SortedMapWritable copyOfCopy = new SortedMapWritable(outMap); + SortedMapWritable outMap = new SortedMapWritable(inMap); + SortedMapWritable copyOfCopy = new SortedMapWritable(outMap); assertEquals(1, copyOfCopy.getNewClasses()); } @@ -112,8 +113,8 @@ public class TestSortedMapWritable { @Test public void testEqualsAndHashCode() { String failureReason; - SortedMapWritable mapA = new SortedMapWritable(); - SortedMapWritable mapB = new SortedMapWritable(); + SortedMapWritable mapA = new SortedMapWritable(); + SortedMapWritable mapB = new SortedMapWritable(); // Sanity checks failureReason = "SortedMapWritable couldn't be initialized. Got null reference"; @@ -167,8 +168,8 @@ public class TestSortedMapWritable { @Test(timeout = 1000) public void testPutAll() { - SortedMapWritable map1 = new SortedMapWritable(); - SortedMapWritable map2 = new SortedMapWritable(); + SortedMapWritable map1 = new SortedMapWritable(); + SortedMapWritable map2 = new SortedMapWritable(); map1.put(new Text("key"), new Text("value")); map2.putAll(map1); diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableInput.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableInput.java index a7ca12cf93c..70b2100361c 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableInput.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableInput.java @@ -330,22 +330,25 @@ public class TypedBytesWritableInput implements Configurable { return readMap(null); } - public SortedMapWritable readSortedMap(SortedMapWritable mw) + public > + SortedMapWritable readSortedMap(SortedMapWritable mw) throws IOException { if (mw == null) { - mw = new SortedMapWritable(); + mw = new SortedMapWritable(); } int length = in.readMapHeader(); for (int i = 0; i < length; i++) { - WritableComparable key = (WritableComparable) read(); + @SuppressWarnings("unchecked") + K key = (K) read(); Writable value = read(); mw.put(key, value); } return mw; } - public SortedMapWritable readSortedMap() throws IOException { - return readSortedMap(null); + public > SortedMapWritable + readSortedMap() throws IOException { + return readSortedMap((SortedMapWritable)null); } public Writable readWritable(Writable writable) throws IOException { diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableOutput.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableOutput.java index 8dd047271ab..e7afe93548c 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableOutput.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableOutput.java @@ -131,7 +131,7 @@ public class TypedBytesWritableOutput { } else if (w instanceof MapWritable) { writeMap((MapWritable) w); } else if (w instanceof SortedMapWritable) { - writeSortedMap((SortedMapWritable) w); + writeSortedMap((SortedMapWritable) w); } else if (w instanceof Record) { writeRecord((Record) w); } else { @@ -200,9 +200,9 @@ public class TypedBytesWritableOutput { } } - public void writeSortedMap(SortedMapWritable smw) throws IOException { + public void writeSortedMap(SortedMapWritable smw) throws IOException { out.writeMapHeader(smw.size()); - for (Map.Entry entry : smw.entrySet()) { + for (Map.Entry, Writable> entry : smw.entrySet()) { write(entry.getKey()); write(entry.getValue()); }