Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-4771. KeyFieldBasedPartitioner not partitioning properly when configured (jlowe via bobby) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1405975 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-11-05 21:52:06 +00:00
parent b1aa62a848
commit 0eadfcfcf5
3 changed files with 23 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -631,6 +631,9 @@ Release 0.23.5 - UNRELEASED
MAPREDUCE-4763 repair test TestUmbilicalProtocolWithJobToken (Ivan A. MAPREDUCE-4763 repair test TestUmbilicalProtocolWithJobToken (Ivan A.
Veselovsky via bobby) Veselovsky via bobby)
MAPREDUCE-4771. KeyFieldBasedPartitioner not partitioning properly when
configured (jlowe via bobby)
Release 0.23.4 - UNRELEASED Release 0.23.4 - UNRELEASED

1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java
View File

@ -63,6 +63,7 @@ public class KeyFieldBasedPartitioner<K2, V2> extends Partitioner<K2, V2>
public void setConf(Configuration conf) { public void setConf(Configuration conf) {
this.conf = conf; this.conf = conf;
keyFieldHelper = new KeyFieldHelper();
String keyFieldSeparator = String keyFieldSeparator =
conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, "\t"); conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, "\t");
keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator); keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator);

23
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
View File

@ -17,17 +17,18 @@
*/ */
package org.apache.hadoop.mapred.lib; package org.apache.hadoop.mapred.lib;
import static org.junit.Assert.assertEquals;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner; import org.junit.Test;
import junit.framework.TestCase; public class TestKeyFieldBasedPartitioner {
public class TestKeyFieldBasedPartitioner extends TestCase {
/** /**
* Test is key-field-based partitioned works with empty key. * Test is key-field-based partitioned works with empty key.
*/ */
@Test
public void testEmptyKey() throws Exception { public void testEmptyKey() throws Exception {
KeyFieldBasedPartitioner<Text, Text> kfbp = KeyFieldBasedPartitioner<Text, Text> kfbp =
new KeyFieldBasedPartitioner<Text, Text>(); new KeyFieldBasedPartitioner<Text, Text>();
@ -37,4 +38,18 @@ public class TestKeyFieldBasedPartitioner extends TestCase {
assertEquals("Empty key should map to 0th partition", assertEquals("Empty key should map to 0th partition",
0, kfbp.getPartition(new Text(), new Text(), 10)); 0, kfbp.getPartition(new Text(), new Text(), 10));
} }
@Test
public void testMultiConfigure() {
KeyFieldBasedPartitioner<Text, Text> kfbp =
new KeyFieldBasedPartitioner<Text, Text>();
JobConf conf = new JobConf();
conf.set(KeyFieldBasedPartitioner.PARTITIONER_OPTIONS, "-k1,1");
kfbp.setConf(conf);
Text key = new Text("foo\tbar");
Text val = new Text("val");
int partNum = kfbp.getPartition(key, val, 4096);
kfbp.configure(conf);
assertEquals(partNum, kfbp.getPartition(key,val, 4096));
}
} }