diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 7cd61f5bc30..879589e833d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -376,6 +376,8 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4233. NPE can happen in RMNMNodeInfo. (bobby) + MAPREDUCE-4238. mavenize data_join. (tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 7ac4f120029..5466a88be1c 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -231,6 +231,11 @@ hadoop-distcp ${project.version} + + org.apache.hadoop + hadoop-datajoin + ${project.version} + org.apache.hadoop hadoop-rumen diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml new file mode 100644 index 00000000000..eed6d33bf66 --- /dev/null +++ b/hadoop-tools/hadoop-datajoin/pom.xml @@ -0,0 +1,120 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 2.0.0-SNAPSHOT + ../../hadoop-project + + org.apache.hadoop + hadoop-datajoin + 2.0.0-SNAPSHOT + Apache Hadoop Data Join + Apache Hadoop Data Join + jar + + + ${project.build.directory}/log + + + + + org.apache.hadoop + hadoop-annotations + provided + + + org.apache.hadoop + hadoop-mapreduce-client-hs + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + provided + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + provided + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + test + test-jar + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + org.apache.hadoop + hadoop-common + test + test-jar + + + org.apache.hadoop + hadoop-hdfs + test + test-jar + + + org.apache.hadoop + hadoop-yarn-server-tests + test-jar + test + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + create-log-dir + process-test-resources + + run + + + + + + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + + diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/ArrayListBackedIterator.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/ArrayListBackedIterator.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/ArrayListBackedIterator.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/ArrayListBackedIterator.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinMapperBase.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinMapperBase.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinMapperBase.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinMapperBase.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinReducerBase.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinReducerBase.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinReducerBase.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinReducerBase.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/JobBase.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/JobBase.java similarity index 97% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/JobBase.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/JobBase.java index dd34a4bb566..9ef21b3085e 100644 --- a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/JobBase.java +++ b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/JobBase.java @@ -51,7 +51,7 @@ public abstract class JobBase implements Mapper, Reducer { * the value for the counter */ protected void setLongValue(Object name, long value) { - this.longCounters.put(name, new Long(value)); + this.longCounters.put(name, Long.valueOf(value)); } /** @@ -100,9 +100,9 @@ public abstract class JobBase implements Mapper, Reducer { Long val = this.longCounters.get(name); Long retv = null; if (val == null) { - retv = new Long(inc); + retv = Long.valueOf(inc); } else { - retv = new Long(val.longValue() + inc); + retv = Long.valueOf(val.longValue() + inc); } this.longCounters.put(name, retv); return retv; diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/ResetableIterator.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/ResetableIterator.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/ResetableIterator.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/ResetableIterator.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/TaggedMapOutput.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/TaggedMapOutput.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/TaggedMapOutput.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/TaggedMapOutput.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/README.txt b/hadoop-tools/hadoop-datajoin/src/test/java/README.txt similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/README.txt rename to hadoop-tools/hadoop-datajoin/src/test/java/README.txt diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/SampleDataJoinMapper.java b/hadoop-tools/hadoop-datajoin/src/test/java/SampleDataJoinMapper.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/SampleDataJoinMapper.java rename to hadoop-tools/hadoop-datajoin/src/test/java/SampleDataJoinMapper.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/SampleDataJoinReducer.java b/hadoop-tools/hadoop-datajoin/src/test/java/SampleDataJoinReducer.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/SampleDataJoinReducer.java rename to hadoop-tools/hadoop-datajoin/src/test/java/SampleDataJoinReducer.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/SampleTaggedMapOutput.java b/hadoop-tools/hadoop-datajoin/src/test/java/SampleTaggedMapOutput.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/SampleTaggedMapOutput.java rename to hadoop-tools/hadoop-datajoin/src/test/java/SampleTaggedMapOutput.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/test/org/apache/hadoop/contrib/utils/join/TestDataJoin.java b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/TestDataJoin.java similarity index 98% rename from hadoop-mapreduce-project/src/contrib/data_join/src/test/org/apache/hadoop/contrib/utils/join/TestDataJoin.java rename to hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/TestDataJoin.java index 1eab9593df8..37df5b002e6 100644 --- a/hadoop-mapreduce-project/src/contrib/data_join/src/test/org/apache/hadoop/contrib/utils/join/TestDataJoin.java +++ b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/TestDataJoin.java @@ -57,6 +57,7 @@ public class TestDataJoin extends TestCase { public void testDataJoin() throws Exception { final int srcs = 4; JobConf job = new JobConf(); + job.setBoolean("mapreduce.fileoutputcommitter.marksuccessfuljobs", false); Path base = cluster.getFileSystem().makeQualified(new Path("/inner")); Path[] src = writeSimpleSrc(base, job, srcs); job.setInputFormat(SequenceFileInputFormat.class); diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml index 43edf485a93..595e86344ae 100644 --- a/hadoop-tools/hadoop-tools-dist/pom.xml +++ b/hadoop-tools/hadoop-tools-dist/pom.xml @@ -52,6 +52,11 @@ hadoop-rumen compile + + org.apache.hadoop + hadoop-datajoin + compile + org.apache.hadoop hadoop-extras diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index 28cf27feb8c..dbf08e18136 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -32,6 +32,7 @@ hadoop-distcp hadoop-archives hadoop-rumen + hadoop-datajoin hadoop-tools-dist hadoop-extras