From ee04add6961f25b5b0f8ca78c20040c6aa0e85fd Mon Sep 17 00:00:00 2001 From: Haohui Mai Date: Tue, 13 Oct 2015 15:24:24 -0700 Subject: [PATCH] HDFS-9167. Update pom.xml in other modules to depend on hdfs-client instead of hdfs. Contributed by Mingliang Liu. --- hadoop-client/pom.xml | 6 +----- hadoop-dist/pom.xml | 2 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 5 ----- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 5 ----- .../hadoop-mapreduce-client-hs/pom.xml | 2 +- .../hadoop-mapreduce-examples/pom.xml | 2 +- hadoop-tools/hadoop-ant/pom.xml | 2 +- .../src/main/java/org/apache/hadoop/ant/DfsTask.java | 9 +++++++-- hadoop-tools/hadoop-archives/pom.xml | 12 +++++++++++- hadoop-tools/hadoop-datajoin/pom.xml | 12 +++++++++++- hadoop-tools/hadoop-distcp/pom.xml | 7 ++++++- .../java/org/apache/hadoop/tools/DistCpSync.java | 8 ++++---- hadoop-tools/hadoop-extras/pom.xml | 12 +++++++++++- hadoop-tools/hadoop-gridmix/pom.xml | 7 ++++++- hadoop-tools/hadoop-rumen/pom.xml | 2 +- hadoop-tools/hadoop-streaming/pom.xml | 12 +++++++++++- 17 files changed, 76 insertions(+), 32 deletions(-) diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml index 5d7443f8ecf..c8d7fa930a6 100644 --- a/hadoop-client/pom.xml +++ b/hadoop-client/pom.xml @@ -100,13 +100,9 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client compile - - commons-daemon - commons-daemon - org.apache.avro avro diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index e09e108baf3..7cbdc6e33df 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -39,7 +39,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml index 2d76261dd1b..b09905beff6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml @@ -56,11 +56,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-hdfs compile - - org.apache.hadoop - hadoop-hdfs-client - compile - org.apache.hadoop hadoop-hdfs diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index bea2244254d..5b3d98587e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -720,6 +720,9 @@ Release 2.8.0 - UNRELEASED HDFS-9145. Tracking methods that hold FSNamesytemLock for too long. (Mingliang Liu via wheat9) + HDFS-9167. Update pom.xml in other modules to depend on hdfs-client instead + of hdfs. (Mingliang Liu via wheat9) + BUG FIXES HDFS-8091: ACLStatus and XAttributes should be presented to diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml index b644b04c819..45e8e686291 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml @@ -52,11 +52,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-hdfs provided - - org.apache.hadoop - hadoop-hdfs-client - provided - org.apache.hadoop hadoop-hdfs diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml index afd2a64ecd3..c164c782ef6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml @@ -35,7 +35,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client org.apache.hadoop diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml index 006cb6c4491..3d36378bf6b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml @@ -68,7 +68,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client runtime diff --git a/hadoop-tools/hadoop-ant/pom.xml b/hadoop-tools/hadoop-ant/pom.xml index 6d3c89db7e7..03a829c6a4e 100644 --- a/hadoop-tools/hadoop-ant/pom.xml +++ b/hadoop-tools/hadoop-ant/pom.xml @@ -48,7 +48,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided diff --git a/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java index 9d0b3a42c22..8dc7f3766eb 100644 --- a/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java +++ b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java @@ -32,7 +32,6 @@ import org.apache.tools.ant.BuildException; import org.apache.tools.ant.Task; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.HdfsConfiguration; /** * {@link org.apache.hadoop.fs.FsShell FsShell} wrapper for ant Task. @@ -40,6 +39,12 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; @InterfaceAudience.Private public class DfsTask extends Task { + static { + // adds the default resources + Configuration.addDefaultResource("hdfs-default.xml"); + Configuration.addDefaultResource("hdfs-site.xml"); + } + /** * Default sink for {@link java.lang.System#out} * and {@link java.lang.System#err}. @@ -187,7 +192,7 @@ public class DfsTask extends Task { try { pushContext(); - Configuration conf = new HdfsConfiguration(); + Configuration conf = new Configuration(); conf.setClassLoader(confloader); exit_code = ToolRunner.run(conf, shell, argv.toArray(new String[argv.size()])); diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml index 453d81bf315..8d40cf83c77 100644 --- a/hadoop-tools/hadoop-archives/pom.xml +++ b/hadoop-tools/hadoop-archives/pom.xml @@ -40,6 +40,11 @@ junit test + + org.mockito + mockito-all + test + org.apache.hadoop hadoop-annotations @@ -73,7 +78,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided @@ -82,6 +87,11 @@ test test-jar + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml index 5e302da7635..9dce969e837 100644 --- a/hadoop-tools/hadoop-datajoin/pom.xml +++ b/hadoop-tools/hadoop-datajoin/pom.xml @@ -37,6 +37,11 @@ junit test + + org.mockito + mockito-all + test + org.apache.hadoop hadoop-annotations @@ -70,7 +75,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided @@ -79,6 +84,11 @@ test test-jar + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml index cb8490396c5..eeec27d74d8 100644 --- a/hadoop-tools/hadoop-distcp/pom.xml +++ b/hadoop-tools/hadoop-distcp/pom.xml @@ -79,9 +79,14 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java index 47a28a6f927..2b8ed719d94 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java @@ -20,7 +20,7 @@ package org.apache.hadoop.tools; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; @@ -165,13 +165,13 @@ class DistCpSync { entry.getType() == SnapshotDiffReport.DiffType.CREATE || entry.getType() == SnapshotDiffReport.DiffType.DELETE) { final Path source = - new Path(DFSUtil.bytes2String(entry.getSourcePath())); + new Path(DFSUtilClient.bytes2String(entry.getSourcePath())); list.add(new DiffInfo(source, null, entry.getType())); } else if (entry.getType() == SnapshotDiffReport.DiffType.RENAME) { final Path source = - new Path(DFSUtil.bytes2String(entry.getSourcePath())); + new Path(DFSUtilClient.bytes2String(entry.getSourcePath())); final Path target = - new Path(DFSUtil.bytes2String(entry.getTargetPath())); + new Path(DFSUtilClient.bytes2String(entry.getTargetPath())); list.add(new DiffInfo(source, target, entry.getType())); } } diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml index 6ca049bdaa9..ba6f9c32b0e 100644 --- a/hadoop-tools/hadoop-extras/pom.xml +++ b/hadoop-tools/hadoop-extras/pom.xml @@ -40,6 +40,11 @@ junit test + + org.mockito + mockito-all + test + org.apache.hadoop hadoop-annotations @@ -73,7 +78,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided @@ -82,6 +87,11 @@ test test-jar + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml index 6eab8c307bb..a6abda33928 100644 --- a/hadoop-tools/hadoop-gridmix/pom.xml +++ b/hadoop-tools/hadoop-gridmix/pom.xml @@ -75,7 +75,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided @@ -84,6 +84,11 @@ test test-jar + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml index 96e61db60ac..382a33871b3 100644 --- a/hadoop-tools/hadoop-rumen/pom.xml +++ b/hadoop-tools/hadoop-rumen/pom.xml @@ -73,7 +73,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml index 2c79d46a091..c63d1429fcc 100644 --- a/hadoop-tools/hadoop-streaming/pom.xml +++ b/hadoop-tools/hadoop-streaming/pom.xml @@ -41,6 +41,11 @@ junit test + + org.mockito + mockito-all + test + org.apache.hadoop hadoop-annotations @@ -79,7 +84,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided @@ -88,6 +93,11 @@ test test-jar + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs