From da8441d0fe9149bb845dcf701fdc86e786b6afba Mon Sep 17 00:00:00 2001 From: Haohui Mai Date: Tue, 13 Oct 2015 15:24:24 -0700 Subject: [PATCH] HDFS-9167. Update pom.xml in other modules to depend on hdfs-client instead of hdfs. Contributed by Mingliang Liu. --- hadoop-client/pom.xml | 6 +----- hadoop-dist/pom.xml | 2 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 5 ----- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 5 ----- .../hadoop-mapreduce-client-hs/pom.xml | 2 +- .../hadoop-mapreduce-examples/pom.xml | 2 +- hadoop-tools/hadoop-ant/pom.xml | 2 +- .../src/main/java/org/apache/hadoop/ant/DfsTask.java | 9 +++++++-- hadoop-tools/hadoop-archives/pom.xml | 12 +++++++++++- hadoop-tools/hadoop-datajoin/pom.xml | 12 +++++++++++- hadoop-tools/hadoop-distcp/pom.xml | 7 ++++++- .../java/org/apache/hadoop/tools/DistCpSync.java | 8 ++++---- hadoop-tools/hadoop-extras/pom.xml | 12 +++++++++++- hadoop-tools/hadoop-gridmix/pom.xml | 7 ++++++- hadoop-tools/hadoop-rumen/pom.xml | 2 +- hadoop-tools/hadoop-streaming/pom.xml | 12 +++++++++++- 17 files changed, 76 insertions(+), 32 deletions(-) diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml index 2324dba8e96..10646593f18 100644 --- a/hadoop-client/pom.xml +++ b/hadoop-client/pom.xml @@ -104,13 +104,9 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client compile - - commons-daemon - commons-daemon - org.apache.avro avro diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index f1644a82c55..9d37599c92b 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -39,7 +39,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml index ac8930c15b3..9a9d29ceec3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml @@ -56,11 +56,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-hdfs compile - - org.apache.hadoop - hadoop-hdfs-client - compile - org.apache.hadoop hadoop-hdfs diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e3b4bcfa390..cdc8dfd0f0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1550,6 +1550,9 @@ Release 2.8.0 - UNRELEASED HDFS-9145. Tracking methods that hold FSNamesytemLock for too long. (Mingliang Liu via wheat9) + HDFS-9167. Update pom.xml in other modules to depend on hdfs-client instead + of hdfs. (Mingliang Liu via wheat9) + BUG FIXES HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml index 005ee4dbdcc..047f0e14139 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml @@ -52,11 +52,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-hdfs provided - - org.apache.hadoop - hadoop-hdfs-client - provided - org.apache.hadoop hadoop-hdfs diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml index 337201904d4..b5509bd3810 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml @@ -35,7 +35,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client org.apache.hadoop diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml index a94e663ee8a..041a5a829d9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml @@ -67,7 +67,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client runtime diff --git a/hadoop-tools/hadoop-ant/pom.xml b/hadoop-tools/hadoop-ant/pom.xml index 197b527edc5..a09fdd11242 100644 --- a/hadoop-tools/hadoop-ant/pom.xml +++ b/hadoop-tools/hadoop-ant/pom.xml @@ -48,7 +48,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided diff --git a/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java index 9d0b3a42c22..8dc7f3766eb 100644 --- a/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java +++ b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java @@ -32,7 +32,6 @@ import org.apache.tools.ant.BuildException; import org.apache.tools.ant.Task; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.HdfsConfiguration; /** * {@link org.apache.hadoop.fs.FsShell FsShell} wrapper for ant Task. @@ -40,6 +39,12 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; @InterfaceAudience.Private public class DfsTask extends Task { + static { + // adds the default resources + Configuration.addDefaultResource("hdfs-default.xml"); + Configuration.addDefaultResource("hdfs-site.xml"); + } + /** * Default sink for {@link java.lang.System#out} * and {@link java.lang.System#err}. @@ -187,7 +192,7 @@ public class DfsTask extends Task { try { pushContext(); - Configuration conf = new HdfsConfiguration(); + Configuration conf = new Configuration(); conf.setClassLoader(confloader); exit_code = ToolRunner.run(conf, shell, argv.toArray(new String[argv.size()])); diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml index 5334f48ff47..30baa1c663c 100644 --- a/hadoop-tools/hadoop-archives/pom.xml +++ b/hadoop-tools/hadoop-archives/pom.xml @@ -40,6 +40,11 @@ junit test + + org.mockito + mockito-all + test + org.apache.hadoop hadoop-annotations @@ -73,7 +78,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided @@ -82,6 +87,11 @@ test test-jar + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml index a4d00f8cf88..ff511f197fb 100644 --- a/hadoop-tools/hadoop-datajoin/pom.xml +++ b/hadoop-tools/hadoop-datajoin/pom.xml @@ -37,6 +37,11 @@ junit test + + org.mockito + mockito-all + test + org.apache.hadoop hadoop-annotations @@ -70,7 +75,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided @@ -79,6 +84,11 @@ test test-jar + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml index 374155d47c6..7099cec9b2c 100644 --- a/hadoop-tools/hadoop-distcp/pom.xml +++ b/hadoop-tools/hadoop-distcp/pom.xml @@ -79,9 +79,14 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java index 47a28a6f927..2b8ed719d94 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java @@ -20,7 +20,7 @@ package org.apache.hadoop.tools; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; @@ -165,13 +165,13 @@ class DistCpSync { entry.getType() == SnapshotDiffReport.DiffType.CREATE || entry.getType() == SnapshotDiffReport.DiffType.DELETE) { final Path source = - new Path(DFSUtil.bytes2String(entry.getSourcePath())); + new Path(DFSUtilClient.bytes2String(entry.getSourcePath())); list.add(new DiffInfo(source, null, entry.getType())); } else if (entry.getType() == SnapshotDiffReport.DiffType.RENAME) { final Path source = - new Path(DFSUtil.bytes2String(entry.getSourcePath())); + new Path(DFSUtilClient.bytes2String(entry.getSourcePath())); final Path target = - new Path(DFSUtil.bytes2String(entry.getTargetPath())); + new Path(DFSUtilClient.bytes2String(entry.getTargetPath())); list.add(new DiffInfo(source, target, entry.getType())); } } diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml index 429f4a303a9..5f758758f10 100644 --- a/hadoop-tools/hadoop-extras/pom.xml +++ b/hadoop-tools/hadoop-extras/pom.xml @@ -40,6 +40,11 @@ junit test + + org.mockito + mockito-all + test + org.apache.hadoop hadoop-annotations @@ -73,7 +78,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided @@ -82,6 +87,11 @@ test test-jar + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml index a3cac91406a..baa72ca3261 100644 --- a/hadoop-tools/hadoop-gridmix/pom.xml +++ b/hadoop-tools/hadoop-gridmix/pom.xml @@ -75,7 +75,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided @@ -84,6 +84,11 @@ test test-jar + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml index 0475128a363..6828b1c1f1b 100644 --- a/hadoop-tools/hadoop-rumen/pom.xml +++ b/hadoop-tools/hadoop-rumen/pom.xml @@ -73,7 +73,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml index 9acc43b06a6..5ea10f63182 100644 --- a/hadoop-tools/hadoop-streaming/pom.xml +++ b/hadoop-tools/hadoop-streaming/pom.xml @@ -41,6 +41,11 @@ junit test + + org.mockito + mockito-all + test + org.apache.hadoop hadoop-annotations @@ -79,7 +84,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client provided @@ -88,6 +93,11 @@ test test-jar + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs