From dfad16fed270b9014b8486eb2c201cb86f89b85e Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Mon, 13 Aug 2012 21:48:33 +0000 Subject: [PATCH 01/18] MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over group names deprecated names don't show up (Robert Evans via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1372636 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../mapreduce/counters/AbstractCounters.java | 11 ++++++++++- .../org/apache/hadoop/mapred/TestCounters.java | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e0b33bd2f63..9f3ddc0ec8b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -809,6 +809,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason Lowe via bobby) + MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over + group names deprecated names don't show up (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java index 82ab06d7bf2..17b433d095b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java @@ -24,6 +24,7 @@ import static org.apache.hadoop.mapreduce.counters.CounterGroupFactory.isFramewo import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentSkipListMap; @@ -185,7 +186,15 @@ public abstract class AbstractCounters getGroupNames() { - return Iterables.concat(fgroups.keySet(), groups.keySet()); + HashSet deprecated = new HashSet(); + for(Map.Entry entry : legacyMap.entrySet()) { + String newGroup = entry.getValue(); + boolean isFGroup = isFrameworkGroup(newGroup); + if(isFGroup ? fgroups.containsKey(newGroup) : groups.containsKey(newGroup)) { + deprecated.add(entry.getKey()); + } + } + return Iterables.concat(fgroups.keySet(), groups.keySet(), deprecated); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java index 8d8074a752b..da6c12dc8d3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.text.ParseException; +import java.util.HashSet; import java.util.Iterator; import java.util.Random; @@ -224,6 +225,23 @@ public class TestCounters { iterator.next(); } + @Test + public void testLegacyGetGroupNames() { + Counters counters = new Counters(); + // create 2 filesystem counter groups + counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1); + counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1); + counters.incrCounter("group1", "counter1", 1); + + HashSet groups = new HashSet(counters.getGroupNames()); + HashSet expectedGroups = new HashSet(); + expectedGroups.add("group1"); + expectedGroups.add("FileSystemCounter"); //Legacy Name + expectedGroups.add("org.apache.hadoop.mapreduce.FileSystemCounter"); + + assertEquals(expectedGroups, groups); + } + @Test public void testMakeCompactString() { final String GC1 = "group1.counter1:1"; From d71ea73012409c977d292cc954e6ec400bdcd2cf Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 13 Aug 2012 22:00:33 +0000 Subject: [PATCH 02/18] HADOOP-8581 Amendment to CHANGES.txt setting right JIRA number, add support for HTTPS to the web UIs. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1372644 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 4d4857a55a9..4911152148f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -213,7 +213,7 @@ Branch-2 ( Unreleased changes ) HADOOP-8644. AuthenticatedURL should be able to use SSLFactory. (tucu) - HADOOP-8681. add support for HTTPS to the web UIs. (tucu) + HADOOP-8581. add support for HTTPS to the web UIs. (tucu) IMPROVEMENTS From c959c1ac750d82fca1ddaf877e970a39ab0d2e4c Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Mon, 13 Aug 2012 22:08:39 +0000 Subject: [PATCH 03/18] HADOOP-8687. Upgrade log4j to 1.2.17. Contributed by Eli Collins git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1372649 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ hadoop-project/pom.xml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 4911152148f..50e8a08d0a2 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -284,6 +284,8 @@ Branch-2 ( Unreleased changes ) HADOOP-8620. Add -Drequire.fuse and -Drequire.snappy. (Colin Patrick McCabe via eli) + HADOOP-8687. Upgrade log4j to 1.2.17. (eli) + BUG FIXES HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index e7825c12afc..8398e50a691 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -458,7 +458,7 @@ log4j log4j - 1.2.15 + 1.2.17 com.sun.jdmk From 4139298b61bceddb98d97a3734771f034411150e Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Tue, 14 Aug 2012 00:04:55 +0000 Subject: [PATCH 04/18] HDFS-3790. test_fuse_dfs.c doesn't compile on centos 5. Contributed by Colin Patrick McCabe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1372676 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop-hdfs/src/main/native/fuse-dfs/test/test_fuse_dfs.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e1466388104..a75e011c041 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -587,6 +587,9 @@ Branch-2 ( Unreleased changes ) IOExceptions of stream closures can mask root exceptions. (Uma Maheswara Rao G via szetszwo) + HDFS-3790. test_fuse_dfs.c doesn't compile on centos 5. (Colin Patrick + McCabe via atm) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/test_fuse_dfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/test_fuse_dfs.c index 3b479fbe241..f4212a6c519 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/test_fuse_dfs.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/test_fuse_dfs.c @@ -75,7 +75,7 @@ static int fuserMount(int *procRet, ...) { int ret, status; size_t i = 0; - char *args[64], *c, *env[] = { NULL }; + char *args[64], *c; va_list ap; pid_t pid, pret; @@ -99,7 +99,7 @@ static int fuserMount(int *procRet, ...) ret, strerror(ret)); return -ret; } else if (pid == 0) { - if (execvpe("fusermount", args, env)) { + if (execvp("fusermount", args)) { ret = errno; fprintf(stderr, "FUSE_TEST: failed to execute fusermount: " "error %d: %s\n", ret, strerror(ret)); From ab3ab17a433bf0bc87c2434ccf24fa4ab65eeb50 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Tue, 14 Aug 2012 00:52:10 +0000 Subject: [PATCH 05/18] HDFS-3792. Fix two findbugs introduced by HDFS-3695. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1372690 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../org/apache/hadoop/hdfs/server/namenode/FSEditLog.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a75e011c041..a645a936994 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -183,6 +183,8 @@ Trunk (unreleased changes) HDFS-3625. Fix TestBackupNode by properly initializing edit log during startup. (Junping Du via todd) + HDFS-3792. Fix two findbugs introduced by HDFS-3695 (todd) + Branch-2 ( Unreleased changes ) INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index 54b5f1a6a38..11c6948b1cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -341,7 +341,7 @@ public class FSEditLog { * File-based journals are skipped, since they are formatted by the * Storage format code. */ - void formatNonFileJournals(NamespaceInfo nsInfo) throws IOException { + synchronized void formatNonFileJournals(NamespaceInfo nsInfo) throws IOException { Preconditions.checkState(state == State.BETWEEN_LOG_SEGMENTS, "Bad state: %s", state); @@ -352,7 +352,7 @@ public class FSEditLog { } } - List getFormatConfirmables() { + synchronized List getFormatConfirmables() { Preconditions.checkState(state == State.BETWEEN_LOG_SEGMENTS, "Bad state: %s", state); From 0c27c770802e298862e4b60a964fd0a8011d0a69 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 14 Aug 2012 03:02:43 +0000 Subject: [PATCH 06/18] HDFS-3658. Fix bugs in TestDFSClientRetries and add more tests. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1372707 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hdfs/server/namenode/FSNamesystem.java | 5 ++ .../apache/hadoop/hdfs/MiniDFSCluster.java | 6 +- .../hadoop/hdfs/TestDFSClientRetries.java | 71 +++++++++++++++++-- 4 files changed, 76 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a645a936994..30e08c6eba8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -592,6 +592,8 @@ Branch-2 ( Unreleased changes ) HDFS-3790. test_fuse_dfs.c doesn't compile on centos 5. (Colin Patrick McCabe via atm) + HDFS-3658. Fix bugs in TestDFSClientRetries and add more tests. (szetszwo) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 20a655979a7..7887eafe374 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3742,6 +3742,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats, this.extension = conf.getInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0); this.safeReplication = conf.getInt(DFS_NAMENODE_REPLICATION_MIN_KEY, DFS_NAMENODE_REPLICATION_MIN_DEFAULT); + + LOG.info(DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY + " = " + threshold); + LOG.info(DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY + " = " + datanodeThreshold); + LOG.info(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY + " = " + extension); + // default to safe mode threshold (i.e., don't populate queues before leaving safe mode) this.replQueueThreshold = conf.getFloat(DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index c7e55d93c80..a5cc0b293b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -118,6 +118,8 @@ public class MiniDFSCluster { public static final String PROP_TEST_BUILD_DATA = "test.build.data"; /** Configuration option to set the data dir: {@value} */ public static final String HDFS_MINIDFS_BASEDIR = "hdfs.minidfs.basedir"; + public static final String DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY + = DFS_NAMENODE_SAFEMODE_EXTENSION_KEY + ".testing"; static { DefaultMetricsSystem.setMiniClusterMode(true); } @@ -587,7 +589,9 @@ public class MiniDFSCluster { int replication = conf.getInt(DFS_REPLICATION_KEY, 3); conf.setInt(DFS_REPLICATION_KEY, Math.min(replication, numDataNodes)); - conf.setInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0); + int safemodeExtension = conf.getInt( + DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY, 0); + conf.setInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, safemodeExtension); conf.setInt(DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 3); // 3 second conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, StaticMapping.class, DNSToSwitchMapping.class); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index d8771426115..a06263afd22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -41,6 +41,7 @@ import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Random; import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; @@ -79,8 +80,10 @@ import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.log4j.Level; +import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; import org.mockito.internal.stubbing.answers.ThrowsException; @@ -842,6 +845,8 @@ public class TestDFSClientRetries { final Path dir = new Path("/testNamenodeRestart"); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, true); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, 1); + conf.setInt(MiniDFSCluster.DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY, 5000); final short numDatanodes = 3; final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) @@ -864,11 +869,38 @@ public class TestDFSClientRetries { final FileStatus s1 = fs.getFileStatus(file1); assertEquals(length, s1.getLen()); + //create file4, write some data but not close + final Path file4 = new Path(dir, "file4"); + final FSDataOutputStream out4 = fs.create(file4, false, 4096, + fs.getDefaultReplication(file4), 1024L, null); + final byte[] bytes = new byte[1000]; + new Random().nextBytes(bytes); + out4.write(bytes); + out4.write(bytes); + out4.hflush(); + //shutdown namenode assertTrue(HdfsUtils.isHealthy(uri)); cluster.shutdownNameNode(0); assertFalse(HdfsUtils.isHealthy(uri)); + //namenode is down, continue writing file4 in a thread + final Thread file4thread = new Thread(new Runnable() { + @Override + public void run() { + try { + //write some more data and then close the file + out4.write(bytes); + out4.write(bytes); + out4.write(bytes); + out4.close(); + } catch (Exception e) { + exceptions.add(e); + } + } + }); + file4thread.start(); + //namenode is down, read the file in a thread final Thread reader = new Thread(new Runnable() { @Override @@ -927,10 +959,26 @@ public class TestDFSClientRetries { //check file1 and file3 thread.join(); + assertEmpty(exceptions); assertEquals(s1.getLen(), fs.getFileStatus(file3).getLen()); assertEquals(fs.getFileChecksum(file1), fs.getFileChecksum(file3)); reader.join(); + assertEmpty(exceptions); + + //check file4 + file4thread.join(); + assertEmpty(exceptions); + { + final FSDataInputStream in = fs.open(file4); + int count = 0; + for(int r; (r = in.read()) != -1; count++) { + Assert.assertEquals(String.format("count=%d", count), + bytes[count % bytes.length], (byte)r); + } + Assert.assertEquals(5 * bytes.length, count); + in.close(); + } //enter safe mode assertTrue(HdfsUtils.isHealthy(uri)); @@ -970,18 +1018,27 @@ public class TestDFSClientRetries { LOG.info("GOOD!", fnfe); } - if (!exceptions.isEmpty()) { - LOG.error("There are " + exceptions.size() + " exception(s):"); - for(int i = 0; i < exceptions.size(); i++) { - LOG.error("Exception " + i, exceptions.get(i)); - } - fail(); - } + assertEmpty(exceptions); } finally { cluster.shutdown(); } } + static void assertEmpty(final List exceptions) { + if (!exceptions.isEmpty()) { + final StringBuilder b = new StringBuilder("There are ") + .append(exceptions.size()) + .append(" exception(s):"); + for(int i = 0; i < exceptions.size(); i++) { + b.append("\n Exception ") + .append(i) + .append(": ") + .append(StringUtils.stringifyException(exceptions.get(i))); + } + fail(b.toString()); + } + } + private static FileSystem createFsWithDifferentUsername( final Configuration conf, final boolean isWebHDFS ) throws IOException, InterruptedException { From b38bd555e837569672dfd48ea1b60e60efc71648 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Tue, 14 Aug 2012 20:11:40 +0000 Subject: [PATCH 07/18] HDFS-3765. namenode -initializeSharedEdits should be able to initialize all shared storages. Contributed by Vinay and Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373061 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/hadoop/util/ExitUtil.java | 11 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../TestBookKeeperAsHASharedDir.java | 108 +++++++++++++++++ .../hadoop/hdfs/server/namenode/NameNode.java | 113 ++++++++++-------- .../hdfs/server/namenode/ha/HATestUtil.java | 2 +- .../ha/TestInitializeSharedEdits.java | 7 ++ 6 files changed, 192 insertions(+), 52 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java index a4af34666ef..aad7ca03be9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java @@ -64,6 +64,15 @@ public final class ExitUtil { return firstExitException; } + /** + * Reset the tracking of process termination. This is for use + * in unit tests where one test in the suite expects an exit + * but others do not. + */ + public static void resetFirstExitException() { + firstExitException = null; + } + /** * Terminate the current process. Note that terminate is the *only* method * that should be used to terminate the daemon processes. @@ -103,4 +112,4 @@ public final class ExitUtil { public static void terminate(int status) throws ExitException { terminate(status, "ExitException"); } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 30e08c6eba8..5cd7f668419 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -383,6 +383,9 @@ Branch-2 ( Unreleased changes ) HDFS-3276. initializeSharedEdits should have a -nonInteractive flag (todd) + HDFS-3765. namenode -initializeSharedEdits should be able to initialize + all shared storages. (Vinay and todd via todd) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java index d1a80f637af..ebbf80aa375 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java @@ -18,6 +18,7 @@ package org.apache.hadoop.contrib.bkjournal; import static org.junit.Assert.*; + import org.junit.Test; import org.junit.Before; import org.junit.After; @@ -25,6 +26,9 @@ import org.junit.BeforeClass; import org.junit.AfterClass; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ha.ServiceFailedException; +import org.apache.hadoop.ha.HAServiceProtocol.RequestSource; +import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HAUtil; @@ -35,12 +39,16 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.FSEditLogTestUtil; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.ExitUtil.ExitException; import org.apache.bookkeeper.proto.BookieServer; @@ -48,7 +56,9 @@ import org.apache.bookkeeper.proto.BookieServer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import java.io.File; import java.io.IOException; +import java.net.URISyntaxException; /** * Integration test to ensure that the BookKeeper JournalManager @@ -67,6 +77,11 @@ public class TestBookKeeperAsHASharedDir { bkutil = new BKJMUtil(numBookies); bkutil.start(); } + + @Before + public void clearExitStatus() { + ExitUtil.resetFirstExitException(); + } @AfterClass public static void teardownBookkeeper() throws Exception { @@ -244,4 +259,97 @@ public class TestBookKeeperAsHASharedDir { } } } + + /** + * Use NameNode INTIALIZESHAREDEDITS to initialize the shared edits. i.e. copy + * the edits log segments to new bkjm shared edits. + * + * @throws Exception + */ + @Test + public void testInitializeBKSharedEdits() throws Exception { + MiniDFSCluster cluster = null; + try { + Configuration conf = new Configuration(); + HAUtil.setAllowStandbyReads(conf, true); + conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); + + MiniDFSNNTopology topology = MiniDFSNNTopology.simpleHATopology(); + cluster = new MiniDFSCluster.Builder(conf).nnTopology(topology) + .numDataNodes(0).build(); + cluster.waitActive(); + // Shutdown and clear the current filebased shared dir. + cluster.shutdownNameNodes(); + File shareddir = new File(cluster.getSharedEditsDir(0, 1)); + assertTrue("Initial Shared edits dir not fully deleted", + FileUtil.fullyDelete(shareddir)); + + // Check namenodes should not start without shared dir. + assertCanNotStartNamenode(cluster, 0); + assertCanNotStartNamenode(cluster, 1); + + // Configure bkjm as new shared edits dir in both namenodes + Configuration nn1Conf = cluster.getConfiguration(0); + Configuration nn2Conf = cluster.getConfiguration(1); + nn1Conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil + .createJournalURI("/initializeSharedEdits").toString()); + nn2Conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil + .createJournalURI("/initializeSharedEdits").toString()); + BKJMUtil.addJournalManagerDefinition(nn1Conf); + BKJMUtil.addJournalManagerDefinition(nn2Conf); + + // Initialize the BKJM shared edits. + assertFalse(NameNode.initializeSharedEdits(nn1Conf)); + + // NameNode should be able to start and should be in sync with BKJM as + // shared dir + assertCanStartHANameNodes(cluster, conf, "/testBKJMInitialize"); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + private void assertCanNotStartNamenode(MiniDFSCluster cluster, int nnIndex) { + try { + cluster.restartNameNode(nnIndex, false); + fail("Should not have been able to start NN" + (nnIndex) + + " without shared dir"); + } catch (IOException ioe) { + LOG.info("Got expected exception", ioe); + GenericTestUtils.assertExceptionContains( + "Cannot start an HA namenode with name dirs that need recovery", ioe); + } + } + + private void assertCanStartHANameNodes(MiniDFSCluster cluster, + Configuration conf, String path) throws ServiceFailedException, + IOException, URISyntaxException, InterruptedException { + // Now should be able to start both NNs. Pass "false" here so that we don't + // try to waitActive on all NNs, since the second NN doesn't exist yet. + cluster.restartNameNode(0, false); + cluster.restartNameNode(1, true); + + // Make sure HA is working. + cluster + .getNameNode(0) + .getRpcServer() + .transitionToActive( + new StateChangeRequestInfo(RequestSource.REQUEST_BY_USER)); + FileSystem fs = null; + try { + Path newPath = new Path(path); + fs = HATestUtil.configureFailoverFs(cluster, conf); + assertTrue(fs.mkdirs(newPath)); + HATestUtil.waitForStandbyToCatchUp(cluster.getNameNode(0), + cluster.getNameNode(1)); + assertTrue(NameNodeAdapter.getFileInfo(cluster.getNameNode(1), + newPath.toString(), false).isDir()); + } finally { + if (fs != null) { + fs.close(); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 38c84151656..20e7aafba9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -17,18 +17,12 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Iterator; import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -42,10 +36,10 @@ import org.apache.hadoop.ha.HealthCheckFailedException; import org.apache.hadoop.ha.ServiceFailedException; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Trash; import static org.apache.hadoop.hdfs.DFSConfigKeys.*; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -53,9 +47,6 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; -import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; -import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream; import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState; import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby; import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; @@ -68,8 +59,6 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; -import org.apache.hadoop.hdfs.util.AtomicFileOutputStream; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; @@ -88,6 +77,7 @@ import static org.apache.hadoop.util.ExitUtil.terminate; import static org.apache.hadoop.util.ToolRunner.confirmPrompt; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; /********************************************************** @@ -767,9 +757,18 @@ public class NameNode { String nsId = DFSUtil.getNamenodeNameServiceId(conf); String namenodeId = HAUtil.getNameNodeId(conf, nsId); initializeGenericKeys(conf, nsId, namenodeId); + + if (conf.get(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY) == null) { + LOG.fatal("No shared edits directory configured for namespace " + + nsId + " namenode " + namenodeId); + return false; + } + NNStorage existingStorage = null; try { - FSNamesystem fsns = FSNamesystem.loadFromDisk(conf, + Configuration confWithoutShared = new Configuration(conf); + confWithoutShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY); + FSNamesystem fsns = FSNamesystem.loadFromDisk(confWithoutShared, FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf, false)); @@ -799,11 +798,9 @@ public class NameNode { fsns.getFSImage().getEditLog().close(); fsns.getFSImage().getEditLog().initJournalsForWrite(); fsns.getFSImage().getEditLog().recoverUnclosedStreams(); - - if (copyEditLogSegmentsToSharedDir(fsns, sharedEditsDirs, - newSharedStorage, conf)) { - return true; // aborted - } + + copyEditLogSegmentsToSharedDir(fsns, sharedEditsDirs, newSharedStorage, + conf); } catch (IOException ioe) { LOG.error("Could not initialize shared edits dir", ioe); return true; // aborted @@ -821,43 +818,59 @@ public class NameNode { } return false; // did not abort } - - private static boolean copyEditLogSegmentsToSharedDir(FSNamesystem fsns, + + private static void copyEditLogSegmentsToSharedDir(FSNamesystem fsns, Collection sharedEditsDirs, NNStorage newSharedStorage, - Configuration conf) throws FileNotFoundException, IOException { + Configuration conf) throws IOException { + Preconditions.checkArgument(!sharedEditsDirs.isEmpty(), + "No shared edits specified"); // Copy edit log segments into the new shared edits dir. - for (JournalAndStream jas : fsns.getFSImage().getEditLog().getJournals()) { - FileJournalManager fjm = null; - if (!(jas.getManager() instanceof FileJournalManager)) { - LOG.error("Cannot populate shared edits dir from non-file " + - "journal manager: " + jas.getManager()); - return true; // aborted - } else { - fjm = (FileJournalManager) jas.getManager(); - } - for (EditLogFile elf : fjm.getLogFiles(fsns.getFSImage() - .getMostRecentCheckpointTxId())) { - File editLogSegment = elf.getFile(); - for (URI sharedEditsUri : sharedEditsDirs) { - StorageDirectory sharedEditsDir = newSharedStorage - .getStorageDirectory(sharedEditsUri); - File targetFile = new File(sharedEditsDir.getCurrentDir(), - editLogSegment.getName()); - if (!targetFile.exists()) { - InputStream in = null; - OutputStream out = null; - try { - in = new FileInputStream(editLogSegment); - out = new AtomicFileOutputStream(targetFile); - IOUtils.copyBytes(in, out, conf); - } finally { - IOUtils.cleanup(LOG, in, out); - } - } + List sharedEditsUris = new ArrayList(sharedEditsDirs); + FSEditLog newSharedEditLog = new FSEditLog(conf, newSharedStorage, + sharedEditsUris); + newSharedEditLog.initJournalsForWrite(); + newSharedEditLog.recoverUnclosedStreams(); + + FSEditLog sourceEditLog = fsns.getFSImage().editLog; + + long fromTxId = fsns.getFSImage().getMostRecentCheckpointTxId(); + Collection streams = sourceEditLog.selectInputStreams( + fromTxId+1, 0); + + // Set the nextTxid to the CheckpointTxId+1 + newSharedEditLog.setNextTxId(fromTxId + 1); + + // Copy all edits after last CheckpointTxId to shared edits dir + for (EditLogInputStream stream : streams) { + LOG.debug("Beginning to copy stream " + stream + " to shared edits"); + FSEditLogOp op; + boolean segmentOpen = false; + while ((op = stream.readOp()) != null) { + if (LOG.isTraceEnabled()) { + LOG.trace("copying op: " + op); + } + if (!segmentOpen) { + newSharedEditLog.startLogSegment(op.txid, false); + segmentOpen = true; + } + + newSharedEditLog.logEdit(op); + + if (op.opCode == FSEditLogOpCodes.OP_END_LOG_SEGMENT) { + newSharedEditLog.logSync(); + newSharedEditLog.endCurrentLogSegment(false); + LOG.debug("ending log segment because of END_LOG_SEGMENT op in " + stream); + segmentOpen = false; } } + + if (segmentOpen) { + LOG.debug("ending log segment because of end of stream in " + stream); + newSharedEditLog.logSync(); + newSharedEditLog.endCurrentLogSegment(false); + segmentOpen = false; + } } - return false; // did not abort } private static boolean finalize(Configuration conf, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java index 40d7e0861a0..9b87eb7497f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java @@ -65,7 +65,7 @@ public abstract class HATestUtil { * @throws CouldNotCatchUpException if the standby doesn't catch up to the * active in NN_LAG_TIMEOUT milliseconds */ - static void waitForStandbyToCatchUp(NameNode active, + public static void waitForStandbyToCatchUp(NameNode active, NameNode standby) throws InterruptedException, IOException, CouldNotCatchUpException { long activeTxId = active.getNamesystem().getFSImage().getEditLog() diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java index 72110b29a84..47182d2798a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java @@ -158,6 +158,13 @@ public class TestInitializeSharedEdits { assertCanStartHaNameNodes("2"); } + @Test + public void testFailWhenNoSharedEditsSpecified() throws Exception { + Configuration confNoShared = new Configuration(conf); + confNoShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY); + assertFalse(NameNode.initializeSharedEdits(confNoShared, true)); + } + @Test public void testDontOverWriteExistingDir() { assertFalse(NameNode.initializeSharedEdits(conf, false)); From f46b6ea55c91fa7db87f4defed3d36e68b51ed1f Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Tue, 14 Aug 2012 20:56:49 +0000 Subject: [PATCH 08/18] HDFS-3718. Datanode won't shutdown because of runaway DataBlockScanner thread (Kihwal Lee via daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373090 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/server/datanode/DataNode.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5cd7f668419..041b751d8aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1472,6 +1472,9 @@ Release 0.23.3 - UNRELEASED HDFS-3553. Hftp proxy tokens are broken (daryn) + HDFS-3718. Datanode won't shutdown because of runaway DataBlockScanner + thread (Kihwal Lee via daryn) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index dc6e3bc08cd..56770cd342c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -1061,6 +1061,7 @@ public class DataNode extends Configured } } + this.shouldRun = false; shutdownPeriodicScanners(); if (infoServer != null) { @@ -1074,7 +1075,6 @@ public class DataNode extends Configured ipcServer.stop(); } - this.shouldRun = false; if (dataXceiverServer != null) { ((DataXceiverServer) this.dataXceiverServer.getRunnable()).kill(); this.dataXceiverServer.interrupt(); From f98d8eb291be364102b5c3011ce72e8f43eab389 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Tue, 14 Aug 2012 20:59:54 +0000 Subject: [PATCH 09/18] HDFS-3150. Add option for clients to contact DNs via hostname. Contributed by Eli Collins git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373094 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../apache/hadoop/hdfs/BlockReaderLocal.java | 17 +++--- .../org/apache/hadoop/hdfs/DFSClient.java | 33 +++++++++--- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++ .../apache/hadoop/hdfs/DFSInputStream.java | 16 ++++-- .../apache/hadoop/hdfs/DFSOutputStream.java | 13 ++--- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 10 ++-- .../hadoop/hdfs/protocol/DatanodeID.java | 25 ++++++++- .../ClientDatanodeProtocolTranslatorPB.java | 23 +++++--- .../hadoop/hdfs/server/datanode/DNConf.java | 6 ++- .../hadoop/hdfs/server/datanode/DataNode.java | 31 ++++++----- .../hdfs/server/datanode/DataXceiver.java | 19 +++++-- .../server/namenode/FileChecksumServlets.java | 2 +- .../src/main/resources/hdfs-default.xml | 18 ++++++- .../apache/hadoop/hdfs/MiniDFSCluster.java | 53 +++++++++++++++---- .../hadoop/hdfs/TestDFSClientRetries.java | 2 +- .../hdfs/TestDistributedFileSystem.java | 1 - .../apache/hadoop/hdfs/TestFileCreation.java | 29 ++++++++-- .../hadoop/hdfs/TestHftpFileSystem.java | 1 - .../hadoop/hdfs/TestMiniDFSCluster.java | 23 ++++++++ .../hdfs/TestShortCircuitLocalRead.java | 4 +- .../security/token/block/TestBlockToken.java | 2 +- .../server/datanode/DataNodeTestUtils.java | 9 ++-- .../impl/TestInterDatanodeProtocol.java | 40 +++++++++++--- 24 files changed, 291 insertions(+), 92 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 041b751d8aa..ae2d2662586 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -211,6 +211,8 @@ Branch-2 ( Unreleased changes ) HDFS-3637. Add support for encrypting the DataTransferProtocol. (atm) + HDFS-3150. Add option for clients to contact DNs via hostname. (eli) + IMPROVEMENTS HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java index 6db6e75198b..7d4fb7a0610 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java @@ -86,11 +86,11 @@ class BlockReaderLocal implements BlockReader { } private synchronized ClientDatanodeProtocol getDatanodeProxy( - DatanodeInfo node, Configuration conf, int socketTimeout) - throws IOException { + DatanodeInfo node, Configuration conf, int socketTimeout, + boolean connectToDnViaHostname) throws IOException { if (proxy == null) { proxy = DFSUtil.createClientDatanodeProtocolProxy(node, conf, - socketTimeout); + socketTimeout, connectToDnViaHostname); } return proxy; } @@ -156,14 +156,16 @@ class BlockReaderLocal implements BlockReader { */ static BlockReaderLocal newBlockReader(Configuration conf, String file, ExtendedBlock blk, Token token, DatanodeInfo node, - int socketTimeout, long startOffset, long length) throws IOException { + int socketTimeout, long startOffset, long length, + boolean connectToDnViaHostname) throws IOException { LocalDatanodeInfo localDatanodeInfo = getLocalDatanodeInfo(node .getIpcPort()); // check the cache first BlockLocalPathInfo pathinfo = localDatanodeInfo.getBlockLocalPathInfo(blk); if (pathinfo == null) { - pathinfo = getBlockPathInfo(blk, node, conf, socketTimeout, token); + pathinfo = getBlockPathInfo(blk, node, conf, socketTimeout, token, + connectToDnViaHostname); } // check to see if the file exists. It may so happen that the @@ -241,11 +243,12 @@ class BlockReaderLocal implements BlockReader { private static BlockLocalPathInfo getBlockPathInfo(ExtendedBlock blk, DatanodeInfo node, Configuration conf, int timeout, - Token token) throws IOException { + Token token, boolean connectToDnViaHostname) + throws IOException { LocalDatanodeInfo localDatanodeInfo = getLocalDatanodeInfo(node.getIpcPort()); BlockLocalPathInfo pathinfo = null; ClientDatanodeProtocol proxy = localDatanodeInfo.getDatanodeProxy(node, - conf, timeout); + conf, timeout, connectToDnViaHostname); try { // make RPC to local datanode to find local pathnames of blocks pathinfo = proxy.getBlockLocalPathInfo(blk, token); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 81101e614b3..a126c046789 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -49,6 +49,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_DEF import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT; import java.io.BufferedOutputStream; import java.io.DataInputStream; @@ -213,6 +215,7 @@ public class DFSClient implements java.io.Closeable { final String taskId; final FsPermission uMask; final boolean useLegacyBlockReader; + final boolean connectToDnViaHostname; Conf(Configuration conf) { maxFailoverAttempts = conf.getInt( @@ -263,6 +266,8 @@ public class DFSClient implements java.io.Closeable { useLegacyBlockReader = conf.getBoolean( DFS_CLIENT_USE_LEGACY_BLOCKREADER, DFS_CLIENT_USE_LEGACY_BLOCKREADER_DEFAULT); + connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME, + DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT); } private int getChecksumType(Configuration conf) { @@ -473,6 +478,14 @@ public class DFSClient implements java.io.Closeable { return clientName; } + /** + * @return whether the client should use hostnames instead of IPs + * when connecting to DataNodes + */ + boolean connectToDnViaHostname() { + return dfsClientConf.connectToDnViaHostname; + } + void checkOpen() throws IOException { if (!clientRunning) { IOException result = new IOException("Filesystem closed"); @@ -729,12 +742,12 @@ public class DFSClient implements java.io.Closeable { */ static BlockReader getLocalBlockReader(Configuration conf, String src, ExtendedBlock blk, Token accessToken, - DatanodeInfo chosenNode, int socketTimeout, long offsetIntoBlock) - throws InvalidToken, IOException { + DatanodeInfo chosenNode, int socketTimeout, long offsetIntoBlock, + boolean connectToDnViaHostname) throws InvalidToken, IOException { try { return BlockReaderLocal.newBlockReader(conf, src, blk, accessToken, chosenNode, socketTimeout, offsetIntoBlock, blk.getNumBytes() - - offsetIntoBlock); + - offsetIntoBlock, connectToDnViaHostname); } catch (RemoteException re) { throw re.unwrapRemoteException(InvalidToken.class, AccessControlException.class); @@ -1425,7 +1438,8 @@ public class DFSClient implements java.io.Closeable { public MD5MD5CRC32FileChecksum getFileChecksum(String src) throws IOException { checkOpen(); return getFileChecksum(src, namenode, socketFactory, - dfsClientConf.socketTimeout, getDataEncryptionKey()); + dfsClientConf.socketTimeout, getDataEncryptionKey(), + dfsClientConf.connectToDnViaHostname); } @InterfaceAudience.Private @@ -1471,7 +1485,8 @@ public class DFSClient implements java.io.Closeable { */ public static MD5MD5CRC32FileChecksum getFileChecksum(String src, ClientProtocol namenode, SocketFactory socketFactory, int socketTimeout, - DataEncryptionKey encryptionKey) throws IOException { + DataEncryptionKey encryptionKey, boolean connectToDnViaHostname) + throws IOException { //get all block locations LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE); if (null == blockLocations) { @@ -1509,9 +1524,11 @@ public class DFSClient implements java.io.Closeable { try { //connect to a datanode sock = socketFactory.createSocket(); - NetUtils.connect(sock, - NetUtils.createSocketAddr(datanodes[j].getXferAddr()), - timeout); + String dnAddr = datanodes[j].getXferAddr(connectToDnViaHostname); + if (LOG.isDebugEnabled()) { + LOG.debug("Connecting to datanode " + dnAddr); + } + NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout); sock.setSoTimeout(timeout); OutputStream unbufOut = NetUtils.getOutputStream(sock); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 48e9297005d..222511eba6c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -52,6 +52,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_CLIENT_WRITE_REPLACE_DATANODE_ON_FAILURE_POLICY_DEFAULT = "DEFAULT"; public static final String DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY = "dfs.client.socketcache.capacity"; public static final int DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT = 16; + public static final String DFS_CLIENT_USE_DN_HOSTNAME = "dfs.client.use.datanode.hostname"; + public static final boolean DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT = false; // HA related configuration public static final String DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX = "dfs.client.failover.proxy.provider"; @@ -81,6 +83,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_DATANODE_SYNC_BEHIND_WRITES_DEFAULT = false; public static final String DFS_DATANODE_DROP_CACHE_BEHIND_READS_KEY = "dfs.datanode.drop.cache.behind.reads"; public static final boolean DFS_DATANODE_DROP_CACHE_BEHIND_READS_DEFAULT = false; + public static final String DFS_DATANODE_USE_DN_HOSTNAME = "dfs.datanode.use.datanode.hostname"; + public static final boolean DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT = false; public static final String DFS_NAMENODE_HTTP_PORT_KEY = "dfs.http.port"; public static final int DFS_NAMENODE_HTTP_PORT_DEFAULT = 50070; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 91026bea9a5..ee4dc8918c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -199,7 +199,8 @@ public class DFSInputStream extends FSInputStream implements ByteBufferReadable try { cdp = DFSUtil.createClientDatanodeProtocolProxy( - datanode, dfsClient.conf, dfsClient.getConf().socketTimeout, locatedblock); + datanode, dfsClient.conf, dfsClient.getConf().socketTimeout, + dfsClient.getConf().connectToDnViaHostname, locatedblock); final long n = cdp.getReplicaVisibleLength(locatedblock.getBlock()); @@ -716,8 +717,12 @@ public class DFSInputStream extends FSInputStream implements ByteBufferReadable DatanodeInfo[] nodes = block.getLocations(); try { DatanodeInfo chosenNode = bestNode(nodes, deadNodes); - InetSocketAddress targetAddr = - NetUtils.createSocketAddr(chosenNode.getXferAddr()); + final String dnAddr = + chosenNode.getXferAddr(dfsClient.connectToDnViaHostname()); + if (DFSClient.LOG.isDebugEnabled()) { + DFSClient.LOG.debug("Connecting to datanode " + dnAddr); + } + InetSocketAddress targetAddr = NetUtils.createSocketAddr(dnAddr); return new DNAddrPair(chosenNode, targetAddr); } catch (IOException ie) { String blockInfo = block.getBlock() + " file=" + src; @@ -875,7 +880,8 @@ public class DFSInputStream extends FSInputStream implements ByteBufferReadable if (dfsClient.shouldTryShortCircuitRead(dnAddr)) { return DFSClient.getLocalBlockReader(dfsClient.conf, src, block, - blockToken, chosenNode, dfsClient.hdfsTimeout, startOffset); + blockToken, chosenNode, dfsClient.hdfsTimeout, startOffset, + dfsClient.connectToDnViaHostname()); } IOException err = null; @@ -1183,7 +1189,7 @@ public class DFSInputStream extends FSInputStream implements ByteBufferReadable throw new IOException("No live nodes contain current block"); } - /** Utility class to encapsulate data node info and its ip address. */ + /** Utility class to encapsulate data node info and its address. */ static class DNAddrPair { DatanodeInfo info; InetSocketAddress addr; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 50785098bfa..a04d8af4c35 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -1100,7 +1100,7 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable { if (ie instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) { DFSClient.LOG.info("Will fetch a new encryption key and retry, " + "encryption key was invalid when connecting to " - + nodes[0].getXferAddr() + " : " + ie); + + nodes[0] + " : " + ie); // The encryption key used is invalid. refetchEncryptionKey--; dfsClient.clearDataEncryptionKey(); @@ -1112,7 +1112,8 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable { // find the datanode that matches if (firstBadLink.length() != 0) { for (int i = 0; i < nodes.length; i++) { - if (nodes[i].getXferAddr().equals(firstBadLink)) { + // NB: Unconditionally using the xfer addr w/o hostname + if (firstBadLink.equals(nodes[i].getXferAddr())) { errorIndex = i; break; } @@ -1216,11 +1217,11 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable { */ static Socket createSocketForPipeline(final DatanodeInfo first, final int length, final DFSClient client) throws IOException { - if(DFSClient.LOG.isDebugEnabled()) { - DFSClient.LOG.debug("Connecting to datanode " + first); + final String dnAddr = first.getXferAddr(client.connectToDnViaHostname()); + if (DFSClient.LOG.isDebugEnabled()) { + DFSClient.LOG.debug("Connecting to datanode " + dnAddr); } - final InetSocketAddress isa = - NetUtils.createSocketAddr(first.getXferAddr()); + final InetSocketAddress isa = NetUtils.createSocketAddr(dnAddr); final Socket sock = client.socketFactory.createSocket(); final int timeout = client.getDatanodeReadTimeout(length); NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(), timeout); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 0246620e905..8d59ae65886 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -841,17 +841,17 @@ public class DFSUtil { /** Create a {@link ClientDatanodeProtocol} proxy */ public static ClientDatanodeProtocol createClientDatanodeProtocolProxy( DatanodeID datanodeid, Configuration conf, int socketTimeout, - LocatedBlock locatedBlock) throws IOException { + boolean connectToDnViaHostname, LocatedBlock locatedBlock) throws IOException { return new ClientDatanodeProtocolTranslatorPB(datanodeid, conf, socketTimeout, - locatedBlock); + connectToDnViaHostname, locatedBlock); } /** Create {@link ClientDatanodeProtocol} proxy using kerberos ticket */ static ClientDatanodeProtocol createClientDatanodeProtocolProxy( - DatanodeID datanodeid, Configuration conf, int socketTimeout) - throws IOException { + DatanodeID datanodeid, Configuration conf, int socketTimeout, + boolean connectToDnViaHostname) throws IOException { return new ClientDatanodeProtocolTranslatorPB( - datanodeid, conf, socketTimeout); + datanodeid, conf, socketTimeout, connectToDnViaHostname); } /** Create a {@link ClientDatanodeProtocol} proxy */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java index 4ed9f5666ac..1a923153402 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java @@ -104,7 +104,7 @@ public class DatanodeID implements Comparable { /** * @return IP:ipcPort string */ - public String getIpcAddr() { + private String getIpcAddr() { return ipAddr + ":" + ipcPort; } @@ -122,6 +122,29 @@ public class DatanodeID implements Comparable { return hostName + ":" + xferPort; } + /** + * @return hostname:ipcPort + */ + private String getIpcAddrWithHostname() { + return hostName + ":" + ipcPort; + } + + /** + * @param useHostname true to use the DN hostname, use the IP otherwise + * @return name:xferPort + */ + public String getXferAddr(boolean useHostname) { + return useHostname ? getXferAddrWithHostname() : getXferAddr(); + } + + /** + * @param useHostname true to use the DN hostname, use the IP otherwise + * @return name:ipcPort + */ + public String getIpcAddr(boolean useHostname) { + return useHostname ? getIpcAddrWithHostname() : getIpcAddr(); + } + /** * @return data storage ID. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java index d28dbff10b0..1f6c9b113ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java @@ -73,10 +73,10 @@ public class ClientDatanodeProtocolTranslatorPB implements RefreshNamenodesRequestProto.newBuilder().build(); public ClientDatanodeProtocolTranslatorPB(DatanodeID datanodeid, - Configuration conf, int socketTimeout, LocatedBlock locatedBlock) - throws IOException { + Configuration conf, int socketTimeout, boolean connectToDnViaHostname, + LocatedBlock locatedBlock) throws IOException { rpcProxy = createClientDatanodeProtocolProxy( datanodeid, conf, - socketTimeout, locatedBlock); + socketTimeout, connectToDnViaHostname, locatedBlock); } public ClientDatanodeProtocolTranslatorPB(InetSocketAddress addr, @@ -90,11 +90,17 @@ public class ClientDatanodeProtocolTranslatorPB implements * @param datanodeid Datanode to connect to. * @param conf Configuration. * @param socketTimeout Socket timeout to use. + * @param connectToDnViaHostname connect to the Datanode using its hostname * @throws IOException */ public ClientDatanodeProtocolTranslatorPB(DatanodeID datanodeid, - Configuration conf, int socketTimeout) throws IOException { - InetSocketAddress addr = NetUtils.createSocketAddr(datanodeid.getIpcAddr()); + Configuration conf, int socketTimeout, boolean connectToDnViaHostname) + throws IOException { + final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname); + InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr); + if (LOG.isDebugEnabled()) { + LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr); + } rpcProxy = createClientDatanodeProtocolProxy(addr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), socketTimeout); @@ -102,10 +108,11 @@ public class ClientDatanodeProtocolTranslatorPB implements static ClientDatanodeProtocolPB createClientDatanodeProtocolProxy( DatanodeID datanodeid, Configuration conf, int socketTimeout, - LocatedBlock locatedBlock) throws IOException { - InetSocketAddress addr = NetUtils.createSocketAddr(datanodeid.getIpcAddr()); + boolean connectToDnViaHostname, LocatedBlock locatedBlock) throws IOException { + final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname); + InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr); if (LOG.isDebugEnabled()) { - LOG.debug("ClientDatanodeProtocol addr=" + addr); + LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr); } // Since we're creating a new UserGroupInformation here, we know that no diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java index 3f37a7eea38..d4b0ffd1f85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java @@ -55,7 +55,7 @@ class DNConf { final boolean dropCacheBehindReads; final boolean syncOnClose; final boolean encryptDataTransfer; - + final boolean connectToDnViaHostname; final long readaheadLength; final long heartBeatInterval; @@ -97,7 +97,9 @@ class DNConf { dropCacheBehindReads = conf.getBoolean( DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_READS_KEY, DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_READS_DEFAULT); - + connectToDnViaHostname = conf.getBoolean( + DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, + DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); this.blockReportInterval = conf.getLong(DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 56770cd342c..a456d133d4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -276,6 +276,7 @@ public class DataNode extends Configured private Configuration conf; private final String userWithLocalPathAccess; + private boolean connectToDnViaHostname; ReadaheadPool readaheadPool; /** @@ -296,8 +297,11 @@ public class DataNode extends Configured final SecureResources resources) throws IOException { super(conf); - this.userWithLocalPathAccess = conf - .get(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY); + this.userWithLocalPathAccess = + conf.get(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY); + this.connectToDnViaHostname = conf.getBoolean( + DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, + DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); try { hostName = getHostName(conf); LOG.info("Configured hostname is " + hostName); @@ -878,7 +882,7 @@ public class DataNode extends Configured /** * NB: The datanode can perform data transfer on the streaming * address however clients are given the IPC IP address for data - * transfer, and that may be be a different address. + * transfer, and that may be a different address. * * @return socket address for data transfer */ @@ -925,12 +929,12 @@ public class DataNode extends Configured } public static InterDatanodeProtocol createInterDataNodeProtocolProxy( - DatanodeID datanodeid, final Configuration conf, final int socketTimeout) - throws IOException { - final InetSocketAddress addr = - NetUtils.createSocketAddr(datanodeid.getIpcAddr()); - if (InterDatanodeProtocol.LOG.isDebugEnabled()) { - InterDatanodeProtocol.LOG.debug("InterDatanodeProtocol addr=" + addr); + DatanodeID datanodeid, final Configuration conf, final int socketTimeout, + final boolean connectToDnViaHostname) throws IOException { + final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname); + final InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr); + if (LOG.isDebugEnabled()) { + LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr); } final UserGroupInformation loginUgi = UserGroupInformation.getLoginUser(); try { @@ -1386,8 +1390,11 @@ public class DataNode extends Configured final boolean isClient = clientname.length() > 0; try { - InetSocketAddress curTarget = - NetUtils.createSocketAddr(targets[0].getXferAddr()); + final String dnAddr = targets[0].getXferAddr(connectToDnViaHostname); + InetSocketAddress curTarget = NetUtils.createSocketAddr(dnAddr); + if (LOG.isDebugEnabled()) { + LOG.debug("Connecting to datanode " + dnAddr); + } sock = newSocket(); NetUtils.connect(sock, curTarget, dnConf.socketTimeout); sock.setSoTimeout(targets.length * dnConf.socketTimeout); @@ -1843,7 +1850,7 @@ public class DataNode extends Configured DatanodeRegistration bpReg = bpos.bpRegistration; InterDatanodeProtocol datanode = bpReg.equals(id)? this: DataNode.createInterDataNodeProtocolProxy(id, getConf(), - dnConf.socketTimeout); + dnConf.socketTimeout, dnConf.connectToDnViaHostname); ReplicaRecoveryInfo info = callInitReplicaRecovery(datanode, rBlock); if (info != null && info.getGenerationStamp() >= block.getGenerationStamp() && diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index 8547df94873..d0c5aaff892 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -86,7 +86,7 @@ class DataXceiver extends Receiver implements Runnable { private final DataNode datanode; private final DNConf dnConf; private final DataXceiverServer dataXceiverServer; - + private final boolean connectToDnViaHostname; private long opStartTime; //the start time of receiving an Op private final SocketInputWrapper socketIn; private OutputStream socketOut; @@ -113,6 +113,7 @@ class DataXceiver extends Receiver implements Runnable { this.isLocal = s.getInetAddress().equals(s.getLocalAddress()); this.datanode = datanode; this.dataXceiverServer = dataXceiverServer; + this.connectToDnViaHostname = datanode.getDnConf().connectToDnViaHostname; remoteAddress = s.getRemoteSocketAddress().toString(); localAddress = s.getLocalSocketAddress().toString(); @@ -404,7 +405,10 @@ class DataXceiver extends Receiver implements Runnable { if (targets.length > 0) { InetSocketAddress mirrorTarget = null; // Connect to backup machine - mirrorNode = targets[0].getXferAddr(); + mirrorNode = targets[0].getXferAddr(connectToDnViaHostname); + if (LOG.isDebugEnabled()) { + LOG.debug("Connecting to datanode " + mirrorNode); + } mirrorTarget = NetUtils.createSocketAddr(mirrorNode); mirrorSock = datanode.newSocket(); try { @@ -457,7 +461,8 @@ class DataXceiver extends Receiver implements Runnable { if (isClient) { BlockOpResponseProto.newBuilder() .setStatus(ERROR) - .setFirstBadLink(mirrorNode) + // NB: Unconditionally using the xfer addr w/o hostname + .setFirstBadLink(targets[0].getXferAddr()) .build() .writeDelimitedTo(replyOut); replyOut.flush(); @@ -729,8 +734,11 @@ class DataXceiver extends Receiver implements Runnable { try { // get the output stream to the proxy - InetSocketAddress proxyAddr = - NetUtils.createSocketAddr(proxySource.getXferAddr()); + final String dnAddr = proxySource.getXferAddr(connectToDnViaHostname); + if (LOG.isDebugEnabled()) { + LOG.debug("Connecting to datanode " + dnAddr); + } + InetSocketAddress proxyAddr = NetUtils.createSocketAddr(dnAddr); proxySock = datanode.newSocket(); NetUtils.connect(proxySock, proxyAddr, dnConf.socketTimeout); proxySock.setSoTimeout(dnConf.socketTimeout); @@ -891,6 +899,7 @@ class DataXceiver extends Receiver implements Runnable { if (mode == BlockTokenSecretManager.AccessMode.WRITE) { DatanodeRegistration dnR = datanode.getDNRegistrationForBP(blk.getBlockPoolId()); + // NB: Unconditionally using the xfer addr w/o hostname resp.setFirstBadLink(dnR.getXferAddr()); } resp.build().writeDelimitedTo(out); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java index 39f9094e684..fd3b439355a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java @@ -127,7 +127,7 @@ public class FileChecksumServlets { datanode, conf, getUGI(request, conf)); final ClientProtocol nnproxy = dfs.getNamenode(); final MD5MD5CRC32FileChecksum checksum = DFSClient.getFileChecksum( - path, nnproxy, socketFactory, socketTimeout, dfs.getDataEncryptionKey()); + path, nnproxy, socketFactory, socketTimeout, dfs.getDataEncryptionKey(), false); MD5MD5CRC32FileChecksum.write(xml, checksum); } catch(IOException ioe) { writeXml(ioe, path, xml); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 95e2f8c0814..16510168445 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -53,7 +53,7 @@ dfs.datanode.address 0.0.0.0:50010 - The address where the datanode server will listen to. + The datanode server address and port for data transfer. If the port is 0 then the server will start on a free port. @@ -920,6 +920,22 @@ + + dfs.client.use.datanode.hostname + false + Whether clients should use datanode hostnames when + connecting to datanodes. + + + + + dfs.datanode.use.datanode.hostname + false + Whether datanodes should use datanode hostnames when + connecting to other datanodes for data transfer. + + + dfs.client.local.interfaces diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index a5cc0b293b9..78483466792 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -145,6 +145,7 @@ public class MiniDFSCluster { private boolean setupHostsFile = false; private MiniDFSNNTopology nnTopology = null; private boolean checkExitOnShutdown = true; + private boolean checkDataNodeHostConfig = false; public Builder(Configuration conf) { this.conf = conf; @@ -262,6 +263,14 @@ public class MiniDFSCluster { return this; } + /** + * Default: false + */ + public Builder checkDataNodeHostConfig(boolean val) { + this.checkDataNodeHostConfig = val; + return this; + } + /** * Default: null */ @@ -326,7 +335,8 @@ public class MiniDFSCluster { builder.waitSafeMode, builder.setupHostsFile, builder.nnTopology, - builder.checkExitOnShutdown); + builder.checkExitOnShutdown, + builder.checkDataNodeHostConfig); } public class DataNodeProperties { @@ -563,7 +573,7 @@ public class MiniDFSCluster { manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs, operation, racks, hosts, simulatedCapacities, null, true, false, - MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true); + MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false); } private void initMiniDFSCluster( @@ -573,7 +583,8 @@ public class MiniDFSCluster { boolean manageDataDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, String clusterId, boolean waitSafeMode, boolean setupHostsFile, - MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown) + MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown, + boolean checkDataNodeHostConfig) throws IOException { ExitUtil.disableSystemExit(); @@ -630,7 +641,7 @@ public class MiniDFSCluster { // Start the DataNodes startDataNodes(conf, numDataNodes, manageDataDfsDirs, operation, racks, - hosts, simulatedCapacities, setupHostsFile); + hosts, simulatedCapacities, setupHostsFile, false, checkDataNodeHostConfig); waitClusterUp(); //make sure ProxyUsers uses the latest conf ProxyUsers.refreshSuperUserGroupsConfiguration(conf); @@ -982,7 +993,21 @@ public class MiniDFSCluster { long[] simulatedCapacities, boolean setupHostsFile) throws IOException { startDataNodes(conf, numDataNodes, manageDfsDirs, operation, racks, hosts, - simulatedCapacities, setupHostsFile, false); + simulatedCapacities, setupHostsFile, false, false); + } + + /** + * @see MiniDFSCluster#startDataNodes(Configuration, int, boolean, StartupOption, + * String[], String[], long[], boolean, boolean, boolean) + */ + public synchronized void startDataNodes(Configuration conf, int numDataNodes, + boolean manageDfsDirs, StartupOption operation, + String[] racks, String[] hosts, + long[] simulatedCapacities, + boolean setupHostsFile, + boolean checkDataNodeAddrConfig) throws IOException { + startDataNodes(conf, numDataNodes, manageDfsDirs, operation, racks, hosts, + simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false); } /** @@ -1008,19 +1033,25 @@ public class MiniDFSCluster { * @param simulatedCapacities array of capacities of the simulated data nodes * @param setupHostsFile add new nodes to dfs hosts files * @param checkDataNodeAddrConfig if true, only set DataNode port addresses if not already set in config + * @param checkDataNodeHostConfig if true, only set DataNode hostname key if not already set in config * * @throws IllegalStateException if NameNode has been shutdown */ public synchronized void startDataNodes(Configuration conf, int numDataNodes, - boolean manageDfsDirs, StartupOption operation, - String[] racks, String[] hosts, - long[] simulatedCapacities, - boolean setupHostsFile, - boolean checkDataNodeAddrConfig) throws IOException { + boolean manageDfsDirs, StartupOption operation, + String[] racks, String[] hosts, + long[] simulatedCapacities, + boolean setupHostsFile, + boolean checkDataNodeAddrConfig, + boolean checkDataNodeHostConfig) throws IOException { if (operation == StartupOption.RECOVER) { return; } - conf.set(DFS_DATANODE_HOST_NAME_KEY, "127.0.0.1"); + if (checkDataNodeHostConfig) { + conf.setIfUnset(DFS_DATANODE_HOST_NAME_KEY, "127.0.0.1"); + } else { + conf.set(DFS_DATANODE_HOST_NAME_KEY, "127.0.0.1"); + } int curDatanodesNum = dataNodes.size(); // for mincluster's the default initialDelay for BRs is 0 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index a06263afd22..9500ec2bce9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -768,7 +768,7 @@ public class TestDFSClientRetries { try { proxy = DFSUtil.createClientDatanodeProtocolProxy( - fakeDnId, conf, 500, fakeBlock); + fakeDnId, conf, 500, false, fakeBlock); proxy.getReplicaVisibleLength(new ExtendedBlock("bpid", 1)); fail ("Did not get expected exception: SocketTimeoutException"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 2768b0c5d4e..825348d11cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -417,7 +417,6 @@ public class TestDistributedFileSystem { final Configuration conf = getTestConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); - conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost"); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem hdfs = cluster.getFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java index dbf22b4392f..9391c00d658 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java @@ -171,7 +171,14 @@ public class TestFileCreation { @Test public void testFileCreation() throws IOException { - checkFileCreation(null); + checkFileCreation(null, false); + } + + /** Same test but the client should use DN hostnames */ + @Test + public void testFileCreationUsingHostname() throws IOException { + assumeTrue(System.getProperty("os.name").startsWith("Linux")); + checkFileCreation(null, true); } /** Same test but the client should bind to a local interface */ @@ -180,10 +187,10 @@ public class TestFileCreation { assumeTrue(System.getProperty("os.name").startsWith("Linux")); // The mini cluster listens on the loopback so we can use it here - checkFileCreation("lo"); + checkFileCreation("lo", false); try { - checkFileCreation("bogus-interface"); + checkFileCreation("bogus-interface", false); fail("Able to specify a bogus interface"); } catch (UnknownHostException e) { assertEquals("No such interface bogus-interface", e.getMessage()); @@ -193,16 +200,28 @@ public class TestFileCreation { /** * Test if file creation and disk space consumption works right * @param netIf the local interface, if any, clients should use to access DNs + * @param useDnHostname whether the client should contact DNs by hostname */ - public void checkFileCreation(String netIf) throws IOException { + public void checkFileCreation(String netIf, boolean useDnHostname) + throws IOException { Configuration conf = new HdfsConfiguration(); if (netIf != null) { conf.set(DFSConfigKeys.DFS_CLIENT_LOCAL_INTERFACES, netIf); } + conf.setBoolean(DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME, useDnHostname); + if (useDnHostname) { + // Since the mini cluster only listens on the loopback we have to + // ensure the hostname used to access DNs maps to the loopback. We + // do this by telling the DN to advertise localhost as its hostname + // instead of the default hostname. + conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost"); + } if (simulatedStorage) { SimulatedFSDataset.setFactory(conf); } - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .checkDataNodeHostConfig(true) + .build(); FileSystem fs = cluster.getFileSystem(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java index 5e20d46e2c5..3fa5eaad0a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java @@ -92,7 +92,6 @@ public class TestHftpFileSystem { RAN.setSeed(seed); config = new Configuration(); - config.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost"); cluster = new MiniDFSCluster.Builder(config).numDataNodes(2).build(); hdfs = cluster.getFileSystem(); blockPoolId = cluster.getNamesystem().getBlockPoolId(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java index a7fd82aea0b..99f30dd73ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; import java.io.File; @@ -41,6 +42,7 @@ public class TestMiniDFSCluster { private static final String CLUSTER_2 = "cluster2"; private static final String CLUSTER_3 = "cluster3"; private static final String CLUSTER_4 = "cluster4"; + private static final String CLUSTER_5 = "cluster5"; protected String testDataPath; protected File testDataDir; @Before @@ -125,4 +127,25 @@ public class TestMiniDFSCluster { } } } + + /** MiniDFSCluster should not clobber dfs.datanode.hostname if requested */ + @Test(timeout=100000) + public void testClusterSetDatanodeHostname() throws Throwable { + assumeTrue(System.getProperty("os.name").startsWith("Linux")); + Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "MYHOST"); + File testDataCluster5 = new File(testDataPath, CLUSTER_5); + String c5Path = testDataCluster5.getAbsolutePath(); + conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c5Path); + MiniDFSCluster cluster5 = new MiniDFSCluster.Builder(conf) + .numDataNodes(1) + .checkDataNodeHostConfig(true) + .build(); + try { + assertEquals("DataNode hostname config not respected", "MYHOST", + cluster5.getDataNodes().get(0).getDatanodeId().getHostName()); + } finally { + MiniDFSCluster.shutdownCluster(cluster5); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java index 378969adf9a..8b8879d5305 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java @@ -246,7 +246,7 @@ public class TestShortCircuitLocalRead { @Override public ClientDatanodeProtocol run() throws Exception { return DFSUtil.createClientDatanodeProtocolProxy(dnInfo, conf, - 60000); + 60000, false); } }); @@ -264,7 +264,7 @@ public class TestShortCircuitLocalRead { @Override public ClientDatanodeProtocol run() throws Exception { return DFSUtil.createClientDatanodeProtocolProxy(dnInfo, conf, - 60000); + 60000, false); } }); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java index 4b9f2f4faaa..a9b55621887 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java @@ -304,7 +304,7 @@ public class TestBlockToken { long endTime = Time.now() + 3000; while (Time.now() < endTime) { proxy = DFSUtil.createClientDatanodeProtocolProxy(fakeDnId, conf, 1000, - fakeBlock); + false, fakeBlock); assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3)); if (proxy != null) { RPC.stopProxy(proxy); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java index 2f9ed12099e..55b4bf57db6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java @@ -105,10 +105,13 @@ public class DataNodeTestUtils { } public static InterDatanodeProtocol createInterDatanodeProtocolProxy( - DataNode dn, DatanodeID datanodeid, final Configuration conf - ) throws IOException { + DataNode dn, DatanodeID datanodeid, final Configuration conf, + boolean connectToDnViaHostname) throws IOException { + if (connectToDnViaHostname != dn.getDnConf().connectToDnViaHostname) { + throw new AssertionError("Unexpected DN hostname configuration"); + } return DataNode.createInterDataNodeProtocolProxy(datanodeid, conf, - dn.getDnConf().socketTimeout); + dn.getDnConf().socketTimeout, dn.getDnConf().connectToDnViaHostname); } public static void shutdownBlockScanner(DataNode dn) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java index 8ff6cb88866..b1733efa9f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java @@ -29,6 +29,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSClientAdapter; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -59,6 +60,8 @@ import org.apache.hadoop.net.NetUtils; import org.junit.Assert; import org.junit.Test; +import static org.junit.Assume.assumeTrue; + /** * This tests InterDataNodeProtocol for block handling. */ @@ -125,17 +128,42 @@ public class TestInterDatanodeProtocol { return blocks.get(blocks.size() - 1); } + /** Test block MD access via a DN */ + @Test + public void testBlockMetaDataInfo() throws Exception { + checkBlockMetaDataInfo(false); + } + + /** The same as above, but use hostnames for DN<->DN communication */ + @Test + public void testBlockMetaDataInfoWithHostname() throws Exception { + assumeTrue(System.getProperty("os.name").startsWith("Linux")); + checkBlockMetaDataInfo(true); + } + /** * The following test first creates a file. * It verifies the block information from a datanode. - * Then, it updates the block with new information and verifies again. + * Then, it updates the block with new information and verifies again. + * @param useDnHostname whether DNs should connect to other DNs by hostname */ - @Test - public void testBlockMetaDataInfo() throws Exception { + private void checkBlockMetaDataInfo(boolean useDnHostname) throws Exception { MiniDFSCluster cluster = null; + conf.setBoolean(DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, useDnHostname); + if (useDnHostname) { + // Since the mini cluster only listens on the loopback we have to + // ensure the hostname used to access DNs maps to the loopback. We + // do this by telling the DN to advertise localhost as its hostname + // instead of the default hostname. + conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost"); + } + try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(3) + .checkDataNodeHostConfig(true) + .build(); cluster.waitActive(); //create a file @@ -154,7 +182,7 @@ public class TestInterDatanodeProtocol { //connect to a data node DataNode datanode = cluster.getDataNode(datanodeinfo[0].getIpcPort()); InterDatanodeProtocol idp = DataNodeTestUtils.createInterDatanodeProtocolProxy( - datanode, datanodeinfo[0], conf); + datanode, datanodeinfo[0], conf, useDnHostname); //stop block scanner, so we could compare lastScanTime DataNodeTestUtils.shutdownBlockScanner(datanode); @@ -364,7 +392,7 @@ public class TestInterDatanodeProtocol { try { proxy = DataNode.createInterDataNodeProtocolProxy( - dInfo, conf, 500); + dInfo, conf, 500, false); proxy.initReplicaRecovery(new RecoveringBlock( new ExtendedBlock("bpid", 1), null, 100)); fail ("Expected SocketTimeoutException exception, but did not get."); From f026d8bb1ba6f9b059db5526f1fb1261f818ffd0 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 15 Aug 2012 00:03:19 +0000 Subject: [PATCH 10/18] HDFS-3723. Add support -h, -help to all the commands. Contributed by Jing Zhao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373170 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../java/org/apache/hadoop/hdfs/DFSUtil.java | 68 +++++++++++++++++-- .../hadoop/hdfs/server/balancer/Balancer.java | 26 ++++--- .../hadoop/hdfs/server/datanode/DataNode.java | 22 +++--- .../hadoop/hdfs/server/namenode/NameNode.java | 51 +++++++------- .../server/namenode/SecondaryNameNode.java | 24 ++++++- .../apache/hadoop/hdfs/tools/DFSHAAdmin.java | 4 ++ .../hdfs/tools/DFSZKFailoverController.java | 4 ++ .../org/apache/hadoop/hdfs/tools/DFSck.java | 57 +++++++++------- .../hdfs/tools/DelegationTokenFetcher.java | 13 ++-- .../org/apache/hadoop/hdfs/tools/GetConf.java | 4 ++ .../apache/hadoop/hdfs/tools/GetGroups.java | 7 ++ .../hadoop/hdfs/tools/TestDFSHAAdmin.java | 16 +++-- 13 files changed, 216 insertions(+), 83 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ae2d2662586..64a4fdf568e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -114,6 +114,9 @@ Trunk (unreleased changes) HDFS-3789. JournalManager#format() should be able to throw IOException (Ivan Kelly via todd) + HDFS-3723. Add support -h, -help to all the commands. (Jing Zhao via + suresh) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 8d59ae65886..d263acd5906 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -18,8 +18,21 @@ package org.apache.hadoop.hdfs; -import static org.apache.hadoop.hdfs.DFSConfigKeys.*; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; + import java.io.IOException; +import java.io.PrintStream; import java.io.UnsupportedEncodingException; import java.net.InetSocketAddress; import java.net.URI; @@ -33,10 +46,17 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.StringTokenizer; import javax.net.SocketFactory; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -57,8 +77,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.ToolRunner; import com.google.common.base.Joiner; import com.google.common.collect.Lists; @@ -424,7 +443,6 @@ public class DFSUtil { * * @param conf configuration * @return list of InetSocketAddresses - * @throws IOException if no addresses are configured */ public static Map> getHaNnRpcAddresses( Configuration conf) { @@ -1073,4 +1091,44 @@ public class DFSUtil { return null; } } + + public static Options helpOptions = new Options(); + public static Option helpOpt = new Option("h", "help", false, + "get help information"); + + static { + helpOptions.addOption(helpOpt); + } + + /** + * Parse the arguments for commands + * + * @param args the argument to be parsed + * @param helpDescription help information to be printed out + * @param out Printer + * @param printGenericCommandUsage whether to print the + * generic command usage defined in ToolRunner + * @return true when the argument matches help option, false if not + */ + public static boolean parseHelpArgument(String[] args, + String helpDescription, PrintStream out, boolean printGenericCommandUsage) { + if (args.length == 1) { + try { + CommandLineParser parser = new PosixParser(); + CommandLine cmdLine = parser.parse(helpOptions, args); + if (cmdLine.hasOption(helpOpt.getOpt()) + || cmdLine.hasOption(helpOpt.getLongOpt())) { + // should print out the help information + out.println(helpDescription + "\n"); + if (printGenericCommandUsage) { + ToolRunner.printGenericCommandUsage(out); + } + return true; + } + } catch (ParseException pe) { + return false; + } + } + return false; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index f949c924a1e..577d73b76be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.balancer; +import static com.google.common.base.Preconditions.checkArgument; import static org.apache.hadoop.hdfs.protocol.HdfsProtoUtil.vintPrefixed; import java.io.BufferedInputStream; @@ -26,6 +27,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.PrintStream; import java.net.Socket; import java.net.URI; import java.text.DateFormat; @@ -68,7 +70,6 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations; import org.apache.hadoop.io.IOUtils; @@ -79,7 +80,6 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import static com.google.common.base.Preconditions.checkArgument; /**

The balancer is a tool that balances disk space usage on an HDFS cluster * when some datanodes become full or when new empty nodes join the cluster. @@ -189,6 +189,13 @@ public class Balancer { */ public static final int MAX_NUM_CONCURRENT_MOVES = 5; + private static final String USAGE = "Usage: java " + + Balancer.class.getSimpleName() + + "\n\t[-policy ]\tthe balancing policy: " + + BalancingPolicy.Node.INSTANCE.getName() + " or " + + BalancingPolicy.Pool.INSTANCE.getName() + + "\n\t[-threshold ]\tPercentage of disk capacity"; + private final NameNodeConnector nnc; private final BalancingPolicy policy; private final double threshold; @@ -1550,7 +1557,7 @@ public class Balancer { } } } catch(RuntimeException e) { - printUsage(); + printUsage(System.err); throw e; } } @@ -1558,13 +1565,8 @@ public class Balancer { return new Parameters(policy, threshold); } - private static void printUsage() { - System.out.println("Usage: java " + Balancer.class.getSimpleName()); - System.out.println(" [-policy ]\tthe balancing policy: " - + BalancingPolicy.Node.INSTANCE.getName() + " or " - + BalancingPolicy.Pool.INSTANCE.getName()); - System.out.println( - " [-threshold ]\tPercentage of disk capacity"); + private static void printUsage(PrintStream out) { + out.println(USAGE + "\n"); } } @@ -1573,6 +1575,10 @@ public class Balancer { * @param args Command line arguments */ public static void main(String[] args) { + if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { + System.exit(0); + } + try { System.exit(ToolRunner.run(new HdfsConfiguration(), new Cli(), args)); } catch (Throwable e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index a456d133d4a..83219cca48b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -46,6 +46,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOUR import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STARTUP_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTPS_ENABLE_KEY; +import static org.apache.hadoop.util.ExitUtil.terminate; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; @@ -55,6 +56,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.PrintStream; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; @@ -98,8 +100,8 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsProtoUtil; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage; -import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol; import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferEncryptor; +import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol; import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ClientDatanodeProtocolService; @@ -124,9 +126,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.common.Util; - -import static org.apache.hadoop.util.ExitUtil.terminate; - import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; @@ -171,9 +170,9 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.util.VersionInfo; import org.mortbay.util.ajax.JSON; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; -import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.BlockingService; /********************************************************** @@ -230,6 +229,8 @@ public class DataNode extends Configured static final Log ClientTraceLog = LogFactory.getLog(DataNode.class.getName() + ".clienttrace"); + + private static final String USAGE = "Usage: java DataNode [-rollback | -regular]"; /** * Use {@link NetUtils#createSocketAddr(String)} instead. @@ -1541,7 +1542,7 @@ public class DataNode extends Configured } if (!parseArguments(args, conf)) { - printUsage(); + printUsage(System.err); return null; } Collection dataDirs = getStorageDirs(conf); @@ -1655,9 +1656,8 @@ public class DataNode extends Configured + xmitsInProgress.get() + "}"; } - private static void printUsage() { - System.err.println("Usage: java DataNode"); - System.err.println(" [-rollback]"); + private static void printUsage(PrintStream out) { + out.println(USAGE + "\n"); } /** @@ -1742,6 +1742,10 @@ public class DataNode extends Configured } public static void main(String args[]) { + if (DFSUtil.parseHelpArgument(args, DataNode.USAGE, System.out, true)) { + System.exit(0); + } + secureMain(args, null); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 20e7aafba9f..2df693b3c4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.IOException; +import java.io.PrintStream; import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; @@ -38,6 +39,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Trash; import static org.apache.hadoop.hdfs.DFSConfigKeys.*; +import static org.apache.hadoop.util.ExitUtil.terminate; +import static org.apache.hadoop.util.ToolRunner.confirmPrompt; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; @@ -69,12 +72,9 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol; import org.apache.hadoop.tools.GetUserMappingsProtocol; +import org.apache.hadoop.util.ExitUtil.ExitException; import org.apache.hadoop.util.ServicePlugin; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.ExitUtil.ExitException; - -import static org.apache.hadoop.util.ExitUtil.terminate; -import static org.apache.hadoop.util.ToolRunner.confirmPrompt; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -188,6 +188,22 @@ public class NameNode { DFS_HA_AUTO_FAILOVER_ENABLED_KEY }; + private static final String USAGE = "Usage: java NameNode [" + + StartupOption.BACKUP.getName() + "] | [" + + StartupOption.CHECKPOINT.getName() + "] | [" + + StartupOption.FORMAT.getName() + " [" + + StartupOption.CLUSTERID.getName() + " cid ] [" + + StartupOption.FORCE.getName() + "] [" + + StartupOption.NONINTERACTIVE.getName() + "] ] | [" + + StartupOption.UPGRADE.getName() + "] | [" + + StartupOption.ROLLBACK.getName() + "] | [" + + StartupOption.FINALIZE.getName() + "] | [" + + StartupOption.IMPORT.getName() + "] | [" + + StartupOption.INITIALIZESHAREDEDITS.getName() + "] | [" + + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" + + StartupOption.RECOVER.getName() + " [ " + StartupOption.FORCE.getName() + + " ] ]"; + public long getProtocolVersion(String protocol, long clientVersion) throws IOException { if (protocol.equals(ClientProtocol.class.getName())) { @@ -895,25 +911,8 @@ public class NameNode { return false; } - private static void printUsage() { - System.err.println( - "Usage: java NameNode [" + - StartupOption.BACKUP.getName() + "] | [" + - StartupOption.CHECKPOINT.getName() + "] | [" + - StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() + - " cid ] [" + StartupOption.FORCE.getName() + "] [" + - StartupOption.NONINTERACTIVE.getName() + "] ] | [" + - StartupOption.UPGRADE.getName() + "] | [" + - StartupOption.ROLLBACK.getName() + "] | [" + - StartupOption.FINALIZE.getName() + "] | [" + - StartupOption.IMPORT.getName() + "] | [" + - StartupOption.INITIALIZESHAREDEDITS.getName() + - " [" + StartupOption.FORCE.getName() + "] [" + - StartupOption.NONINTERACTIVE.getName() + "]" + - "] | [" + - StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" + - StartupOption.RECOVER.getName() + " [ " + - StartupOption.FORCE.getName() + " ] ]"); + private static void printUsage(PrintStream out) { + out.println(USAGE + "\n"); } private static StartupOption parseArguments(String args[]) { @@ -1061,7 +1060,7 @@ public class NameNode { conf = new HdfsConfiguration(); StartupOption startOpt = parseArguments(argv); if (startOpt == null) { - printUsage(); + printUsage(System.err); return null; } setStartupOption(conf, startOpt); @@ -1175,6 +1174,10 @@ public class NameNode { /** */ public static void main(String argv[]) throws Exception { + if (DFSUtil.parseHelpArgument(argv, NameNode.USAGE, System.out, true)) { + System.exit(0); + } + try { StringUtils.startupShutdownMessage(NameNode.class, argv, LOG); NameNode namenode = createNameNode(argv, null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index 8057955dfac..47d09ef993c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -562,6 +562,9 @@ public class SecondaryNameNode implements Runnable { if (opts == null) { LOG.fatal("Failed to parse options"); terminate(1); + } else if (opts.shouldPrintHelp()) { + opts.usage(); + System.exit(0); } StringUtils.startupShutdownMessage(SecondaryNameNode.class, argv, LOG); @@ -595,6 +598,7 @@ public class SecondaryNameNode implements Runnable { private final Option geteditsizeOpt; private final Option checkpointOpt; private final Option formatOpt; + private final Option helpOpt; Command cmd; @@ -605,6 +609,7 @@ public class SecondaryNameNode implements Runnable { private boolean shouldForce; private boolean shouldFormat; + private boolean shouldPrintHelp; CommandLineOpts() { geteditsizeOpt = new Option("geteditsize", @@ -612,20 +617,32 @@ public class SecondaryNameNode implements Runnable { checkpointOpt = OptionBuilder.withArgName("force") .hasOptionalArg().withDescription("checkpoint on startup").create("checkpoint");; formatOpt = new Option("format", "format the local storage during startup"); + helpOpt = new Option("h", "help", false, "get help information"); options.addOption(geteditsizeOpt); options.addOption(checkpointOpt); options.addOption(formatOpt); + options.addOption(helpOpt); } public boolean shouldFormat() { return shouldFormat; } + public boolean shouldPrintHelp() { + return shouldPrintHelp; + } + public void parse(String ... argv) throws ParseException { CommandLineParser parser = new PosixParser(); CommandLine cmdLine = parser.parse(options, argv); + if (cmdLine.hasOption(helpOpt.getOpt()) + || cmdLine.hasOption(helpOpt.getLongOpt())) { + shouldPrintHelp = true; + return; + } + boolean hasGetEdit = cmdLine.hasOption(geteditsizeOpt.getOpt()); boolean hasCheckpoint = cmdLine.hasOption(checkpointOpt.getOpt()); if (hasGetEdit && hasCheckpoint) { @@ -662,8 +679,13 @@ public class SecondaryNameNode implements Runnable { } void usage() { + String header = "The Secondary NameNode is a helper " + + "to the primary NameNode. The Secondary is responsible " + + "for supporting periodic checkpoints of the HDFS metadata. " + + "The current design allows only one Secondary NameNode " + + "per HDFS cluster."; HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("secondarynamenode", options); + formatter.printHelp("secondarynamenode", header, options, "", false); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java index d4397276ea1..47c852d5963 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java @@ -42,6 +42,10 @@ public class DFSHAAdmin extends HAAdmin { protected void setErrOut(PrintStream errOut) { this.errOut = errOut; } + + protected void setOut(PrintStream out) { + this.out = out; + } @Override public void setConf(Configuration conf) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java index b1163d6885b..e18c9a86ad0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java @@ -162,6 +162,10 @@ public class DFSZKFailoverController extends ZKFailoverController { public static void main(String args[]) throws Exception { + if (DFSUtil.parseHelpArgument(args, + ZKFailoverController.USAGE, System.out, true)) { + System.exit(0); + } GenericOptionsParser parser = new GenericOptionsParser( new HdfsConfiguration(), args); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java index 566d77a5fbc..c3238f0de30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java @@ -73,6 +73,25 @@ public class DFSck extends Configured implements Tool { HdfsConfiguration.init(); } + private static final String USAGE = "Usage: DFSck " + + "[-list-corruptfileblocks | " + + "[-move | -delete | -openforwrite] " + + "[-files [-blocks [-locations | -racks]]]]\n" + + "\t\tstart checking from this path\n" + + "\t-move\tmove corrupted files to /lost+found\n" + + "\t-delete\tdelete corrupted files\n" + + "\t-files\tprint out files being checked\n" + + "\t-openforwrite\tprint out files opened for write\n" + + "\t-list-corruptfileblocks\tprint out list of missing " + + "blocks and files they belong to\n" + + "\t-blocks\tprint out block report\n" + + "\t-locations\tprint out locations for every block\n" + + "\t-racks\tprint out network topology for data-node locations\n" + + "\t\tBy default fsck ignores files opened for write, " + + "use -openforwrite to report such files. They are usually " + + " tagged CORRUPT or HEALTHY depending on their block " + + "allocation status"; + private final UserGroupInformation ugi; private final PrintStream out; @@ -93,25 +112,9 @@ public class DFSck extends Configured implements Tool { /** * Print fsck usage information */ - static void printUsage() { - System.err.println("Usage: DFSck [-list-corruptfileblocks | " + - "[-move | -delete | -openforwrite] " + - "[-files [-blocks [-locations | -racks]]]]"); - System.err.println("\t\tstart checking from this path"); - System.err.println("\t-move\tmove corrupted files to /lost+found"); - System.err.println("\t-delete\tdelete corrupted files"); - System.err.println("\t-files\tprint out files being checked"); - System.err.println("\t-openforwrite\tprint out files opened for write"); - System.err.println("\t-list-corruptfileblocks\tprint out list of missing " - + "blocks and files they belong to"); - System.err.println("\t-blocks\tprint out block report"); - System.err.println("\t-locations\tprint out locations for every block"); - System.err.println("\t-racks\tprint out network topology for data-node locations"); - System.err.println("\t\tBy default fsck ignores files opened for write, " + - "use -openforwrite to report such files. They are usually " + - " tagged CORRUPT or HEALTHY depending on their block " + - "allocation status"); - ToolRunner.printGenericCommandUsage(System.err); + static void printUsage(PrintStream out) { + out.println(USAGE + "\n"); + ToolRunner.printGenericCommandUsage(out); } /** * @param args @@ -119,7 +122,7 @@ public class DFSck extends Configured implements Tool { @Override public int run(final String[] args) throws IOException { if (args.length == 0) { - printUsage(); + printUsage(System.err); return -1; } @@ -258,12 +261,12 @@ public class DFSck extends Configured implements Tool { } else { System.err.println("fsck: can only operate on one path at a time '" + args[idx] + "'"); - printUsage(); + printUsage(System.err); return -1; } } else { System.err.println("fsck: Illegal option '" + args[idx] + "'"); - printUsage(); + printUsage(System.err); return -1; } } @@ -304,10 +307,14 @@ public class DFSck extends Configured implements Tool { // -files option is also used by GenericOptionsParser // Make sure that is not the first argument for fsck int res = -1; - if ((args.length == 0 ) || ("-files".equals(args[0]))) - printUsage(); - else + if ((args.length == 0) || ("-files".equals(args[0]))) { + printUsage(System.err); + ToolRunner.printGenericCommandUsage(System.err); + } else if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { + res = 0; + } else { res = ToolRunner.run(new DFSck(new HdfsConfiguration()), args); + } System.exit(res); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java index e0935d475cf..f74b4e88962 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java @@ -40,7 +40,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.HftpFileSystem; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -48,9 +47,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretMan import org.apache.hadoop.hdfs.server.namenode.CancelDelegationTokenServlet; import org.apache.hadoop.hdfs.server.namenode.GetDelegationTokenServlet; import org.apache.hadoop.hdfs.server.namenode.RenewDelegationTokenServlet; -import org.apache.hadoop.hdfs.web.URLUtils; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; @@ -71,8 +68,10 @@ public class DelegationTokenFetcher { private static final String CANCEL = "cancel"; private static final String RENEW = "renew"; private static final String PRINT = "print"; + private static final String HELP = "help"; + private static final String HELP_SHORT = "h"; - private static void printUsage(PrintStream err) throws IOException { + private static void printUsage(PrintStream err) { err.println("fetchdt retrieves delegation tokens from the NameNode"); err.println(); err.println("fetchdt "); @@ -107,6 +106,7 @@ public class DelegationTokenFetcher { fetcherOptions.addOption(CANCEL, false, "cancel the token"); fetcherOptions.addOption(RENEW, false, "renew the token"); fetcherOptions.addOption(PRINT, false, "print the token"); + fetcherOptions.addOption(HELP_SHORT, HELP, false, "print out help information"); GenericOptionsParser parser = new GenericOptionsParser(conf, fetcherOptions, args); CommandLine cmd = parser.getCommandLine(); @@ -119,9 +119,14 @@ public class DelegationTokenFetcher { final boolean cancel = cmd.hasOption(CANCEL); final boolean renew = cmd.hasOption(RENEW); final boolean print = cmd.hasOption(PRINT); + final boolean help = cmd.hasOption(HELP); String[] remaining = parser.getRemainingArgs(); // check option validity + if (help) { + printUsage(System.out); + System.exit(0); + } if (cancel && renew || cancel && print || renew && print || cancel && renew && print) { System.err.println("ERROR: Only specify cancel, renew or print."); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java index adf3293edf5..778ac59ee25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java @@ -324,6 +324,10 @@ public class GetConf extends Configured implements Tool { } public static void main(String[] args) throws Exception { + if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { + System.exit(0); + } + int res = ToolRunner.run(new GetConf(new HdfsConfiguration()), args); System.exit(res); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java index c0e415a8433..49d56d534fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -43,6 +44,8 @@ import org.apache.hadoop.util.ToolRunner; public class GetGroups extends GetGroupsBase { private static final Log LOG = LogFactory.getLog(GetGroups.class); + + static final String USAGE = "Usage: hdfs groups [username ...]"; static{ HdfsConfiguration.init(); @@ -86,6 +89,10 @@ public class GetGroups extends GetGroupsBase { } public static void main(String[] argv) throws Exception { + if (DFSUtil.parseHelpArgument(argv, USAGE, System.out, true)) { + System.exit(0); + } + int res = ToolRunner.run(new GetGroups(new HdfsConfiguration()), argv); System.exit(res); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java index 61e8ebef5c5..666e52b484f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java @@ -55,7 +55,9 @@ public class TestDFSHAAdmin { private DFSHAAdmin tool; private ByteArrayOutputStream errOutBytes = new ByteArrayOutputStream(); + private ByteArrayOutputStream outBytes = new ByteArrayOutputStream(); private String errOutput; + private String output; private HAServiceProtocol mockProtocol; private ZKFCProtocol mockZkfcProtocol; @@ -111,12 +113,14 @@ public class TestDFSHAAdmin { }; tool.setConf(getHAConf()); tool.setErrOut(new PrintStream(errOutBytes)); + tool.setOut(new PrintStream(outBytes)); } private void assertOutputContains(String string) { - if (!errOutput.contains(string)) { - fail("Expected output to contain '" + string + "' but was:\n" + - errOutput); + if (!errOutput.contains(string) && !output.contains(string)) { + fail("Expected output to contain '" + string + + "' but err_output was:\n" + errOutput + + "\n and output was: \n" + output); } } @@ -143,7 +147,7 @@ public class TestDFSHAAdmin { @Test public void testHelp() throws Exception { - assertEquals(-1, runTool("-help")); + assertEquals(0, runTool("-help")); assertEquals(0, runTool("-help", "transitionToActive")); assertOutputContains("Transitions the service into Active"); } @@ -378,10 +382,12 @@ public class TestDFSHAAdmin { private Object runTool(String ... args) throws Exception { errOutBytes.reset(); + outBytes.reset(); LOG.info("Running: DFSHAAdmin " + Joiner.on(" ").join(args)); int ret = tool.run(args); errOutput = new String(errOutBytes.toByteArray(), Charsets.UTF_8); - LOG.info("Output:\n" + errOutput); + output = new String(outBytes.toByteArray(), Charsets.UTF_8); + LOG.info("Err_output:\n" + errOutput + "\nOutput:\n" + output); return ret; } From 231a52a7dfa73f7f302cc3bf671d433312373873 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 15 Aug 2012 00:25:07 +0000 Subject: [PATCH 11/18] Reverting previous incomplete change r1373170 for HDFS-3723 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373172 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 - .../java/org/apache/hadoop/hdfs/DFSUtil.java | 68 ++----------------- .../hadoop/hdfs/server/balancer/Balancer.java | 26 +++---- .../hadoop/hdfs/server/datanode/DataNode.java | 22 +++--- .../hadoop/hdfs/server/namenode/NameNode.java | 51 +++++++------- .../server/namenode/SecondaryNameNode.java | 24 +------ .../apache/hadoop/hdfs/tools/DFSHAAdmin.java | 4 -- .../hdfs/tools/DFSZKFailoverController.java | 4 -- .../org/apache/hadoop/hdfs/tools/DFSck.java | 57 +++++++--------- .../hdfs/tools/DelegationTokenFetcher.java | 13 ++-- .../org/apache/hadoop/hdfs/tools/GetConf.java | 4 -- .../apache/hadoop/hdfs/tools/GetGroups.java | 7 -- .../hadoop/hdfs/tools/TestDFSHAAdmin.java | 16 ++--- 13 files changed, 83 insertions(+), 216 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 64a4fdf568e..ae2d2662586 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -114,9 +114,6 @@ Trunk (unreleased changes) HDFS-3789. JournalManager#format() should be able to throw IOException (Ivan Kelly via todd) - HDFS-3723. Add support -h, -help to all the commands. (Jing Zhao via - suresh) - OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index d263acd5906..8d59ae65886 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -18,21 +18,8 @@ package org.apache.hadoop.hdfs; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; - +import static org.apache.hadoop.hdfs.DFSConfigKeys.*; import java.io.IOException; -import java.io.PrintStream; import java.io.UnsupportedEncodingException; import java.net.InetSocketAddress; import java.net.URI; @@ -46,17 +33,10 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.StringTokenizer; import javax.net.SocketFactory; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -77,7 +57,8 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.ToolRunner; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import com.google.common.base.Joiner; import com.google.common.collect.Lists; @@ -443,6 +424,7 @@ public class DFSUtil { * * @param conf configuration * @return list of InetSocketAddresses + * @throws IOException if no addresses are configured */ public static Map> getHaNnRpcAddresses( Configuration conf) { @@ -1091,44 +1073,4 @@ public class DFSUtil { return null; } } - - public static Options helpOptions = new Options(); - public static Option helpOpt = new Option("h", "help", false, - "get help information"); - - static { - helpOptions.addOption(helpOpt); - } - - /** - * Parse the arguments for commands - * - * @param args the argument to be parsed - * @param helpDescription help information to be printed out - * @param out Printer - * @param printGenericCommandUsage whether to print the - * generic command usage defined in ToolRunner - * @return true when the argument matches help option, false if not - */ - public static boolean parseHelpArgument(String[] args, - String helpDescription, PrintStream out, boolean printGenericCommandUsage) { - if (args.length == 1) { - try { - CommandLineParser parser = new PosixParser(); - CommandLine cmdLine = parser.parse(helpOptions, args); - if (cmdLine.hasOption(helpOpt.getOpt()) - || cmdLine.hasOption(helpOpt.getLongOpt())) { - // should print out the help information - out.println(helpDescription + "\n"); - if (printGenericCommandUsage) { - ToolRunner.printGenericCommandUsage(out); - } - return true; - } - } catch (ParseException pe) { - return false; - } - } - return false; - } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 577d73b76be..f949c924a1e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.balancer; -import static com.google.common.base.Preconditions.checkArgument; import static org.apache.hadoop.hdfs.protocol.HdfsProtoUtil.vintPrefixed; import java.io.BufferedInputStream; @@ -27,7 +26,6 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.PrintStream; import java.net.Socket; import java.net.URI; import java.text.DateFormat; @@ -70,6 +68,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations; import org.apache.hadoop.io.IOUtils; @@ -80,6 +79,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import static com.google.common.base.Preconditions.checkArgument; /**

The balancer is a tool that balances disk space usage on an HDFS cluster * when some datanodes become full or when new empty nodes join the cluster. @@ -189,13 +189,6 @@ public class Balancer { */ public static final int MAX_NUM_CONCURRENT_MOVES = 5; - private static final String USAGE = "Usage: java " - + Balancer.class.getSimpleName() - + "\n\t[-policy ]\tthe balancing policy: " - + BalancingPolicy.Node.INSTANCE.getName() + " or " - + BalancingPolicy.Pool.INSTANCE.getName() - + "\n\t[-threshold ]\tPercentage of disk capacity"; - private final NameNodeConnector nnc; private final BalancingPolicy policy; private final double threshold; @@ -1557,7 +1550,7 @@ public class Balancer { } } } catch(RuntimeException e) { - printUsage(System.err); + printUsage(); throw e; } } @@ -1565,8 +1558,13 @@ public class Balancer { return new Parameters(policy, threshold); } - private static void printUsage(PrintStream out) { - out.println(USAGE + "\n"); + private static void printUsage() { + System.out.println("Usage: java " + Balancer.class.getSimpleName()); + System.out.println(" [-policy ]\tthe balancing policy: " + + BalancingPolicy.Node.INSTANCE.getName() + " or " + + BalancingPolicy.Pool.INSTANCE.getName()); + System.out.println( + " [-threshold ]\tPercentage of disk capacity"); } } @@ -1575,10 +1573,6 @@ public class Balancer { * @param args Command line arguments */ public static void main(String[] args) { - if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { - System.exit(0); - } - try { System.exit(ToolRunner.run(new HdfsConfiguration(), new Cli(), args)); } catch (Throwable e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 83219cca48b..a456d133d4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -46,7 +46,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOUR import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STARTUP_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTPS_ENABLE_KEY; -import static org.apache.hadoop.util.ExitUtil.terminate; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; @@ -56,7 +55,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.PrintStream; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; @@ -100,8 +98,8 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsProtoUtil; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage; -import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferEncryptor; import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol; +import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferEncryptor; import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ClientDatanodeProtocolService; @@ -126,6 +124,9 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.common.Util; + +import static org.apache.hadoop.util.ExitUtil.terminate; + import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; @@ -170,9 +171,9 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.util.VersionInfo; import org.mortbay.util.ajax.JSON; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.BlockingService; /********************************************************** @@ -229,8 +230,6 @@ public class DataNode extends Configured static final Log ClientTraceLog = LogFactory.getLog(DataNode.class.getName() + ".clienttrace"); - - private static final String USAGE = "Usage: java DataNode [-rollback | -regular]"; /** * Use {@link NetUtils#createSocketAddr(String)} instead. @@ -1542,7 +1541,7 @@ public class DataNode extends Configured } if (!parseArguments(args, conf)) { - printUsage(System.err); + printUsage(); return null; } Collection dataDirs = getStorageDirs(conf); @@ -1656,8 +1655,9 @@ public class DataNode extends Configured + xmitsInProgress.get() + "}"; } - private static void printUsage(PrintStream out) { - out.println(USAGE + "\n"); + private static void printUsage() { + System.err.println("Usage: java DataNode"); + System.err.println(" [-rollback]"); } /** @@ -1742,10 +1742,6 @@ public class DataNode extends Configured } public static void main(String args[]) { - if (DFSUtil.parseHelpArgument(args, DataNode.USAGE, System.out, true)) { - System.exit(0); - } - secureMain(args, null); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 2df693b3c4e..20e7aafba9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.IOException; -import java.io.PrintStream; import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; @@ -39,8 +38,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Trash; import static org.apache.hadoop.hdfs.DFSConfigKeys.*; -import static org.apache.hadoop.util.ExitUtil.terminate; -import static org.apache.hadoop.util.ToolRunner.confirmPrompt; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; @@ -72,9 +69,12 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol; import org.apache.hadoop.tools.GetUserMappingsProtocol; -import org.apache.hadoop.util.ExitUtil.ExitException; import org.apache.hadoop.util.ServicePlugin; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ExitUtil.ExitException; + +import static org.apache.hadoop.util.ExitUtil.terminate; +import static org.apache.hadoop.util.ToolRunner.confirmPrompt; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -188,22 +188,6 @@ public class NameNode { DFS_HA_AUTO_FAILOVER_ENABLED_KEY }; - private static final String USAGE = "Usage: java NameNode [" - + StartupOption.BACKUP.getName() + "] | [" - + StartupOption.CHECKPOINT.getName() + "] | [" - + StartupOption.FORMAT.getName() + " [" - + StartupOption.CLUSTERID.getName() + " cid ] [" - + StartupOption.FORCE.getName() + "] [" - + StartupOption.NONINTERACTIVE.getName() + "] ] | [" - + StartupOption.UPGRADE.getName() + "] | [" - + StartupOption.ROLLBACK.getName() + "] | [" - + StartupOption.FINALIZE.getName() + "] | [" - + StartupOption.IMPORT.getName() + "] | [" - + StartupOption.INITIALIZESHAREDEDITS.getName() + "] | [" - + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" - + StartupOption.RECOVER.getName() + " [ " + StartupOption.FORCE.getName() - + " ] ]"; - public long getProtocolVersion(String protocol, long clientVersion) throws IOException { if (protocol.equals(ClientProtocol.class.getName())) { @@ -911,8 +895,25 @@ public class NameNode { return false; } - private static void printUsage(PrintStream out) { - out.println(USAGE + "\n"); + private static void printUsage() { + System.err.println( + "Usage: java NameNode [" + + StartupOption.BACKUP.getName() + "] | [" + + StartupOption.CHECKPOINT.getName() + "] | [" + + StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() + + " cid ] [" + StartupOption.FORCE.getName() + "] [" + + StartupOption.NONINTERACTIVE.getName() + "] ] | [" + + StartupOption.UPGRADE.getName() + "] | [" + + StartupOption.ROLLBACK.getName() + "] | [" + + StartupOption.FINALIZE.getName() + "] | [" + + StartupOption.IMPORT.getName() + "] | [" + + StartupOption.INITIALIZESHAREDEDITS.getName() + + " [" + StartupOption.FORCE.getName() + "] [" + + StartupOption.NONINTERACTIVE.getName() + "]" + + "] | [" + + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" + + StartupOption.RECOVER.getName() + " [ " + + StartupOption.FORCE.getName() + " ] ]"); } private static StartupOption parseArguments(String args[]) { @@ -1060,7 +1061,7 @@ public class NameNode { conf = new HdfsConfiguration(); StartupOption startOpt = parseArguments(argv); if (startOpt == null) { - printUsage(System.err); + printUsage(); return null; } setStartupOption(conf, startOpt); @@ -1174,10 +1175,6 @@ public class NameNode { /** */ public static void main(String argv[]) throws Exception { - if (DFSUtil.parseHelpArgument(argv, NameNode.USAGE, System.out, true)) { - System.exit(0); - } - try { StringUtils.startupShutdownMessage(NameNode.class, argv, LOG); NameNode namenode = createNameNode(argv, null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index 47d09ef993c..8057955dfac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -562,9 +562,6 @@ public class SecondaryNameNode implements Runnable { if (opts == null) { LOG.fatal("Failed to parse options"); terminate(1); - } else if (opts.shouldPrintHelp()) { - opts.usage(); - System.exit(0); } StringUtils.startupShutdownMessage(SecondaryNameNode.class, argv, LOG); @@ -598,7 +595,6 @@ public class SecondaryNameNode implements Runnable { private final Option geteditsizeOpt; private final Option checkpointOpt; private final Option formatOpt; - private final Option helpOpt; Command cmd; @@ -609,7 +605,6 @@ public class SecondaryNameNode implements Runnable { private boolean shouldForce; private boolean shouldFormat; - private boolean shouldPrintHelp; CommandLineOpts() { geteditsizeOpt = new Option("geteditsize", @@ -617,32 +612,20 @@ public class SecondaryNameNode implements Runnable { checkpointOpt = OptionBuilder.withArgName("force") .hasOptionalArg().withDescription("checkpoint on startup").create("checkpoint");; formatOpt = new Option("format", "format the local storage during startup"); - helpOpt = new Option("h", "help", false, "get help information"); options.addOption(geteditsizeOpt); options.addOption(checkpointOpt); options.addOption(formatOpt); - options.addOption(helpOpt); } public boolean shouldFormat() { return shouldFormat; } - public boolean shouldPrintHelp() { - return shouldPrintHelp; - } - public void parse(String ... argv) throws ParseException { CommandLineParser parser = new PosixParser(); CommandLine cmdLine = parser.parse(options, argv); - if (cmdLine.hasOption(helpOpt.getOpt()) - || cmdLine.hasOption(helpOpt.getLongOpt())) { - shouldPrintHelp = true; - return; - } - boolean hasGetEdit = cmdLine.hasOption(geteditsizeOpt.getOpt()); boolean hasCheckpoint = cmdLine.hasOption(checkpointOpt.getOpt()); if (hasGetEdit && hasCheckpoint) { @@ -679,13 +662,8 @@ public class SecondaryNameNode implements Runnable { } void usage() { - String header = "The Secondary NameNode is a helper " - + "to the primary NameNode. The Secondary is responsible " - + "for supporting periodic checkpoints of the HDFS metadata. " - + "The current design allows only one Secondary NameNode " - + "per HDFS cluster."; HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("secondarynamenode", header, options, "", false); + formatter.printHelp("secondarynamenode", options); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java index 47c852d5963..d4397276ea1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java @@ -42,10 +42,6 @@ public class DFSHAAdmin extends HAAdmin { protected void setErrOut(PrintStream errOut) { this.errOut = errOut; } - - protected void setOut(PrintStream out) { - this.out = out; - } @Override public void setConf(Configuration conf) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java index e18c9a86ad0..b1163d6885b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java @@ -162,10 +162,6 @@ public class DFSZKFailoverController extends ZKFailoverController { public static void main(String args[]) throws Exception { - if (DFSUtil.parseHelpArgument(args, - ZKFailoverController.USAGE, System.out, true)) { - System.exit(0); - } GenericOptionsParser parser = new GenericOptionsParser( new HdfsConfiguration(), args); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java index c3238f0de30..566d77a5fbc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java @@ -73,25 +73,6 @@ public class DFSck extends Configured implements Tool { HdfsConfiguration.init(); } - private static final String USAGE = "Usage: DFSck " - + "[-list-corruptfileblocks | " - + "[-move | -delete | -openforwrite] " - + "[-files [-blocks [-locations | -racks]]]]\n" - + "\t\tstart checking from this path\n" - + "\t-move\tmove corrupted files to /lost+found\n" - + "\t-delete\tdelete corrupted files\n" - + "\t-files\tprint out files being checked\n" - + "\t-openforwrite\tprint out files opened for write\n" - + "\t-list-corruptfileblocks\tprint out list of missing " - + "blocks and files they belong to\n" - + "\t-blocks\tprint out block report\n" - + "\t-locations\tprint out locations for every block\n" - + "\t-racks\tprint out network topology for data-node locations\n" - + "\t\tBy default fsck ignores files opened for write, " - + "use -openforwrite to report such files. They are usually " - + " tagged CORRUPT or HEALTHY depending on their block " - + "allocation status"; - private final UserGroupInformation ugi; private final PrintStream out; @@ -112,9 +93,25 @@ public class DFSck extends Configured implements Tool { /** * Print fsck usage information */ - static void printUsage(PrintStream out) { - out.println(USAGE + "\n"); - ToolRunner.printGenericCommandUsage(out); + static void printUsage() { + System.err.println("Usage: DFSck [-list-corruptfileblocks | " + + "[-move | -delete | -openforwrite] " + + "[-files [-blocks [-locations | -racks]]]]"); + System.err.println("\t\tstart checking from this path"); + System.err.println("\t-move\tmove corrupted files to /lost+found"); + System.err.println("\t-delete\tdelete corrupted files"); + System.err.println("\t-files\tprint out files being checked"); + System.err.println("\t-openforwrite\tprint out files opened for write"); + System.err.println("\t-list-corruptfileblocks\tprint out list of missing " + + "blocks and files they belong to"); + System.err.println("\t-blocks\tprint out block report"); + System.err.println("\t-locations\tprint out locations for every block"); + System.err.println("\t-racks\tprint out network topology for data-node locations"); + System.err.println("\t\tBy default fsck ignores files opened for write, " + + "use -openforwrite to report such files. They are usually " + + " tagged CORRUPT or HEALTHY depending on their block " + + "allocation status"); + ToolRunner.printGenericCommandUsage(System.err); } /** * @param args @@ -122,7 +119,7 @@ public class DFSck extends Configured implements Tool { @Override public int run(final String[] args) throws IOException { if (args.length == 0) { - printUsage(System.err); + printUsage(); return -1; } @@ -261,12 +258,12 @@ public class DFSck extends Configured implements Tool { } else { System.err.println("fsck: can only operate on one path at a time '" + args[idx] + "'"); - printUsage(System.err); + printUsage(); return -1; } } else { System.err.println("fsck: Illegal option '" + args[idx] + "'"); - printUsage(System.err); + printUsage(); return -1; } } @@ -307,14 +304,10 @@ public class DFSck extends Configured implements Tool { // -files option is also used by GenericOptionsParser // Make sure that is not the first argument for fsck int res = -1; - if ((args.length == 0) || ("-files".equals(args[0]))) { - printUsage(System.err); - ToolRunner.printGenericCommandUsage(System.err); - } else if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { - res = 0; - } else { + if ((args.length == 0 ) || ("-files".equals(args[0]))) + printUsage(); + else res = ToolRunner.run(new DFSck(new HdfsConfiguration()), args); - } System.exit(res); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java index f74b4e88962..e0935d475cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java @@ -40,6 +40,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.HftpFileSystem; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -47,7 +48,9 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretMan import org.apache.hadoop.hdfs.server.namenode.CancelDelegationTokenServlet; import org.apache.hadoop.hdfs.server.namenode.GetDelegationTokenServlet; import org.apache.hadoop.hdfs.server.namenode.RenewDelegationTokenServlet; +import org.apache.hadoop.hdfs.web.URLUtils; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; @@ -68,10 +71,8 @@ public class DelegationTokenFetcher { private static final String CANCEL = "cancel"; private static final String RENEW = "renew"; private static final String PRINT = "print"; - private static final String HELP = "help"; - private static final String HELP_SHORT = "h"; - private static void printUsage(PrintStream err) { + private static void printUsage(PrintStream err) throws IOException { err.println("fetchdt retrieves delegation tokens from the NameNode"); err.println(); err.println("fetchdt "); @@ -106,7 +107,6 @@ public class DelegationTokenFetcher { fetcherOptions.addOption(CANCEL, false, "cancel the token"); fetcherOptions.addOption(RENEW, false, "renew the token"); fetcherOptions.addOption(PRINT, false, "print the token"); - fetcherOptions.addOption(HELP_SHORT, HELP, false, "print out help information"); GenericOptionsParser parser = new GenericOptionsParser(conf, fetcherOptions, args); CommandLine cmd = parser.getCommandLine(); @@ -119,14 +119,9 @@ public class DelegationTokenFetcher { final boolean cancel = cmd.hasOption(CANCEL); final boolean renew = cmd.hasOption(RENEW); final boolean print = cmd.hasOption(PRINT); - final boolean help = cmd.hasOption(HELP); String[] remaining = parser.getRemainingArgs(); // check option validity - if (help) { - printUsage(System.out); - System.exit(0); - } if (cancel && renew || cancel && print || renew && print || cancel && renew && print) { System.err.println("ERROR: Only specify cancel, renew or print."); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java index 778ac59ee25..adf3293edf5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java @@ -324,10 +324,6 @@ public class GetConf extends Configured implements Tool { } public static void main(String[] args) throws Exception { - if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { - System.exit(0); - } - int res = ToolRunner.run(new GetConf(new HdfsConfiguration()), args); System.exit(res); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java index 49d56d534fb..c0e415a8433 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java @@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -44,8 +43,6 @@ import org.apache.hadoop.util.ToolRunner; public class GetGroups extends GetGroupsBase { private static final Log LOG = LogFactory.getLog(GetGroups.class); - - static final String USAGE = "Usage: hdfs groups [username ...]"; static{ HdfsConfiguration.init(); @@ -89,10 +86,6 @@ public class GetGroups extends GetGroupsBase { } public static void main(String[] argv) throws Exception { - if (DFSUtil.parseHelpArgument(argv, USAGE, System.out, true)) { - System.exit(0); - } - int res = ToolRunner.run(new GetGroups(new HdfsConfiguration()), argv); System.exit(res); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java index 666e52b484f..61e8ebef5c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java @@ -55,9 +55,7 @@ public class TestDFSHAAdmin { private DFSHAAdmin tool; private ByteArrayOutputStream errOutBytes = new ByteArrayOutputStream(); - private ByteArrayOutputStream outBytes = new ByteArrayOutputStream(); private String errOutput; - private String output; private HAServiceProtocol mockProtocol; private ZKFCProtocol mockZkfcProtocol; @@ -113,14 +111,12 @@ public class TestDFSHAAdmin { }; tool.setConf(getHAConf()); tool.setErrOut(new PrintStream(errOutBytes)); - tool.setOut(new PrintStream(outBytes)); } private void assertOutputContains(String string) { - if (!errOutput.contains(string) && !output.contains(string)) { - fail("Expected output to contain '" + string + - "' but err_output was:\n" + errOutput + - "\n and output was: \n" + output); + if (!errOutput.contains(string)) { + fail("Expected output to contain '" + string + "' but was:\n" + + errOutput); } } @@ -147,7 +143,7 @@ public class TestDFSHAAdmin { @Test public void testHelp() throws Exception { - assertEquals(0, runTool("-help")); + assertEquals(-1, runTool("-help")); assertEquals(0, runTool("-help", "transitionToActive")); assertOutputContains("Transitions the service into Active"); } @@ -382,12 +378,10 @@ public class TestDFSHAAdmin { private Object runTool(String ... args) throws Exception { errOutBytes.reset(); - outBytes.reset(); LOG.info("Running: DFSHAAdmin " + Joiner.on(" ").join(args)); int ret = tool.run(args); errOutput = new String(errOutBytes.toByteArray(), Charsets.UTF_8); - output = new String(outBytes.toByteArray(), Charsets.UTF_8); - LOG.info("Err_output:\n" + errOutput + "\nOutput:\n" + output); + LOG.info("Output:\n" + errOutput); return ret; } From 28e87740c5797e87a9038d7f151c8b21a5b04592 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 15 Aug 2012 00:28:41 +0000 Subject: [PATCH 12/18] HDFS-3723. Add support -h, -help to all the commands. Contributed by Jing Zhao git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373173 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/hadoop/ha/HAAdmin.java | 9 ++- .../hadoop/ha/ZKFailoverController.java | 5 +- .../org/apache/hadoop/ha/TestHAAdmin.java | 16 +++-- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../java/org/apache/hadoop/hdfs/DFSUtil.java | 68 +++++++++++++++++-- .../hadoop/hdfs/server/balancer/Balancer.java | 26 ++++--- .../hadoop/hdfs/server/datanode/DataNode.java | 22 +++--- .../hadoop/hdfs/server/namenode/NameNode.java | 51 +++++++------- .../server/namenode/SecondaryNameNode.java | 24 ++++++- .../apache/hadoop/hdfs/tools/DFSHAAdmin.java | 4 ++ .../hdfs/tools/DFSZKFailoverController.java | 4 ++ .../org/apache/hadoop/hdfs/tools/DFSck.java | 57 +++++++++------- .../hdfs/tools/DelegationTokenFetcher.java | 13 ++-- .../org/apache/hadoop/hdfs/tools/GetConf.java | 4 ++ .../apache/hadoop/hdfs/tools/GetGroups.java | 7 ++ .../hadoop/hdfs/tools/TestDFSHAAdmin.java | 16 +++-- 16 files changed, 236 insertions(+), 93 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java index 7d85c016deb..1ce81fd6a22 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java @@ -88,7 +88,7 @@ public abstract class HAAdmin extends Configured implements Tool { /** Output stream for errors, for use in tests */ protected PrintStream errOut = System.err; - PrintStream out = System.out; + protected PrintStream out = System.out; private RequestSource requestSource = RequestSource.REQUEST_BY_USER; protected abstract HAServiceTarget resolveTarget(String string); @@ -439,7 +439,10 @@ public abstract class HAAdmin extends Configured implements Tool { } private int help(String[] argv) { - if (argv.length != 2) { + if (argv.length == 1) { // only -help + printUsage(out); + return 0; + } else if (argv.length != 2) { printUsage(errOut, "-help"); return -1; } @@ -454,7 +457,7 @@ public abstract class HAAdmin extends Configured implements Tool { return -1; } - errOut.println(cmd + " [" + usageInfo.args + "]: " + usageInfo.help); + out.println(cmd + " [" + usageInfo.args + "]: " + usageInfo.help); return 0; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java index c02fe0d1630..35d75b72ae5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java @@ -80,6 +80,8 @@ public abstract class ZKFailoverController { ZK_AUTH_KEY }; + protected static final String USAGE = + "Usage: java zkfc [ -formatZK [-force] [-nonInteractive] ]"; /** Unable to format the parent znode in ZK */ static final int ERR_CODE_FORMAT_DENIED = 2; @@ -248,8 +250,7 @@ public abstract class ZKFailoverController { } private void printUsage() { - System.err.println("Usage: " + this.getClass().getSimpleName() + - " [-formatZK [-force | -nonInteractive]]"); + System.err.println(USAGE + "\n"); } private int formatZK(boolean force, boolean interactive) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java index a3cac1b96f0..1d8f48e2d02 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java @@ -40,7 +40,9 @@ public class TestHAAdmin { private HAAdmin tool; private ByteArrayOutputStream errOutBytes = new ByteArrayOutputStream(); + private ByteArrayOutputStream outBytes = new ByteArrayOutputStream(); private String errOutput; + private String output; @Before public void setup() throws IOException { @@ -53,12 +55,14 @@ public class TestHAAdmin { }; tool.setConf(new Configuration()); tool.errOut = new PrintStream(errOutBytes); + tool.out = new PrintStream(outBytes); } private void assertOutputContains(String string) { - if (!errOutput.contains(string)) { - fail("Expected output to contain '" + string + "' but was:\n" + - errOutput); + if (!errOutput.contains(string) && !output.contains(string)) { + fail("Expected output to contain '" + string + + "' but err_output was:\n" + errOutput + + "\n and output was: \n" + output); } } @@ -88,17 +92,19 @@ public class TestHAAdmin { @Test public void testHelp() throws Exception { - assertEquals(-1, runTool("-help")); + assertEquals(0, runTool("-help")); assertEquals(0, runTool("-help", "transitionToActive")); assertOutputContains("Transitions the service into Active"); } private Object runTool(String ... args) throws Exception { errOutBytes.reset(); + outBytes.reset(); LOG.info("Running: HAAdmin " + Joiner.on(" ").join(args)); int ret = tool.run(args); errOutput = new String(errOutBytes.toByteArray(), Charsets.UTF_8); - LOG.info("Output:\n" + errOutput); + output = new String(outBytes.toByteArray(), Charsets.UTF_8); + LOG.info("Err_output:\n" + errOutput + "\nOutput:\n" + output); return ret; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ae2d2662586..64a4fdf568e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -114,6 +114,9 @@ Trunk (unreleased changes) HDFS-3789. JournalManager#format() should be able to throw IOException (Ivan Kelly via todd) + HDFS-3723. Add support -h, -help to all the commands. (Jing Zhao via + suresh) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 8d59ae65886..d263acd5906 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -18,8 +18,21 @@ package org.apache.hadoop.hdfs; -import static org.apache.hadoop.hdfs.DFSConfigKeys.*; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; + import java.io.IOException; +import java.io.PrintStream; import java.io.UnsupportedEncodingException; import java.net.InetSocketAddress; import java.net.URI; @@ -33,10 +46,17 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.StringTokenizer; import javax.net.SocketFactory; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -57,8 +77,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.ToolRunner; import com.google.common.base.Joiner; import com.google.common.collect.Lists; @@ -424,7 +443,6 @@ public class DFSUtil { * * @param conf configuration * @return list of InetSocketAddresses - * @throws IOException if no addresses are configured */ public static Map> getHaNnRpcAddresses( Configuration conf) { @@ -1073,4 +1091,44 @@ public class DFSUtil { return null; } } + + public static Options helpOptions = new Options(); + public static Option helpOpt = new Option("h", "help", false, + "get help information"); + + static { + helpOptions.addOption(helpOpt); + } + + /** + * Parse the arguments for commands + * + * @param args the argument to be parsed + * @param helpDescription help information to be printed out + * @param out Printer + * @param printGenericCommandUsage whether to print the + * generic command usage defined in ToolRunner + * @return true when the argument matches help option, false if not + */ + public static boolean parseHelpArgument(String[] args, + String helpDescription, PrintStream out, boolean printGenericCommandUsage) { + if (args.length == 1) { + try { + CommandLineParser parser = new PosixParser(); + CommandLine cmdLine = parser.parse(helpOptions, args); + if (cmdLine.hasOption(helpOpt.getOpt()) + || cmdLine.hasOption(helpOpt.getLongOpt())) { + // should print out the help information + out.println(helpDescription + "\n"); + if (printGenericCommandUsage) { + ToolRunner.printGenericCommandUsage(out); + } + return true; + } + } catch (ParseException pe) { + return false; + } + } + return false; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index f949c924a1e..577d73b76be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.balancer; +import static com.google.common.base.Preconditions.checkArgument; import static org.apache.hadoop.hdfs.protocol.HdfsProtoUtil.vintPrefixed; import java.io.BufferedInputStream; @@ -26,6 +27,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.PrintStream; import java.net.Socket; import java.net.URI; import java.text.DateFormat; @@ -68,7 +70,6 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations; import org.apache.hadoop.io.IOUtils; @@ -79,7 +80,6 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import static com.google.common.base.Preconditions.checkArgument; /**

The balancer is a tool that balances disk space usage on an HDFS cluster * when some datanodes become full or when new empty nodes join the cluster. @@ -189,6 +189,13 @@ public class Balancer { */ public static final int MAX_NUM_CONCURRENT_MOVES = 5; + private static final String USAGE = "Usage: java " + + Balancer.class.getSimpleName() + + "\n\t[-policy ]\tthe balancing policy: " + + BalancingPolicy.Node.INSTANCE.getName() + " or " + + BalancingPolicy.Pool.INSTANCE.getName() + + "\n\t[-threshold ]\tPercentage of disk capacity"; + private final NameNodeConnector nnc; private final BalancingPolicy policy; private final double threshold; @@ -1550,7 +1557,7 @@ public class Balancer { } } } catch(RuntimeException e) { - printUsage(); + printUsage(System.err); throw e; } } @@ -1558,13 +1565,8 @@ public class Balancer { return new Parameters(policy, threshold); } - private static void printUsage() { - System.out.println("Usage: java " + Balancer.class.getSimpleName()); - System.out.println(" [-policy ]\tthe balancing policy: " - + BalancingPolicy.Node.INSTANCE.getName() + " or " - + BalancingPolicy.Pool.INSTANCE.getName()); - System.out.println( - " [-threshold ]\tPercentage of disk capacity"); + private static void printUsage(PrintStream out) { + out.println(USAGE + "\n"); } } @@ -1573,6 +1575,10 @@ public class Balancer { * @param args Command line arguments */ public static void main(String[] args) { + if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { + System.exit(0); + } + try { System.exit(ToolRunner.run(new HdfsConfiguration(), new Cli(), args)); } catch (Throwable e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index a456d133d4a..83219cca48b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -46,6 +46,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOUR import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STARTUP_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTPS_ENABLE_KEY; +import static org.apache.hadoop.util.ExitUtil.terminate; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; @@ -55,6 +56,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.PrintStream; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; @@ -98,8 +100,8 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsProtoUtil; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage; -import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol; import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferEncryptor; +import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol; import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ClientDatanodeProtocolService; @@ -124,9 +126,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.common.Util; - -import static org.apache.hadoop.util.ExitUtil.terminate; - import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; @@ -171,9 +170,9 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.util.VersionInfo; import org.mortbay.util.ajax.JSON; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; -import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.BlockingService; /********************************************************** @@ -230,6 +229,8 @@ public class DataNode extends Configured static final Log ClientTraceLog = LogFactory.getLog(DataNode.class.getName() + ".clienttrace"); + + private static final String USAGE = "Usage: java DataNode [-rollback | -regular]"; /** * Use {@link NetUtils#createSocketAddr(String)} instead. @@ -1541,7 +1542,7 @@ public class DataNode extends Configured } if (!parseArguments(args, conf)) { - printUsage(); + printUsage(System.err); return null; } Collection dataDirs = getStorageDirs(conf); @@ -1655,9 +1656,8 @@ public class DataNode extends Configured + xmitsInProgress.get() + "}"; } - private static void printUsage() { - System.err.println("Usage: java DataNode"); - System.err.println(" [-rollback]"); + private static void printUsage(PrintStream out) { + out.println(USAGE + "\n"); } /** @@ -1742,6 +1742,10 @@ public class DataNode extends Configured } public static void main(String args[]) { + if (DFSUtil.parseHelpArgument(args, DataNode.USAGE, System.out, true)) { + System.exit(0); + } + secureMain(args, null); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 20e7aafba9f..2df693b3c4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.IOException; +import java.io.PrintStream; import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; @@ -38,6 +39,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Trash; import static org.apache.hadoop.hdfs.DFSConfigKeys.*; +import static org.apache.hadoop.util.ExitUtil.terminate; +import static org.apache.hadoop.util.ToolRunner.confirmPrompt; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; @@ -69,12 +72,9 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol; import org.apache.hadoop.tools.GetUserMappingsProtocol; +import org.apache.hadoop.util.ExitUtil.ExitException; import org.apache.hadoop.util.ServicePlugin; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.ExitUtil.ExitException; - -import static org.apache.hadoop.util.ExitUtil.terminate; -import static org.apache.hadoop.util.ToolRunner.confirmPrompt; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -188,6 +188,22 @@ public class NameNode { DFS_HA_AUTO_FAILOVER_ENABLED_KEY }; + private static final String USAGE = "Usage: java NameNode [" + + StartupOption.BACKUP.getName() + "] | [" + + StartupOption.CHECKPOINT.getName() + "] | [" + + StartupOption.FORMAT.getName() + " [" + + StartupOption.CLUSTERID.getName() + " cid ] [" + + StartupOption.FORCE.getName() + "] [" + + StartupOption.NONINTERACTIVE.getName() + "] ] | [" + + StartupOption.UPGRADE.getName() + "] | [" + + StartupOption.ROLLBACK.getName() + "] | [" + + StartupOption.FINALIZE.getName() + "] | [" + + StartupOption.IMPORT.getName() + "] | [" + + StartupOption.INITIALIZESHAREDEDITS.getName() + "] | [" + + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" + + StartupOption.RECOVER.getName() + " [ " + StartupOption.FORCE.getName() + + " ] ]"; + public long getProtocolVersion(String protocol, long clientVersion) throws IOException { if (protocol.equals(ClientProtocol.class.getName())) { @@ -895,25 +911,8 @@ public class NameNode { return false; } - private static void printUsage() { - System.err.println( - "Usage: java NameNode [" + - StartupOption.BACKUP.getName() + "] | [" + - StartupOption.CHECKPOINT.getName() + "] | [" + - StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() + - " cid ] [" + StartupOption.FORCE.getName() + "] [" + - StartupOption.NONINTERACTIVE.getName() + "] ] | [" + - StartupOption.UPGRADE.getName() + "] | [" + - StartupOption.ROLLBACK.getName() + "] | [" + - StartupOption.FINALIZE.getName() + "] | [" + - StartupOption.IMPORT.getName() + "] | [" + - StartupOption.INITIALIZESHAREDEDITS.getName() + - " [" + StartupOption.FORCE.getName() + "] [" + - StartupOption.NONINTERACTIVE.getName() + "]" + - "] | [" + - StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" + - StartupOption.RECOVER.getName() + " [ " + - StartupOption.FORCE.getName() + " ] ]"); + private static void printUsage(PrintStream out) { + out.println(USAGE + "\n"); } private static StartupOption parseArguments(String args[]) { @@ -1061,7 +1060,7 @@ public class NameNode { conf = new HdfsConfiguration(); StartupOption startOpt = parseArguments(argv); if (startOpt == null) { - printUsage(); + printUsage(System.err); return null; } setStartupOption(conf, startOpt); @@ -1175,6 +1174,10 @@ public class NameNode { /** */ public static void main(String argv[]) throws Exception { + if (DFSUtil.parseHelpArgument(argv, NameNode.USAGE, System.out, true)) { + System.exit(0); + } + try { StringUtils.startupShutdownMessage(NameNode.class, argv, LOG); NameNode namenode = createNameNode(argv, null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index 8057955dfac..47d09ef993c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -562,6 +562,9 @@ public class SecondaryNameNode implements Runnable { if (opts == null) { LOG.fatal("Failed to parse options"); terminate(1); + } else if (opts.shouldPrintHelp()) { + opts.usage(); + System.exit(0); } StringUtils.startupShutdownMessage(SecondaryNameNode.class, argv, LOG); @@ -595,6 +598,7 @@ public class SecondaryNameNode implements Runnable { private final Option geteditsizeOpt; private final Option checkpointOpt; private final Option formatOpt; + private final Option helpOpt; Command cmd; @@ -605,6 +609,7 @@ public class SecondaryNameNode implements Runnable { private boolean shouldForce; private boolean shouldFormat; + private boolean shouldPrintHelp; CommandLineOpts() { geteditsizeOpt = new Option("geteditsize", @@ -612,20 +617,32 @@ public class SecondaryNameNode implements Runnable { checkpointOpt = OptionBuilder.withArgName("force") .hasOptionalArg().withDescription("checkpoint on startup").create("checkpoint");; formatOpt = new Option("format", "format the local storage during startup"); + helpOpt = new Option("h", "help", false, "get help information"); options.addOption(geteditsizeOpt); options.addOption(checkpointOpt); options.addOption(formatOpt); + options.addOption(helpOpt); } public boolean shouldFormat() { return shouldFormat; } + public boolean shouldPrintHelp() { + return shouldPrintHelp; + } + public void parse(String ... argv) throws ParseException { CommandLineParser parser = new PosixParser(); CommandLine cmdLine = parser.parse(options, argv); + if (cmdLine.hasOption(helpOpt.getOpt()) + || cmdLine.hasOption(helpOpt.getLongOpt())) { + shouldPrintHelp = true; + return; + } + boolean hasGetEdit = cmdLine.hasOption(geteditsizeOpt.getOpt()); boolean hasCheckpoint = cmdLine.hasOption(checkpointOpt.getOpt()); if (hasGetEdit && hasCheckpoint) { @@ -662,8 +679,13 @@ public class SecondaryNameNode implements Runnable { } void usage() { + String header = "The Secondary NameNode is a helper " + + "to the primary NameNode. The Secondary is responsible " + + "for supporting periodic checkpoints of the HDFS metadata. " + + "The current design allows only one Secondary NameNode " + + "per HDFS cluster."; HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("secondarynamenode", options); + formatter.printHelp("secondarynamenode", header, options, "", false); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java index d4397276ea1..47c852d5963 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java @@ -42,6 +42,10 @@ public class DFSHAAdmin extends HAAdmin { protected void setErrOut(PrintStream errOut) { this.errOut = errOut; } + + protected void setOut(PrintStream out) { + this.out = out; + } @Override public void setConf(Configuration conf) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java index b1163d6885b..e18c9a86ad0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java @@ -162,6 +162,10 @@ public class DFSZKFailoverController extends ZKFailoverController { public static void main(String args[]) throws Exception { + if (DFSUtil.parseHelpArgument(args, + ZKFailoverController.USAGE, System.out, true)) { + System.exit(0); + } GenericOptionsParser parser = new GenericOptionsParser( new HdfsConfiguration(), args); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java index 566d77a5fbc..c3238f0de30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java @@ -73,6 +73,25 @@ public class DFSck extends Configured implements Tool { HdfsConfiguration.init(); } + private static final String USAGE = "Usage: DFSck " + + "[-list-corruptfileblocks | " + + "[-move | -delete | -openforwrite] " + + "[-files [-blocks [-locations | -racks]]]]\n" + + "\t\tstart checking from this path\n" + + "\t-move\tmove corrupted files to /lost+found\n" + + "\t-delete\tdelete corrupted files\n" + + "\t-files\tprint out files being checked\n" + + "\t-openforwrite\tprint out files opened for write\n" + + "\t-list-corruptfileblocks\tprint out list of missing " + + "blocks and files they belong to\n" + + "\t-blocks\tprint out block report\n" + + "\t-locations\tprint out locations for every block\n" + + "\t-racks\tprint out network topology for data-node locations\n" + + "\t\tBy default fsck ignores files opened for write, " + + "use -openforwrite to report such files. They are usually " + + " tagged CORRUPT or HEALTHY depending on their block " + + "allocation status"; + private final UserGroupInformation ugi; private final PrintStream out; @@ -93,25 +112,9 @@ public class DFSck extends Configured implements Tool { /** * Print fsck usage information */ - static void printUsage() { - System.err.println("Usage: DFSck [-list-corruptfileblocks | " + - "[-move | -delete | -openforwrite] " + - "[-files [-blocks [-locations | -racks]]]]"); - System.err.println("\t\tstart checking from this path"); - System.err.println("\t-move\tmove corrupted files to /lost+found"); - System.err.println("\t-delete\tdelete corrupted files"); - System.err.println("\t-files\tprint out files being checked"); - System.err.println("\t-openforwrite\tprint out files opened for write"); - System.err.println("\t-list-corruptfileblocks\tprint out list of missing " - + "blocks and files they belong to"); - System.err.println("\t-blocks\tprint out block report"); - System.err.println("\t-locations\tprint out locations for every block"); - System.err.println("\t-racks\tprint out network topology for data-node locations"); - System.err.println("\t\tBy default fsck ignores files opened for write, " + - "use -openforwrite to report such files. They are usually " + - " tagged CORRUPT or HEALTHY depending on their block " + - "allocation status"); - ToolRunner.printGenericCommandUsage(System.err); + static void printUsage(PrintStream out) { + out.println(USAGE + "\n"); + ToolRunner.printGenericCommandUsage(out); } /** * @param args @@ -119,7 +122,7 @@ public class DFSck extends Configured implements Tool { @Override public int run(final String[] args) throws IOException { if (args.length == 0) { - printUsage(); + printUsage(System.err); return -1; } @@ -258,12 +261,12 @@ public class DFSck extends Configured implements Tool { } else { System.err.println("fsck: can only operate on one path at a time '" + args[idx] + "'"); - printUsage(); + printUsage(System.err); return -1; } } else { System.err.println("fsck: Illegal option '" + args[idx] + "'"); - printUsage(); + printUsage(System.err); return -1; } } @@ -304,10 +307,14 @@ public class DFSck extends Configured implements Tool { // -files option is also used by GenericOptionsParser // Make sure that is not the first argument for fsck int res = -1; - if ((args.length == 0 ) || ("-files".equals(args[0]))) - printUsage(); - else + if ((args.length == 0) || ("-files".equals(args[0]))) { + printUsage(System.err); + ToolRunner.printGenericCommandUsage(System.err); + } else if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { + res = 0; + } else { res = ToolRunner.run(new DFSck(new HdfsConfiguration()), args); + } System.exit(res); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java index e0935d475cf..f74b4e88962 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java @@ -40,7 +40,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.HftpFileSystem; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -48,9 +47,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretMan import org.apache.hadoop.hdfs.server.namenode.CancelDelegationTokenServlet; import org.apache.hadoop.hdfs.server.namenode.GetDelegationTokenServlet; import org.apache.hadoop.hdfs.server.namenode.RenewDelegationTokenServlet; -import org.apache.hadoop.hdfs.web.URLUtils; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; @@ -71,8 +68,10 @@ public class DelegationTokenFetcher { private static final String CANCEL = "cancel"; private static final String RENEW = "renew"; private static final String PRINT = "print"; + private static final String HELP = "help"; + private static final String HELP_SHORT = "h"; - private static void printUsage(PrintStream err) throws IOException { + private static void printUsage(PrintStream err) { err.println("fetchdt retrieves delegation tokens from the NameNode"); err.println(); err.println("fetchdt "); @@ -107,6 +106,7 @@ public class DelegationTokenFetcher { fetcherOptions.addOption(CANCEL, false, "cancel the token"); fetcherOptions.addOption(RENEW, false, "renew the token"); fetcherOptions.addOption(PRINT, false, "print the token"); + fetcherOptions.addOption(HELP_SHORT, HELP, false, "print out help information"); GenericOptionsParser parser = new GenericOptionsParser(conf, fetcherOptions, args); CommandLine cmd = parser.getCommandLine(); @@ -119,9 +119,14 @@ public class DelegationTokenFetcher { final boolean cancel = cmd.hasOption(CANCEL); final boolean renew = cmd.hasOption(RENEW); final boolean print = cmd.hasOption(PRINT); + final boolean help = cmd.hasOption(HELP); String[] remaining = parser.getRemainingArgs(); // check option validity + if (help) { + printUsage(System.out); + System.exit(0); + } if (cancel && renew || cancel && print || renew && print || cancel && renew && print) { System.err.println("ERROR: Only specify cancel, renew or print."); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java index adf3293edf5..778ac59ee25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java @@ -324,6 +324,10 @@ public class GetConf extends Configured implements Tool { } public static void main(String[] args) throws Exception { + if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { + System.exit(0); + } + int res = ToolRunner.run(new GetConf(new HdfsConfiguration()), args); System.exit(res); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java index c0e415a8433..49d56d534fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -43,6 +44,8 @@ import org.apache.hadoop.util.ToolRunner; public class GetGroups extends GetGroupsBase { private static final Log LOG = LogFactory.getLog(GetGroups.class); + + static final String USAGE = "Usage: hdfs groups [username ...]"; static{ HdfsConfiguration.init(); @@ -86,6 +89,10 @@ public class GetGroups extends GetGroupsBase { } public static void main(String[] argv) throws Exception { + if (DFSUtil.parseHelpArgument(argv, USAGE, System.out, true)) { + System.exit(0); + } + int res = ToolRunner.run(new GetGroups(new HdfsConfiguration()), argv); System.exit(res); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java index 61e8ebef5c5..666e52b484f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java @@ -55,7 +55,9 @@ public class TestDFSHAAdmin { private DFSHAAdmin tool; private ByteArrayOutputStream errOutBytes = new ByteArrayOutputStream(); + private ByteArrayOutputStream outBytes = new ByteArrayOutputStream(); private String errOutput; + private String output; private HAServiceProtocol mockProtocol; private ZKFCProtocol mockZkfcProtocol; @@ -111,12 +113,14 @@ public class TestDFSHAAdmin { }; tool.setConf(getHAConf()); tool.setErrOut(new PrintStream(errOutBytes)); + tool.setOut(new PrintStream(outBytes)); } private void assertOutputContains(String string) { - if (!errOutput.contains(string)) { - fail("Expected output to contain '" + string + "' but was:\n" + - errOutput); + if (!errOutput.contains(string) && !output.contains(string)) { + fail("Expected output to contain '" + string + + "' but err_output was:\n" + errOutput + + "\n and output was: \n" + output); } } @@ -143,7 +147,7 @@ public class TestDFSHAAdmin { @Test public void testHelp() throws Exception { - assertEquals(-1, runTool("-help")); + assertEquals(0, runTool("-help")); assertEquals(0, runTool("-help", "transitionToActive")); assertOutputContains("Transitions the service into Active"); } @@ -378,10 +382,12 @@ public class TestDFSHAAdmin { private Object runTool(String ... args) throws Exception { errOutBytes.reset(); + outBytes.reset(); LOG.info("Running: DFSHAAdmin " + Joiner.on(" ").join(args)); int ret = tool.run(args); errOutput = new String(errOutBytes.toByteArray(), Charsets.UTF_8); - LOG.info("Output:\n" + errOutput); + output = new String(outBytes.toByteArray(), Charsets.UTF_8); + LOG.info("Err_output:\n" + errOutput + "\nOutput:\n" + output); return ret; } From 6c960396937038fa996707fb1a8a3ffa3d26cdc1 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 15 Aug 2012 00:33:53 +0000 Subject: [PATCH 13/18] HDFS-3803. Change BlockPoolSliceScanner chatty INFO log to DEBUG. Contributed by Andrew Purtell git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373174 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 64a4fdf568e..ccd6dbceb1a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -117,6 +117,9 @@ Trunk (unreleased changes) HDFS-3723. Add support -h, -help to all the commands. (Jing Zhao via suresh) + HDFS-3803. Change BlockPoolSliceScanner chatty INFO log to DEBUG. + (Andrew Purtell via suresh) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java index 142880e0268..3e6d9022232 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java @@ -554,9 +554,11 @@ class BlockPoolSliceScanner { } private synchronized void startNewPeriod() { - LOG.info("Starting a new period : work left in prev period : " + if (LOG.isDebugEnabled()) { + LOG.debug("Starting a new period : work left in prev period : " + String.format("%.2f%%", totalBytesToScan == 0 ? 0 : (bytesLeft * 100.0) / totalBytesToScan)); + } // reset the byte counts : bytesLeft = totalBytesToScan; From f41497c1419708789aca2a0236cd1a5111f71cf3 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 15 Aug 2012 02:23:21 +0000 Subject: [PATCH 14/18] HDFS-3794. WebHDFS OPEN returns the incorrect Content-Length in the HTTP header when offset is specified and length is omitted. Contributed by Ravi Prakash git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373193 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++++ .../server/datanode/web/resources/DatanodeWebHdfsMethods.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ccd6dbceb1a..135dacbd522 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -605,6 +605,10 @@ Branch-2 ( Unreleased changes ) HDFS-3658. Fix bugs in TestDFSClientRetries and add more tests. (szetszwo) + HDFS-3794. WebHDFS OPEN returns the incorrect Content-Length in the HTTP + header when offset is specified and length is omitted. + (Ravi Prakash via szetszwo) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java index d8af04c745a..2aaf157e6f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java @@ -411,7 +411,7 @@ public class DatanodeWebHdfsMethods { } final long n = length.getValue() != null? length.getValue() - : in.getVisibleLength(); + : in.getVisibleLength() - offset.getValue(); return Response.ok(new OpenEntity(in, n, dfsclient)).type( MediaType.APPLICATION_OCTET_STREAM).build(); } From fd8ce04dc555f5bcf32d35b0567db9568c6d1c3d Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 15 Aug 2012 04:21:04 +0000 Subject: [PATCH 15/18] HADOOP-8699. some common testcases create core-site.xml in test-classes making other testcases to fail. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373206 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/http/HttpConfig.java | 6 ++++++ .../org/apache/hadoop/http/TestSSLHttpServer.java | 14 +++++++++++--- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 50e8a08d0a2..7a744221a3e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -420,6 +420,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8660. TestPseudoAuthenticator failing with NPE. (tucu) + HADOOP-8699. some common testcases create core-site.xml in test-classes + making other testcases to fail. (tucu) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java index 4ee2f5582f8..d906d9642a7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.http; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -37,6 +38,11 @@ public class HttpConfig { CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_DEFAULT); } + @VisibleForTesting + static void setSecure(boolean secure) { + sslEnabled = secure; + } + public static boolean isSecure() { return sslEnabled; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java index f5ab9572255..880804ec2c5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java @@ -41,6 +41,8 @@ import java.net.URL; * corresponding HTTPS URL. */ public class TestSSLHttpServer extends HttpServerFunctionalTest { + private static final String CONFIG_SITE_XML = "sslhttpserver-site.xml"; + private static final String BASEDIR = System.getProperty("test.build.dir", "target/test-dir") + "/" + TestSSLHttpServer.class.getSimpleName(); @@ -49,8 +51,10 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest { private static HttpServer server; private static URL baseUrl; + @Before public void setup() throws Exception { + HttpConfig.setSecure(true); File base = new File(BASEDIR); FileUtil.fullyDelete(base); base.mkdirs(); @@ -66,11 +70,12 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest { //we do this trick because the MR AppMaster is started in another VM and //the HttpServer configuration is not loaded from the job.xml but from the //site.xml files in the classpath - Writer writer = new FileWriter(classpathDir + "/core-site.xml"); + Writer writer = new FileWriter(new File(classpathDir, CONFIG_SITE_XML)); conf.writeXml(writer); writer.close(); conf.setInt(HttpServer.HTTP_MAX_THREADS, 10); + conf.addResource(CONFIG_SITE_XML); server = createServer("test", conf); server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class); server.start(); @@ -83,7 +88,8 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest { server.stop(); String classpathDir = KeyStoreTestUtil.getClasspathDir(TestSSLHttpServer.class); - new File(classpathDir + "/core-site.xml").delete(); + new File(classpathDir, CONFIG_SITE_XML).delete(); + HttpConfig.setSecure(false); } @@ -98,7 +104,9 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest { private static String readOut(URL url) throws Exception { StringBuilder out = new StringBuilder(); HttpsURLConnection conn = (HttpsURLConnection) url.openConnection(); - SSLFactory sslf = new SSLFactory(SSLFactory.Mode.CLIENT, new Configuration()); + Configuration conf = new Configuration(); + conf.addResource(CONFIG_SITE_XML); + SSLFactory sslf = new SSLFactory(SSLFactory.Mode.CLIENT, conf); sslf.init(); conn.setSSLSocketFactory(sslf.createSSLSocketFactory()); InputStream in = conn.getInputStream(); From 5aeabcdf6f403afd2517ef0fa169de6b71063d27 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 15 Aug 2012 06:40:29 +0000 Subject: [PATCH 16/18] HADOOP-7754. Expose file descriptors from Hadoop-wrapped local FileSystems (todd and ahmed via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373235 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../hadoop/fs/BufferedFSInputStream.java | 13 +++++- .../apache/hadoop/fs/FSDataInputStream.java | 13 +++++- .../apache/hadoop/fs/HasFileDescriptor.java | 40 +++++++++++++++++++ .../apache/hadoop/fs/RawLocalFileSystem.java | 8 +++- .../apache/hadoop/fs/TestLocalFileSystem.java | 10 +++++ 6 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasFileDescriptor.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7a744221a3e..305f1528854 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -215,6 +215,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8581. add support for HTTPS to the web UIs. (tucu) + HADOOP-7754. Expose file descriptors from Hadoop-wrapped local + FileSystems (todd and ahmed via tucu) + IMPROVEMENTS HADOOP-8340. SNAPSHOT build versions should compare as less than their eventual diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java index bb9d39c4f4a..f3229240125 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java @@ -18,6 +18,8 @@ package org.apache.hadoop.fs; import java.io.BufferedInputStream; +import java.io.FileDescriptor; +import java.io.FileInputStream; import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; @@ -31,7 +33,7 @@ import org.apache.hadoop.classification.InterfaceStability; @InterfaceAudience.Private @InterfaceStability.Unstable public class BufferedFSInputStream extends BufferedInputStream -implements Seekable, PositionedReadable { +implements Seekable, PositionedReadable, HasFileDescriptor { /** * Creates a BufferedFSInputStream * with the specified buffer size, @@ -97,4 +99,13 @@ implements Seekable, PositionedReadable { public void readFully(long position, byte[] buffer) throws IOException { ((FSInputStream)in).readFully(position, buffer); } + + @Override + public FileDescriptor getFileDescriptor() throws IOException { + if (in instanceof HasFileDescriptor) { + return ((HasFileDescriptor) in).getFileDescriptor(); + } else { + return null; + } + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java index 3b14cc77e1f..e47dffb082c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java @@ -28,7 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability; @InterfaceAudience.Public @InterfaceStability.Stable public class FSDataInputStream extends DataInputStream - implements Seekable, PositionedReadable, Closeable, ByteBufferReadable { + implements Seekable, PositionedReadable, Closeable, ByteBufferReadable, HasFileDescriptor { public FSDataInputStream(InputStream in) throws IOException { @@ -125,4 +125,15 @@ public class FSDataInputStream extends DataInputStream throw new UnsupportedOperationException("Byte-buffer read unsupported by input stream"); } + + @Override + public FileDescriptor getFileDescriptor() throws IOException { + if (in instanceof HasFileDescriptor) { + return ((HasFileDescriptor) in).getFileDescriptor(); + } else if (in instanceof FileInputStream) { + return ((FileInputStream) in).getFD(); + } else { + return null; + } + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasFileDescriptor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasFileDescriptor.java new file mode 100644 index 00000000000..bcf325ceca5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasFileDescriptor.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs; + +import java.io.FileDescriptor; +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Having a FileDescriptor + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface HasFileDescriptor { + + /** + * @return the FileDescriptor + * @throws IOException + */ + public FileDescriptor getFileDescriptor() throws IOException; + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index 58492e13181..38e991480af 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -26,6 +26,7 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.io.FileDescriptor; import java.net.URI; import java.nio.ByteBuffer; import java.util.Arrays; @@ -111,7 +112,7 @@ public class RawLocalFileSystem extends FileSystem { /******************************************************* * For open()'s FSInputStream. *******************************************************/ - class LocalFSFileInputStream extends FSInputStream { + class LocalFSFileInputStream extends FSInputStream implements HasFileDescriptor { private FileInputStream fis; private long position; @@ -181,6 +182,11 @@ public class RawLocalFileSystem extends FileSystem { } return value; } + + @Override + public FileDescriptor getFileDescriptor() throws IOException { + return fis.getFD(); + } } public FSDataInputStream open(Path f, int bufferSize) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index 604ea78d0fb..4d821f96f81 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -248,4 +248,14 @@ public class TestLocalFileSystem { } assertEquals(1, fileSchemeCount); } + + public void testHasFileDescriptor() throws IOException { + Configuration conf = new Configuration(); + LocalFileSystem fs = FileSystem.getLocal(conf); + Path path = new Path(TEST_ROOT_DIR, "test-file"); + writeFile(fs, path, 1); + BufferedFSInputStream bis = new BufferedFSInputStream( + new RawLocalFileSystem().new LocalFSFileInputStream(path), 1024); + assertNotNull(bis.getFileDescriptor()); + } } From 9dff7876d9cff64c857767ed517de6fc6c4c63ee Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 15 Aug 2012 08:07:29 +0000 Subject: [PATCH 17/18] HDFS-3802. StartupOption.name in HdfsServerConstants should be final. Contributed by Jing Zhao git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373253 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../apache/hadoop/hdfs/server/common/HdfsServerConstants.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 135dacbd522..1d8c25975cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -394,6 +394,9 @@ Branch-2 ( Unreleased changes ) HDFS-3765. namenode -initializeSharedEdits should be able to initialize all shared storages. (Vinay and todd via todd) + HDFS-3802. StartupOption.name in HdfsServerConstants should be final. + (Jing Zhao via szetszwo) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 993869512d0..c914035230d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -60,7 +60,7 @@ public final class HdfsServerConstants { FORCE("-force"), NONINTERACTIVE("-nonInteractive"); - private String name = null; + private final String name; // Used only with format and upgrade options private String clusterId = null; From 89153a9c684bbafd9a1309ca2a20347251c577e8 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 15 Aug 2012 18:53:13 +0000 Subject: [PATCH 18/18] HDFS-3796. Speed up edit log tests by avoiding fsync(). Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373567 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../namenode/EditLogFileOutputStream.java | 17 ++++++++++++++++- .../hdfs/server/namenode/TestEditLog.java | 5 +++++ .../namenode/TestEditLogFileOutputStream.java | 6 ++++++ .../server/namenode/TestFileJournalManager.java | 6 ++++++ .../server/namenode/TestNameNodeRecovery.java | 1 + .../namenode/TestSecurityTokenEditLog.java | 6 ++++++ .../namenode/ha/TestEditLogsDuringFailover.java | 7 +++++++ 8 files changed, 49 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1d8c25975cb..ce3643fb34e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -397,6 +397,8 @@ Branch-2 ( Unreleased changes ) HDFS-3802. StartupOption.name in HdfsServerConstants should be final. (Jing Zhao via szetszwo) + HDFS-3796. Speed up edit log tests by avoiding fsync() (todd) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java index bb181d743b6..f7a8b337a6d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java @@ -49,6 +49,8 @@ public class EditLogFileOutputStream extends EditLogOutputStream { private EditsDoubleBuffer doubleBuf; static ByteBuffer fill = ByteBuffer.allocateDirect(MIN_PREALLOCATION_LENGTH); + private static boolean shouldSkipFsyncForTests = false; + static { fill.position(0); for (int i = 0; i < fill.capacity(); i++) { @@ -184,7 +186,9 @@ public class EditLogFileOutputStream extends EditLogOutputStream { } preallocate(); // preallocate file if necessay doubleBuf.flushTo(fp); - fc.force(false); // metadata updates not needed + if (!shouldSkipFsyncForTests) { + fc.force(false); // metadata updates not needed + } } /** @@ -247,4 +251,15 @@ public class EditLogFileOutputStream extends EditLogOutputStream { public FileChannel getFileChannelForTesting() { return fc; } + + /** + * For the purposes of unit tests, we don't need to actually + * write durably to disk. So, we can skip the fsync() calls + * for a speed improvement. + * @param skip true if fsync should not be called + */ + @VisibleForTesting + public static void setShouldSkipFsyncForTesting(boolean skip) { + shouldSkipFsyncForTests = skip; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java index e7abc1132d0..d9ac54ed0ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java @@ -119,6 +119,11 @@ public class TestEditLog { "a4ff 0000 0000 0000 0000 0000 0000 0000" ).replace(" ","")); + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } static final byte TRAILER_BYTE = FSEditLogOpCodes.OP_INVALID.getOpCode(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java index ead94968eeb..22ab02d2a9d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java @@ -40,6 +40,12 @@ public class TestEditLogFileOutputStream { final static int MIN_PREALLOCATION_LENGTH = EditLogFileOutputStream.MIN_PREALLOCATION_LENGTH; + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } + @Before @After public void deleteEditsFile() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java index 1a968c7d1d1..722e4e24bf3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java @@ -51,6 +51,12 @@ import com.google.common.collect.ImmutableList; public class TestFileJournalManager { static final Log LOG = LogFactory.getLog(TestFileJournalManager.class); + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } + /** * Find out how many transactions we can read from a * FileJournalManager, starting at a given transaction ID. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java index 1717bb04125..23fd3b51a71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java @@ -57,6 +57,7 @@ public class TestNameNodeRecovery { static { recoverStartOpt.setForce(MetaRecoveryContext.FORCE_ALL); + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); } static void runEditLogTest(EditLogTestSetup elts) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java index e3056e9b0bf..dd679d1a9af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java @@ -49,6 +49,12 @@ public class TestSecurityTokenEditLog { static final int NUM_THREADS = 100; static final int opsPerTrans = 3; + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } + // // an object that does a bunch of transactions // diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java index dd5c1bab75c..5e18381e7a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.hdfs.server.namenode.NNStorage; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; @@ -52,6 +53,12 @@ public class TestEditLogsDuringFailover { private static final Log LOG = LogFactory.getLog(TestEditLogsDuringFailover.class); private static final int NUM_DIRS_IN_LOG = 5; + + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } @Test public void testStartup() throws Exception {