From 5dfe62d845424e688c144fc10446ff6392e54ae4 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Tue, 24 Jan 2012 06:05:52 +0000 Subject: [PATCH] HDFS-2397. Undeprecate SecondaryNameNode. Contributed by Eli Collins git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1235135 13f79535-47bb-0310-9956-ffa450edef68 --- .../content/xdocs/commands_manual.xml | 5 ---- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../content/xdocs/hdfs_user_guide.xml | 20 +++++++------- .../server/namenode/SecondaryNameNode.java | 1 - .../hdfs/server/namenode/TestCheckpoint.java | 26 ------------------- .../server/namenode/TestNameEditsConfigs.java | 4 --- .../server/namenode/TestSecondaryWebUi.java | 1 - .../hdfs/server/namenode/TestStartup.java | 2 -- .../server/namenode/TestStorageRestore.java | 2 -- .../src/site/apt/ClusterSetup.apt.vm | 20 +++++++------- 10 files changed, 21 insertions(+), 62 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/commands_manual.xml b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/commands_manual.xml index b3f25af40e4..19df2fe9cf5 100644 --- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/commands_manual.xml +++ b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/commands_manual.xml @@ -753,11 +753,6 @@
secondarynamenode - - The Secondary NameNode has been deprecated. Instead, consider using the - Checkpoint Node or - Backup Node. -

Runs the HDFS secondary namenode. See Secondary NameNode diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7511d5959e1..d355bf4f803 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -276,6 +276,8 @@ Release 0.23.1 - UNRELEASED HDFS-2818. Fix a missing space issue in HDFS webapps' title tags. (Devaraj K via harsh) + HDFS-2397. Undeprecate SecondaryNameNode (eli) + OPTIMIZATIONS HDFS-2130. Switch default checksum to CRC32C. (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_user_guide.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_user_guide.xml index 0d3ed89c7f8..4d2c6dd0735 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_user_guide.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_user_guide.xml @@ -112,17 +112,18 @@ problems.

  • - Secondary NameNode (deprecated): performs periodic checkpoints of the + Secondary NameNode: performs periodic checkpoints of the namespace and helps keep the size of file containing log of HDFS modifications within certain limits at the NameNode. - Replaced by Checkpoint node.
    • +
  • Checkpoint node: performs periodic checkpoints of the namespace and helps minimize the size of the log stored at the NameNode containing changes to the HDFS. - Replaces the role previously filled by the Secondary NameNode. - NameNode allows multiple Checkpoint nodes simultaneously, + Replaces the role previously filled by the Secondary NameNode, + though is not yet battle hardened. + The NameNode allows multiple Checkpoint nodes simultaneously, as long as there are no Backup nodes registered with the system.
  • @@ -132,6 +133,7 @@ which is always in sync with the active NameNode namespace state. Only one Backup node may be registered with the NameNode at once.
    • + @@ -234,12 +236,6 @@
    Secondary NameNode - - The Secondary NameNode has been deprecated. - Instead, consider using the - Checkpoint Node or - Backup Node. -

    The NameNode stores modifications to the file system as a log appended to a native file system file, edits. @@ -287,7 +283,9 @@ secondarynamenode.

    -
    Checkpoint Node +
    + +
    Checkpoint Node

    NameNode persists its namespace using two files: fsimage, which is the latest checkpoint of the namespace and edits, a journal (log) of changes to the namespace since the checkpoint. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index d84b1a2dd03..f605f617527 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -87,7 +87,6 @@ * primary NameNode. * **********************************************************/ -@Deprecated // use BackupNode with -checkpoint argument instead. @InterfaceAudience.Private public class SecondaryNameNode implements Runnable { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java index 1717f6c0b2e..52da95725ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java @@ -204,7 +204,6 @@ public void testWriteTransactionIdHandlesIOE() throws Exception { /* * Simulate namenode crashing after rolling edit log. */ - @SuppressWarnings("deprecation") public void testSecondaryNamenodeError1() throws IOException { LOG.info("Starting testSecondaryNamenodeError1"); @@ -266,7 +265,6 @@ public void testSecondaryNamenodeError1() /* * Simulate a namenode crash after uploading new image */ - @SuppressWarnings("deprecation") public void testSecondaryNamenodeError2() throws IOException { LOG.info("Starting testSecondaryNamenodeError2"); Configuration conf = new HdfsConfiguration(); @@ -325,7 +323,6 @@ public void testSecondaryNamenodeError2() throws IOException { /* * Simulate a secondary namenode crash after rolling the edit log. */ - @SuppressWarnings("deprecation") public void testSecondaryNamenodeError3() throws IOException { LOG.info("Starting testSecondaryNamenodeError3"); Configuration conf = new HdfsConfiguration(); @@ -395,7 +392,6 @@ public void testSecondaryNamenodeError3() throws IOException { * back to the name-node. * Used to truncate primary fsimage file. */ - @SuppressWarnings("deprecation") public void testSecondaryFailsToReturnImage() throws IOException { LOG.info("Starting testSecondaryFailsToReturnImage"); Configuration conf = new HdfsConfiguration(); @@ -472,7 +468,6 @@ public void testNameNodeImageSendFailWrongDigest() * @param errorType the ErrorSimulator type to trigger * @param exceptionSubstring an expected substring of the triggered exception */ - @SuppressWarnings("deprecation") private void doSendFailTest(int errorType, String exceptionSubstring) throws IOException { Configuration conf = new HdfsConfiguration(); @@ -587,7 +582,6 @@ public void testSeparateEditsDirLocking() throws IOException { /** * Test that the SecondaryNameNode properly locks its storage directories. */ - @SuppressWarnings("deprecation") public void testSecondaryNameNodeLocking() throws Exception { // Start a primary NN so that the secondary will start successfully Configuration conf = new HdfsConfiguration(); @@ -680,7 +674,6 @@ private static void assertClusterStartFailsWhenDirLocked( * 2. if the NN does not contain an image, importing a checkpoint * succeeds and re-saves the image */ - @SuppressWarnings("deprecation") public void testImportCheckpoint() throws Exception { Configuration conf = new HdfsConfiguration(); Path testPath = new Path("/testfile"); @@ -761,16 +754,12 @@ private static void removeAndRecreateDir(File dir) throws IOException { throw new IOException("Cannot create directory " + dir); } - // This deprecation suppress warning does not work due to known Java bug: - // http://bugs.sun.com/view_bug.do?bug_id=6460147 - @SuppressWarnings("deprecation") SecondaryNameNode startSecondaryNameNode(Configuration conf ) throws IOException { conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0"); return new SecondaryNameNode(conf); } - @SuppressWarnings("deprecation") SecondaryNameNode startSecondaryNameNode(Configuration conf, int index) throws IOException { Configuration snnConf = new Configuration(conf); @@ -783,7 +772,6 @@ SecondaryNameNode startSecondaryNameNode(Configuration conf, int index) /** * Tests checkpoint in HDFS. */ - @SuppressWarnings("deprecation") public void testCheckpoint() throws IOException { Path file1 = new Path("checkpoint.dat"); Path file2 = new Path("checkpoint2.dat"); @@ -1010,7 +998,6 @@ public void testCheckpointSignature() throws IOException { * - it then fails again for the same reason * - it then tries to checkpoint a third time */ - @SuppressWarnings("deprecation") public void testCheckpointAfterTwoFailedUploads() throws IOException { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; @@ -1065,7 +1052,6 @@ public void testCheckpointAfterTwoFailedUploads() throws IOException { * * @throws IOException */ - @SuppressWarnings("deprecation") public void testMultipleSecondaryNamenodes() throws IOException { Configuration conf = new HdfsConfiguration(); String nameserviceId1 = "ns1"; @@ -1115,7 +1101,6 @@ public void testMultipleSecondaryNamenodes() throws IOException { * Test that the secondary doesn't have to re-download image * if it hasn't changed. */ - @SuppressWarnings("deprecation") public void testSecondaryImageDownload() throws IOException { LOG.info("Starting testSecondaryImageDownload"); Configuration conf = new HdfsConfiguration(); @@ -1198,7 +1183,6 @@ public void testSecondaryImageDownload() throws IOException { * It verifies that this works even though the earlier-txid checkpoint gets * uploaded after the later-txid checkpoint. */ - @SuppressWarnings("deprecation") public void testMultipleSecondaryNNsAgainstSameNN() throws Exception { Configuration conf = new HdfsConfiguration(); @@ -1284,7 +1268,6 @@ public void testMultipleSecondaryNNsAgainstSameNN() throws Exception { * It verifies that one of the two gets an error that it's uploading a * duplicate checkpoint, and the other one succeeds. */ - @SuppressWarnings("deprecation") public void testMultipleSecondaryNNsAgainstSameNN2() throws Exception { Configuration conf = new HdfsConfiguration(); @@ -1383,7 +1366,6 @@ protected Object passThrough(InvocationOnMock invocation) throws Throwable { * is running. The secondary should shut itself down if if talks to a NN * with the wrong namespace. */ - @SuppressWarnings("deprecation") public void testReformatNNBetweenCheckpoints() throws IOException { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; @@ -1638,7 +1620,6 @@ public void testCheckpointWithSeparateDirsAfterNameFails() throws Exception { /** * Test that the 2NN triggers a checkpoint after the configurable interval */ - @SuppressWarnings("deprecation") public void testCheckpointTriggerOnTxnCount() throws Exception { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; @@ -1692,7 +1673,6 @@ public Boolean get() { * logs that connect the 2NN's old checkpoint to the current txid * get archived. Then, the 2NN tries to checkpoint again. */ - @SuppressWarnings("deprecation") public void testSecondaryHasVeryOutOfDateImage() throws IOException { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; @@ -1730,7 +1710,6 @@ public void testSecondaryHasVeryOutOfDateImage() throws IOException { } } - @SuppressWarnings("deprecation") public void testCommandLineParsing() throws ParseException { SecondaryNameNode.CommandLineOpts opts = new SecondaryNameNode.CommandLineOpts(); @@ -1765,7 +1744,6 @@ public void testCommandLineParsing() throws ParseException { } catch (ParseException e) {} } - @SuppressWarnings("deprecation") private void cleanup(SecondaryNameNode snn) { if (snn != null) { try { @@ -1781,7 +1759,6 @@ private void cleanup(SecondaryNameNode snn) { * Assert that if any two files have the same name across the 2NNs * and NN, they should have the same content too. */ - @SuppressWarnings("deprecation") private void assertParallelFilesInvariant(MiniDFSCluster cluster, ImmutableList secondaries) throws Exception { List allCurrentDirs = Lists.newArrayList(); @@ -1793,7 +1770,6 @@ private void assertParallelFilesInvariant(MiniDFSCluster cluster, ImmutableSet.of("VERSION")); } - @SuppressWarnings("deprecation") private List getCheckpointCurrentDirs(SecondaryNameNode secondary) { List ret = Lists.newArrayList(); for (URI u : secondary.getCheckpointDirs()) { @@ -1803,7 +1779,6 @@ private List getCheckpointCurrentDirs(SecondaryNameNode secondary) { return ret; } - @SuppressWarnings("deprecation") private CheckpointStorage spyOnSecondaryImage(SecondaryNameNode secondary1) { CheckpointStorage spy = Mockito.spy((CheckpointStorage)secondary1.getFSImage());; secondary1.setFSImage(spy); @@ -1813,7 +1788,6 @@ private CheckpointStorage spyOnSecondaryImage(SecondaryNameNode secondary1) { /** * A utility class to perform a checkpoint in a different thread. */ - @SuppressWarnings("deprecation") private static class DoCheckpointThread extends Thread { private final SecondaryNameNode snn; private volatile Throwable thrown = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java index 577eab08987..5a188152c64 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java @@ -120,9 +120,6 @@ private void cleanupFile(FileSystem fileSys, Path name) assertTrue(!fileSys.exists(name)); } - // This deprecation suppress warning does not work due to known Java bug: - // http://bugs.sun.com/view_bug.do?bug_id=6460147 - @SuppressWarnings("deprecation") SecondaryNameNode startSecondaryNameNode(Configuration conf ) throws IOException { conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0"); @@ -142,7 +139,6 @@ SecondaryNameNode startSecondaryNameNode(Configuration conf * sure we are reading proper edits and image. * @throws Exception */ - @SuppressWarnings("deprecation") @Test public void testNameEditsConfigs() throws Exception { Path file1 = new Path("TestNameEditsConfigs1"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryWebUi.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryWebUi.java index b11e26c0644..16b2ac6c65e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryWebUi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryWebUi.java @@ -30,7 +30,6 @@ public class TestSecondaryWebUi { - @SuppressWarnings("deprecation") @Test public void testSecondaryWebUi() throws IOException { Configuration conf = new Configuration(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java index ceb6261db4b..92330096284 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java @@ -121,7 +121,6 @@ public void tearDown() throws Exception { * start MiniDFScluster, create a file (to create edits) and do a checkpoint * @throws IOException */ - @SuppressWarnings("deprecation") public void createCheckPoint() throws IOException { LOG.info("--starting mini cluster"); // manage dirs parameter set to false @@ -303,7 +302,6 @@ public void testChkpointStartup1() throws IOException{ * secondary node copies fsimage and edits into correct separate directories. * @throws IOException */ - @SuppressWarnings("deprecation") public void testSNNStartup() throws IOException{ //setUpConfig(); LOG.info("--starting SecondNN startup test"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java index f3a4638f10b..412ab4a4611 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java @@ -155,7 +155,6 @@ private void printStorages(FSImage image) { * 7. run doCheckpoint * 8. verify that all the image and edits files are the same. */ - @SuppressWarnings("deprecation") @Test public void testStorageRestore() throws Exception { int numDatanodes = 0; @@ -312,7 +311,6 @@ public void testDfsAdminCmd() throws Exception { * then try to perform a checkpoint. The NN should not serve up the image or * edits from the restored (empty) dir. */ - @SuppressWarnings("deprecation") @Test public void testMultipleSecondaryCheckpoint() throws IOException { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm index 8fe515babe2..0290c23b8ae 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm @@ -95,7 +95,7 @@ Hadoop MapReduce Next Generation - Cluster Setup *--------------------------------------+--------------------------------------+ | DataNode | HADOOP_DATANODE_OPTS | *--------------------------------------+--------------------------------------+ -| Backup NameNode | HADOOP_SECONDARYNAMENODE_OPTS | +| Secondary NameNode | HADOOP_SECONDARYNAMENODE_OPTS | *--------------------------------------+--------------------------------------+ | ResourceManager | YARN_RESOURCEMANAGER_OPTS | *--------------------------------------+--------------------------------------+ @@ -537,15 +537,15 @@ Hadoop MapReduce Next Generation - Cluster Setup It's recommended to have them share a Unix group, for e.g. <<>>. -*--------------------------------------+--------------------------------------+ -|| User:Group || Daemons | -*--------------------------------------+--------------------------------------+ -| hdfs:hadoop | NameNode, Backup NameNode, DataNode | -*--------------------------------------+--------------------------------------+ -| yarn:hadoop | ResourceManager, NodeManager | -*--------------------------------------+--------------------------------------+ -| mapred:hadoop | MapReduce JobHistory Server | -*--------------------------------------+--------------------------------------+ +*--------------------------------------+----------------------------------------------------------------------+ +|| User:Group || Daemons | +*--------------------------------------+----------------------------------------------------------------------+ +| hdfs:hadoop | NameNode, Secondary NameNode, Checkpoint Node, Backup Node, DataNode | +*--------------------------------------+----------------------------------------------------------------------+ +| yarn:hadoop | ResourceManager, NodeManager | +*--------------------------------------+----------------------------------------------------------------------+ +| mapred:hadoop | MapReduce JobHistory Server | +*--------------------------------------+----------------------------------------------------------------------+ * <<>>