Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-2397. svn merge -c 1235135 from trunk 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1235136 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2012-01-24 06:10:21 +00:00
parent 439446057d
commit 517766b98b
10 changed files with 21 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/commands_manual.xml
View File

@ -753,11 +753,6 @@
<section>
<title> secondarynamenode </title>
<note>
The Secondary NameNode has been deprecated. Instead, consider using the
<a href="http://hadoop.apache.org/hdfs/docs/current/hdfs_user_guide.html#Checkpoint+Node">Checkpoint Node</a> or
<a href="http://hadoop.apache.org/hdfs/docs/current/hdfs_user_guide.html#Backup+Node">Backup Node</a>.
</note>
<p>
Runs the HDFS secondary
namenode. See <a href="http://hadoop.apache.org/hdfs/docs/current/hdfs_user_guide.html#Secondary+NameNode">Secondary NameNode</a>

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -96,6 +96,8 @@ Release 0.23.1 - UNRELEASED
HDFS-2818. Fix a missing space issue in HDFS webapps' title tags. (Devaraj K via harsh)
HDFS-2397. Undeprecate SecondaryNameNode (eli)
OPTIMIZATIONS
HDFS-2130. Switch default checksum to CRC32C. (todd)

20
hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_user_guide.xml
View File

@ -112,17 +112,18 @@
problems.
</li>
<li>
Secondary NameNode (deprecated): performs periodic checkpoints of the
Secondary NameNode: performs periodic checkpoints of the
namespace and helps keep the size of file containing log of HDFS
modifications within certain limits at the NameNode.
Replaced by Checkpoint node.
</li>
<li>
Checkpoint node: performs periodic checkpoints of the namespace and
helps minimize the size of the log stored at the NameNode
containing changes to the HDFS.
Replaces the role previously filled by the Secondary NameNode.
NameNode allows multiple Checkpoint nodes simultaneously,
Replaces the role previously filled by the Secondary NameNode,
though is not yet battle hardened.
The NameNode allows multiple Checkpoint nodes simultaneously,
as long as there are no Backup nodes registered with the system.
</li>
<li>
@ -132,6 +133,7 @@
which is always in sync with the active NameNode namespace state.
Only one Backup node may be registered with the NameNode at once.
</li>
</ul>
</li>
</ul>
@ -234,12 +236,6 @@
</section>
<section> <title>Secondary NameNode</title>
<note>
The Secondary NameNode has been deprecated.
Instead, consider using the
<a href="hdfs_user_guide.html#Checkpoint+node">Checkpoint Node</a> or
<a href="hdfs_user_guide.html#Backup+node">Backup Node</a>.
</note>
<p>
The NameNode stores modifications to the file system as a log
appended to a native file system file, <code>edits</code>.
@ -287,7 +283,9 @@
<a href="http://hadoop.apache.org/common/docs/current/commands_manual.html#secondarynamenode">secondarynamenode</a>.
</p>
</section><section> <title> Checkpoint Node </title>
</section>
<section> <title> Checkpoint Node </title>
<p>NameNode persists its namespace using two files: <code>fsimage</code>,
which is the latest checkpoint of the namespace and <code>edits</code>,
a journal (log) of changes to the namespace since the checkpoint.

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
View File

@ -87,7 +87,6 @@ import com.google.common.collect.ImmutableList;
* primary NameNode.
*
**********************************************************/
@Deprecated // use BackupNode with -checkpoint argument instead.
@InterfaceAudience.Private
public class SecondaryNameNode implements Runnable {

26
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
View File

@ -203,7 +203,6 @@ public class TestCheckpoint extends TestCase {
/*
* Simulate namenode crashing after rolling edit log.
*/
@SuppressWarnings("deprecation")
public void testSecondaryNamenodeError1()
throws IOException {
LOG.info("Starting testSecondaryNamenodeError1");
@ -265,7 +264,6 @@ public class TestCheckpoint extends TestCase {
/*
* Simulate a namenode crash after uploading new image
*/
@SuppressWarnings("deprecation")
public void testSecondaryNamenodeError2() throws IOException {
LOG.info("Starting testSecondaryNamenodeError2");
Configuration conf = new HdfsConfiguration();
@ -324,7 +322,6 @@ public class TestCheckpoint extends TestCase {
/*
* Simulate a secondary namenode crash after rolling the edit log.
*/
@SuppressWarnings("deprecation")
public void testSecondaryNamenodeError3() throws IOException {
LOG.info("Starting testSecondaryNamenodeError3");
Configuration conf = new HdfsConfiguration();
@ -394,7 +391,6 @@ public class TestCheckpoint extends TestCase {
* back to the name-node.
* Used to truncate primary fsimage file.
*/
@SuppressWarnings("deprecation")
public void testSecondaryFailsToReturnImage() throws IOException {
LOG.info("Starting testSecondaryFailsToReturnImage");
Configuration conf = new HdfsConfiguration();
@ -471,7 +467,6 @@ public class TestCheckpoint extends TestCase {
* @param errorType the ErrorSimulator type to trigger
* @param exceptionSubstring an expected substring of the triggered exception
*/
@SuppressWarnings("deprecation")
private void doSendFailTest(int errorType, String exceptionSubstring)
throws IOException {
Configuration conf = new HdfsConfiguration();
@ -586,7 +581,6 @@ public class TestCheckpoint extends TestCase {
/**
* Test that the SecondaryNameNode properly locks its storage directories.
*/
@SuppressWarnings("deprecation")
public void testSecondaryNameNodeLocking() throws Exception {
// Start a primary NN so that the secondary will start successfully
Configuration conf = new HdfsConfiguration();
@ -679,7 +673,6 @@ public class TestCheckpoint extends TestCase {
* 2. if the NN does not contain an image, importing a checkpoint
* succeeds and re-saves the image
*/
@SuppressWarnings("deprecation")
public void testImportCheckpoint() throws Exception {
Configuration conf = new HdfsConfiguration();
Path testPath = new Path("/testfile");
@ -760,16 +753,12 @@ public class TestCheckpoint extends TestCase {
throw new IOException("Cannot create directory " + dir);
}
// This deprecation suppress warning does not work due to known Java bug:
// http://bugs.sun.com/view_bug.do?bug_id=6460147
@SuppressWarnings("deprecation")
SecondaryNameNode startSecondaryNameNode(Configuration conf
) throws IOException {
conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0");
return new SecondaryNameNode(conf);
}
@SuppressWarnings("deprecation")
SecondaryNameNode startSecondaryNameNode(Configuration conf, int index)
throws IOException {
Configuration snnConf = new Configuration(conf);
@ -782,7 +771,6 @@ public class TestCheckpoint extends TestCase {
/**
* Tests checkpoint in HDFS.
*/
@SuppressWarnings("deprecation")
public void testCheckpoint() throws IOException {
Path file1 = new Path("checkpoint.dat");
Path file2 = new Path("checkpoint2.dat");
@ -1009,7 +997,6 @@ public class TestCheckpoint extends TestCase {
* - it then fails again for the same reason
* - it then tries to checkpoint a third time
*/
@SuppressWarnings("deprecation")
public void testCheckpointAfterTwoFailedUploads() throws IOException {
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
@ -1064,7 +1051,6 @@ public class TestCheckpoint extends TestCase {
*
* @throws IOException
*/
@SuppressWarnings("deprecation")
public void testMultipleSecondaryNamenodes() throws IOException {
Configuration conf = new HdfsConfiguration();
String nameserviceId1 = "ns1";
@ -1114,7 +1100,6 @@ public class TestCheckpoint extends TestCase {
* Test that the secondary doesn't have to re-download image
* if it hasn't changed.
*/
@SuppressWarnings("deprecation")
public void testSecondaryImageDownload() throws IOException {
LOG.info("Starting testSecondaryImageDownload");
Configuration conf = new HdfsConfiguration();
@ -1197,7 +1182,6 @@ public class TestCheckpoint extends TestCase {
* It verifies that this works even though the earlier-txid checkpoint gets
* uploaded after the later-txid checkpoint.
*/
@SuppressWarnings("deprecation")
public void testMultipleSecondaryNNsAgainstSameNN() throws Exception {
Configuration conf = new HdfsConfiguration();
@ -1283,7 +1267,6 @@ public class TestCheckpoint extends TestCase {
* It verifies that one of the two gets an error that it's uploading a
* duplicate checkpoint, and the other one succeeds.
*/
@SuppressWarnings("deprecation")
public void testMultipleSecondaryNNsAgainstSameNN2() throws Exception {
Configuration conf = new HdfsConfiguration();
@ -1382,7 +1365,6 @@ public class TestCheckpoint extends TestCase {
* is running. The secondary should shut itself down if if talks to a NN
* with the wrong namespace.
*/
@SuppressWarnings("deprecation")
public void testReformatNNBetweenCheckpoints() throws IOException {
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
@ -1637,7 +1619,6 @@ public class TestCheckpoint extends TestCase {
/**
* Test that the 2NN triggers a checkpoint after the configurable interval
*/
@SuppressWarnings("deprecation")
public void testCheckpointTriggerOnTxnCount() throws Exception {
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
@ -1691,7 +1672,6 @@ public class TestCheckpoint extends TestCase {
* logs that connect the 2NN's old checkpoint to the current txid
* get archived. Then, the 2NN tries to checkpoint again.
*/
@SuppressWarnings("deprecation")
public void testSecondaryHasVeryOutOfDateImage() throws IOException {
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
@ -1729,7 +1709,6 @@ public class TestCheckpoint extends TestCase {
}
}
@SuppressWarnings("deprecation")
public void testCommandLineParsing() throws ParseException {
SecondaryNameNode.CommandLineOpts opts =
new SecondaryNameNode.CommandLineOpts();
@ -1764,7 +1743,6 @@ public class TestCheckpoint extends TestCase {
} catch (ParseException e) {}
}
@SuppressWarnings("deprecation")
private void cleanup(SecondaryNameNode snn) {
if (snn != null) {
try {
@ -1780,7 +1758,6 @@ public class TestCheckpoint extends TestCase {
* Assert that if any two files have the same name across the 2NNs
* and NN, they should have the same content too.
*/
@SuppressWarnings("deprecation")
private void assertParallelFilesInvariant(MiniDFSCluster cluster,
ImmutableList<SecondaryNameNode> secondaries) throws Exception {
List<File> allCurrentDirs = Lists.newArrayList();
@ -1792,7 +1769,6 @@ public class TestCheckpoint extends TestCase {
ImmutableSet.of("VERSION"));
}
@SuppressWarnings("deprecation")
private List<File> getCheckpointCurrentDirs(SecondaryNameNode secondary) {
List<File> ret = Lists.newArrayList();
for (URI u : secondary.getCheckpointDirs()) {
@ -1802,7 +1778,6 @@ public class TestCheckpoint extends TestCase {
return ret;
}
@SuppressWarnings("deprecation")
private CheckpointStorage spyOnSecondaryImage(SecondaryNameNode secondary1) {
CheckpointStorage spy = Mockito.spy((CheckpointStorage)secondary1.getFSImage());;
secondary1.setFSImage(spy);
@ -1812,7 +1787,6 @@ public class TestCheckpoint extends TestCase {
/**
* A utility class to perform a checkpoint in a different thread.
*/
@SuppressWarnings("deprecation")
private static class DoCheckpointThread extends Thread {
private final SecondaryNameNode snn;
private volatile Throwable thrown = null;

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java
View File

@ -103,9 +103,6 @@ public class TestNameEditsConfigs extends TestCase {
assertTrue(!fileSys.exists(name));
}
// This deprecation suppress warning does not work due to known Java bug:
// http://bugs.sun.com/view_bug.do?bug_id=6460147
@SuppressWarnings("deprecation")
SecondaryNameNode startSecondaryNameNode(Configuration conf
) throws IOException {
conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0");
@ -125,7 +122,6 @@ public class TestNameEditsConfigs extends TestCase {
* sure we are reading proper edits and image.
* @throws Exception
*/
@SuppressWarnings("deprecation")
public void testNameEditsConfigs() throws Exception {
Path file1 = new Path("TestNameEditsConfigs1");
Path file2 = new Path("TestNameEditsConfigs2");

1
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryWebUi.java
View File

@ -30,7 +30,6 @@ import org.junit.Test;
public class TestSecondaryWebUi {
@SuppressWarnings("deprecation")
@Test
public void testSecondaryWebUi() throws IOException {
Configuration conf = new Configuration();

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
View File

@ -120,7 +120,6 @@ public class TestStartup extends TestCase {
* start MiniDFScluster, create a file (to create edits) and do a checkpoint
* @throws IOException
*/
@SuppressWarnings("deprecation")
public void createCheckPoint() throws IOException {
LOG.info("--starting mini cluster");
// manage dirs parameter set to false
@ -300,7 +299,6 @@ public class TestStartup extends TestCase {
* secondary node copies fsimage and edits into correct separate directories.
* @throws IOException
*/
@SuppressWarnings("deprecation")
public void testSNNStartup() throws IOException{
//setUpConfig();
LOG.info("--starting SecondNN startup test");

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
View File

@ -153,7 +153,6 @@ public class TestStorageRestore {
* 7. run doCheckpoint
* 8. verify that all the image and edits files are the same.
*/
@SuppressWarnings("deprecation")
@Test
public void testStorageRestore() throws Exception {
int numDatanodes = 0;
@ -310,7 +309,6 @@ public class TestStorageRestore {
* then try to perform a checkpoint. The NN should not serve up the image or
* edits from the restored (empty) dir.
*/
@SuppressWarnings("deprecation")
@Test
public void testMultipleSecondaryCheckpoint() throws IOException {

20
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm
View File

@ -95,7 +95,7 @@ Hadoop MapReduce Next Generation - Cluster Setup
*--------------------------------------+--------------------------------------+
| DataNode | HADOOP_DATANODE_OPTS |
*--------------------------------------+--------------------------------------+
| Backup NameNode | HADOOP_SECONDARYNAMENODE_OPTS |
| Secondary NameNode | HADOOP_SECONDARYNAMENODE_OPTS |
*--------------------------------------+--------------------------------------+
| ResourceManager | YARN_RESOURCEMANAGER_OPTS |
*--------------------------------------+--------------------------------------+
@ -537,15 +537,15 @@ Hadoop MapReduce Next Generation - Cluster Setup
It's recommended to have them share a Unix group, for e.g. <<<hadoop>>>.
*--------------------------------------+--------------------------------------+
|| User:Group || Daemons |
*--------------------------------------+--------------------------------------+
| hdfs:hadoop | NameNode, Backup NameNode, DataNode |
*--------------------------------------+--------------------------------------+
| yarn:hadoop | ResourceManager, NodeManager |
*--------------------------------------+--------------------------------------+
| mapred:hadoop | MapReduce JobHistory Server |
*--------------------------------------+--------------------------------------+
*--------------------------------------+----------------------------------------------------------------------+
|| User:Group || Daemons |
*--------------------------------------+----------------------------------------------------------------------+
| hdfs:hadoop | NameNode, Secondary NameNode, Checkpoint Node, Backup Node, DataNode |
*--------------------------------------+----------------------------------------------------------------------+
| yarn:hadoop | ResourceManager, NodeManager |
*--------------------------------------+----------------------------------------------------------------------+
| mapred:hadoop | MapReduce JobHistory Server |
*--------------------------------------+----------------------------------------------------------------------+
* <<<Permissions for both HDFS and local fileSystem paths>>>