Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar. 

(cherry picked from commit b6942cbe9b)
(cherry picked from commit 5897095d53)
This commit is contained in:
Arpit Agarwal 2017-10-05 15:26:52 -07:00 committed by vrushali
parent 6a3929f2b2
commit bc3ca4c106
6 changed files with 101 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -244,6 +244,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT = "supergroup";
public static final String DFS_NAMENODE_ACLS_ENABLED_KEY = "dfs.namenode.acls.enabled";
public static final boolean DFS_NAMENODE_ACLS_ENABLED_DEFAULT = false;
public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
public static final String DFS_NAMENODE_XATTRS_ENABLED_KEY = "dfs.namenode.xattrs.enabled";
public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
public static final String DFS_ADMIN = "dfs.cluster.administrators";

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
View File

@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@ -1149,6 +1150,21 @@ public class NameNode extends ReconfigurableBase implements
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
fsImage.getEditLog().initJournalsForWrite();
// Abort NameNode format if reformat is disabled and if
// meta-dir already exists
if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED,
DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) {
force = false;
isInteractive = false;
for (StorageDirectory sd : fsImage.storage.dirIterable(null)) {
if (sd.hasSomeData()) {
throw new NameNodeFormatException(
"NameNode format aborted as reformat is disabled for "
+ "this cluster.");
}
}
}
if (!fsImage.confirmFormat(force, isInteractive)) {
return true; // aborted
}

37
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java Normal file
View File

@ -0,0 +1,37 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
/**
* Thrown when NameNode format fails.
*/
@InterfaceAudience.Private
public class NameNodeFormatException extends IOException {
private static final long serialVersionUID = 1L;
public NameNodeFormatException(String message, Throwable cause) {
super(message, cause);
}
public NameNodeFormatException(String message) {
super(message);
}
}

11
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -4343,4 +4343,15 @@
If no suffix is specified then milliseconds is assumed.
</description>
</property>
<property>
<name>dfs.reformat.disabled</name>
<value>false</value>
<description>
Disable reformat of NameNode. If it's value is set to "true"
and metadata directories already exist then attempt to format NameNode
will throw NameNodeFormatException.
</description>
</property>
</configuration>

2
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
View File

@ -466,7 +466,7 @@ Usage:
|:---- |:---- |
| `-backup` | Start backup node. |
| `-checkpoint` | Start checkpoint node. |
| `-format` `[-clusterid cid]` `[-force]` `[-nonInteractive]` | Formats the specified NameNode. It starts the NameNode, formats it and then shut it down. -force option formats if the name directory exists. -nonInteractive option aborts if the name directory exists, unless -force option is specified. |
| `-format` `[-clusterid cid]` | Formats the specified NameNode. It starts the NameNode, formats it and then shut it down. Will throw NameNodeFormatException if name dir already exist and if reformat is disabled for cluster. |
| `-upgrade` `[-clusterid cid]` [`-renameReserved` \<k-v pairs\>] | Namenode should be started with upgrade option after the distribution of new Hadoop version. |
| `-upgradeOnly` `[-clusterid cid]` [`-renameReserved` \<k-v pairs\>] | Upgrade the specified NameNode and then shutdown it. |
| `-rollback` | Rollback the NameNode to the previous version. This should be used after stopping the cluster and distributing the old Hadoop version. |

34
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
View File

@ -39,9 +39,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.ExitUtil.ExitException;
@ -452,4 +456,34 @@ public class TestClusterId {
File version = new File(hdfsDir, "current/VERSION");
assertFalse("Check version should not exist", version.exists());
}
/**
* Test NameNode format failure when reformat is disabled and metadata
* directories exist.
*/
@Test
public void testNNFormatFailure() throws Exception {
NameNode.initMetrics(config, NamenodeRole.NAMENODE);
DFSTestUtil.formatNameNode(config);
config.setBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, true);
// Call to NameNode format will fail as name dir is not empty
try {
NameNode.format(config);
fail("NN format should fail.");
} catch (NameNodeFormatException e) {
GenericTestUtils.assertExceptionContains("NameNode format aborted as "
+ "reformat is disabled for this cluster", e);
}
}
/**
* Test NameNode format when reformat is disabled and metadata directories do
* not exist.
*/
@Test
public void testNNFormatSuccess() throws Exception {
NameNode.initMetrics(config, NamenodeRole.NAMENODE);
config.setBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, true);
DFSTestUtil.formatNameNode(config);
}
}