Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-2769. HA: When HA is enabled with a shared edits dir, that dir should be marked required. Contributed by Aaron T. Myers. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1239988 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2012-02-03 03:10:57 +00:00
parent 05ab55b705
commit d2b5531b0f
3 changed files with 41 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
View File

@ -155,3 +155,6 @@ HDFS-2859. LOCAL_ADDRESS_MATCHER.match has NPE when called from DFSUtil.getSuffi
HDFS-2861. checkpointing should verify that the dfs.http.address has been configured to a non-loopback for peer NN (todd)
HDFS-2860. TestDFSRollback#testRollback is failing. (atm)
HDFS-2769. HA: When HA is enabled with a shared edits dir, that dir should be
marked required. (atm via eli)

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -632,9 +632,19 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
public static Collection<URI> getNamespaceDirs(Configuration conf) {
return getStorageDirs(conf, DFS_NAMENODE_NAME_DIR_KEY);
}
/**
* Get all edits dirs which are required. If any shared edits dirs are
* configured, these are also included in the set of required dirs.
*
* @param conf the HDFS configuration.
* @return all required dirs.
*/
public static Collection<URI> getRequiredNamespaceEditsDirs(Configuration conf) {
return getStorageDirs(conf, DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY);
Set<URI> ret = new HashSet<URI>();
ret.addAll(getStorageDirs(conf, DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY));
ret.addAll(getSharedEditsDirs(conf));
return ret;
}
private static Collection<URI> getStorageDirs(Configuration conf,

29
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java
View File

@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Collection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -33,14 +34,38 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
import com.google.common.base.Joiner;
public class TestFailureOfSharedDir {
private static final Log LOG = LogFactory.getLog(TestFailureOfSharedDir.class);
/**
* Test that the shared edits dir is automatically added to the list of edits
* dirs that are marked required.
*/
@Test
public void testSharedDirIsAutomaticallyMarkedRequired()
throws URISyntaxException {
URI foo = new URI("file:/foo");
URI bar = new URI("file:/bar");
Configuration conf = new Configuration();
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, Joiner.on(",").join(foo, bar));
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY, foo.toString());
assertFalse(FSNamesystem.getRequiredNamespaceEditsDirs(conf).contains(
bar));
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, bar.toString());
Collection<URI> requiredEditsDirs = FSNamesystem
.getRequiredNamespaceEditsDirs(conf);
assertTrue(Joiner.on(",").join(requiredEditsDirs) + " does not contain " + bar,
requiredEditsDirs.contains(bar));
}
/**
* Test that marking the shared edits dir as being "required" causes the NN to
* fail if that dir can't be accessed.
@ -48,11 +73,9 @@ public class TestFailureOfSharedDir {
@Test
public void testFailureOfSharedDir() throws Exception {
Configuration conf = new Configuration();
// The shared edits dir will automatically be marked required.
URI sharedEditsUri = MiniDFSCluster.formatSharedEditsDir(
new File(MiniDFSCluster.getBaseDirectory()), 0, 1);
// Mark the shared edits dir required.
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY,
sharedEditsUri.toString());
MiniDFSCluster cluster = null;
try {