Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-4734. HDFS Tests that use ShellCommandFencer are broken on Windows. Contributed by Arpit Agarwal. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1476877 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Suresh Srinivas 2013-04-29 02:07:58 +00:00
parent 293fb1ec1e
commit e17d5a9683
3 changed files with 58 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -326,6 +326,9 @@ Trunk (Unreleased)
HDFS-4705. Address HDFS test failures on Windows because of invalid
dfs.namenode.name.dir. (Ivan Mitic via suresh)
HDFS-4734. HDFS Tests that use ShellCommandFencer are broken on Windows.
(Arpit Agarwal via suresh)
BREAKDOWN OF HDFS-347 SUBTASKS AND RELATED JIRAS
HDFS-4353. Encapsulate connections to peers in Peer and PeerServer classes.

46
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
View File

@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.test.MockitoUtil;
import org.apache.hadoop.util.Shell;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
@ -73,6 +74,17 @@ public class TestDFSHAAdmin {
private static String HOST_A = "1.2.3.1";
private static String HOST_B = "1.2.3.2";
// Fencer shell commands that always return true and false respectively
// on Unix.
private static String FENCER_TRUE_COMMAND_UNIX = "shell(true)";
private static String FENCER_FALSE_COMMAND_UNIX = "shell(false)";
// Fencer shell commands that always return true and false respectively
// on Windows. Lacking POSIX 'true' and 'false' commands we use the DOS
// commands 'rem' and 'help.exe'.
private static String FENCER_TRUE_COMMAND_WINDOWS = "shell(rem)";
private static String FENCER_FALSE_COMMAND_WINDOWS = "shell(help.exe /? >NUL)";
private HdfsConfiguration getHAConf() {
HdfsConfiguration conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_NAMESERVICES, NSID);
@ -89,6 +101,16 @@ public class TestDFSHAAdmin {
return conf;
}
public static String getFencerTrueCommand() {
return Shell.WINDOWS ?
FENCER_TRUE_COMMAND_WINDOWS : FENCER_TRUE_COMMAND_UNIX;
}
public static String getFencerFalseCommand() {
return Shell.WINDOWS ?
FENCER_FALSE_COMMAND_WINDOWS : FENCER_FALSE_COMMAND_UNIX;
}
@Before
public void setup() throws IOException {
mockProtocol = MockitoUtil.mockProtocol(HAServiceProtocol.class);
@ -173,7 +195,7 @@ public class TestDFSHAAdmin {
// Turn on auto-HA in the config
HdfsConfiguration conf = getHAConf();
conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true);
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
// Should fail without the forcemanual flag
@ -250,7 +272,7 @@ public class TestDFSHAAdmin {
public void testFailoverWithFencerConfigured() throws Exception {
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "nn1", "nn2"));
}
@ -259,7 +281,7 @@ public class TestDFSHAAdmin {
public void testFailoverWithFencerAndNameservice() throws Exception {
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-ns", "ns1", "-failover", "nn1", "nn2"));
}
@ -268,7 +290,7 @@ public class TestDFSHAAdmin {
public void testFailoverWithFencerConfiguredAndForce() throws Exception {
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
}
@ -277,7 +299,7 @@ public class TestDFSHAAdmin {
public void testFailoverWithForceActive() throws Exception {
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forceactive"));
}
@ -286,7 +308,7 @@ public class TestDFSHAAdmin {
public void testFailoverWithInvalidFenceArg() throws Exception {
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(-1, runTool("-failover", "nn1", "nn2", "notforcefence"));
}
@ -312,7 +334,7 @@ public class TestDFSHAAdmin {
// Turn on auto-HA in the config
HdfsConfiguration conf = getHAConf();
conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true);
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "nn1", "nn2"));
@ -323,7 +345,7 @@ public class TestDFSHAAdmin {
public void testForceFenceOptionListedBeforeArgs() throws Exception {
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2"));
}
@ -359,23 +381,23 @@ public class TestDFSHAAdmin {
HdfsConfiguration conf = getHAConf();
// Set the default fencer to succeed
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
// Set the NN-specific fencer to fail. Should fail to fence.
conf.set(nnSpecificKey, "shell(false)");
conf.set(nnSpecificKey, getFencerFalseCommand());
tool.setConf(conf);
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
conf.unset(nnSpecificKey);
// Set an NS-specific fencer to fail. Should fail.
conf.set(nsSpecificKey, "shell(false)");
conf.set(nsSpecificKey, getFencerFalseCommand());
tool.setConf(conf);
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
// Set the NS-specific fencer to succeed. Should succeed
conf.set(nsSpecificKey, "shell(true)");
conf.set(nsSpecificKey, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
}

30
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
View File

@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.util.Shell;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Before;
@ -114,7 +115,8 @@ public class TestDFSHAAdminMiniCluster {
@Test
public void testTryFailoverToSafeMode() throws Exception {
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
TestDFSHAAdmin.getFencerTrueCommand());
tool.setConf(conf);
NameNodeAdapter.enterSafeMode(cluster.getNameNode(0), false);
@ -136,10 +138,17 @@ public class TestDFSHAAdminMiniCluster {
// tmp file, so we can verify that the args were substituted right
File tmpFile = File.createTempFile("testFencer", ".txt");
tmpFile.deleteOnExit();
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
"shell(echo -n $target_nameserviceid.$target_namenodeid " +
"$target_port $dfs_ha_namenode_id > " +
tmpFile.getAbsolutePath() + ")");
if (Shell.WINDOWS) {
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
"shell(echo %target_nameserviceid%.%target_namenodeid% " +
"%target_port% %dfs_ha_namenode_id% > " +
tmpFile.getAbsolutePath() + ")");
} else {
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
"shell(echo -n $target_nameserviceid.$target_namenodeid " +
"$target_port $dfs_ha_namenode_id > " +
tmpFile.getAbsolutePath() + ")");
}
// Test failover with fencer
tool.setConf(conf);
@ -156,9 +165,11 @@ public class TestDFSHAAdminMiniCluster {
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
// The fence script should run with the configuration from the target
// node, rather than the configuration from the fencing node
assertEquals("minidfs-ns.nn1 " + nn1Port + " nn1",
Files.toString(tmpFile, Charsets.UTF_8));
// node, rather than the configuration from the fencing node. Strip
// out any trailing spaces and CR/LFs which may be present on Windows.
String fenceCommandOutput =Files.toString(tmpFile, Charsets.UTF_8).
replaceAll(" *[\r\n]+", "");
assertEquals("minidfs-ns.nn1 " + nn1Port + " nn1", fenceCommandOutput);
tmpFile.delete();
// Test failover with forceactive option
@ -181,7 +192,8 @@ public class TestDFSHAAdminMiniCluster {
assertFalse(tmpFile.exists());
// Test failover with force fence listed before the other arguments
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
TestDFSHAAdmin.getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2"));
}