HDFS-4734. HDFS Tests that use ShellCommandFencer are broken on Windows. Contributed by Arpit Agarwal.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1476877 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
293fb1ec1e
commit
e17d5a9683
|
@ -326,6 +326,9 @@ Trunk (Unreleased)
|
|||
HDFS-4705. Address HDFS test failures on Windows because of invalid
|
||||
dfs.namenode.name.dir. (Ivan Mitic via suresh)
|
||||
|
||||
HDFS-4734. HDFS Tests that use ShellCommandFencer are broken on Windows.
|
||||
(Arpit Agarwal via suresh)
|
||||
|
||||
BREAKDOWN OF HDFS-347 SUBTASKS AND RELATED JIRAS
|
||||
|
||||
HDFS-4353. Encapsulate connections to peers in Peer and PeerServer classes.
|
||||
|
|
|
@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
|
|||
import org.apache.hadoop.hdfs.DFSUtil;
|
||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||
import org.apache.hadoop.test.MockitoUtil;
|
||||
import org.apache.hadoop.util.Shell;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
@ -73,6 +74,17 @@ public class TestDFSHAAdmin {
|
|||
private static String HOST_A = "1.2.3.1";
|
||||
private static String HOST_B = "1.2.3.2";
|
||||
|
||||
// Fencer shell commands that always return true and false respectively
|
||||
// on Unix.
|
||||
private static String FENCER_TRUE_COMMAND_UNIX = "shell(true)";
|
||||
private static String FENCER_FALSE_COMMAND_UNIX = "shell(false)";
|
||||
|
||||
// Fencer shell commands that always return true and false respectively
|
||||
// on Windows. Lacking POSIX 'true' and 'false' commands we use the DOS
|
||||
// commands 'rem' and 'help.exe'.
|
||||
private static String FENCER_TRUE_COMMAND_WINDOWS = "shell(rem)";
|
||||
private static String FENCER_FALSE_COMMAND_WINDOWS = "shell(help.exe /? >NUL)";
|
||||
|
||||
private HdfsConfiguration getHAConf() {
|
||||
HdfsConfiguration conf = new HdfsConfiguration();
|
||||
conf.set(DFSConfigKeys.DFS_NAMESERVICES, NSID);
|
||||
|
@ -89,6 +101,16 @@ public class TestDFSHAAdmin {
|
|||
return conf;
|
||||
}
|
||||
|
||||
public static String getFencerTrueCommand() {
|
||||
return Shell.WINDOWS ?
|
||||
FENCER_TRUE_COMMAND_WINDOWS : FENCER_TRUE_COMMAND_UNIX;
|
||||
}
|
||||
|
||||
public static String getFencerFalseCommand() {
|
||||
return Shell.WINDOWS ?
|
||||
FENCER_FALSE_COMMAND_WINDOWS : FENCER_FALSE_COMMAND_UNIX;
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setup() throws IOException {
|
||||
mockProtocol = MockitoUtil.mockProtocol(HAServiceProtocol.class);
|
||||
|
@ -173,7 +195,7 @@ public class TestDFSHAAdmin {
|
|||
// Turn on auto-HA in the config
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true);
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
|
||||
tool.setConf(conf);
|
||||
|
||||
// Should fail without the forcemanual flag
|
||||
|
@ -250,7 +272,7 @@ public class TestDFSHAAdmin {
|
|||
public void testFailoverWithFencerConfigured() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "nn1", "nn2"));
|
||||
}
|
||||
|
@ -259,7 +281,7 @@ public class TestDFSHAAdmin {
|
|||
public void testFailoverWithFencerAndNameservice() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-ns", "ns1", "-failover", "nn1", "nn2"));
|
||||
}
|
||||
|
@ -268,7 +290,7 @@ public class TestDFSHAAdmin {
|
|||
public void testFailoverWithFencerConfiguredAndForce() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
}
|
||||
|
@ -277,7 +299,7 @@ public class TestDFSHAAdmin {
|
|||
public void testFailoverWithForceActive() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forceactive"));
|
||||
}
|
||||
|
@ -286,7 +308,7 @@ public class TestDFSHAAdmin {
|
|||
public void testFailoverWithInvalidFenceArg() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
|
||||
tool.setConf(conf);
|
||||
assertEquals(-1, runTool("-failover", "nn1", "nn2", "notforcefence"));
|
||||
}
|
||||
|
@ -312,7 +334,7 @@ public class TestDFSHAAdmin {
|
|||
// Turn on auto-HA in the config
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true);
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
|
||||
tool.setConf(conf);
|
||||
|
||||
assertEquals(0, runTool("-failover", "nn1", "nn2"));
|
||||
|
@ -323,7 +345,7 @@ public class TestDFSHAAdmin {
|
|||
public void testForceFenceOptionListedBeforeArgs() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2"));
|
||||
}
|
||||
|
@ -359,23 +381,23 @@ public class TestDFSHAAdmin {
|
|||
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
// Set the default fencer to succeed
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
|
||||
// Set the NN-specific fencer to fail. Should fail to fence.
|
||||
conf.set(nnSpecificKey, "shell(false)");
|
||||
conf.set(nnSpecificKey, getFencerFalseCommand());
|
||||
tool.setConf(conf);
|
||||
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
conf.unset(nnSpecificKey);
|
||||
|
||||
// Set an NS-specific fencer to fail. Should fail.
|
||||
conf.set(nsSpecificKey, "shell(false)");
|
||||
conf.set(nsSpecificKey, getFencerFalseCommand());
|
||||
tool.setConf(conf);
|
||||
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
|
||||
// Set the NS-specific fencer to succeed. Should succeed
|
||||
conf.set(nsSpecificKey, "shell(true)");
|
||||
conf.set(nsSpecificKey, getFencerTrueCommand());
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
}
|
||||
|
|
|
@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
|
|||
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
|
||||
import org.apache.hadoop.hdfs.server.namenode.NameNode;
|
||||
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
|
||||
import org.apache.hadoop.util.Shell;
|
||||
import org.apache.log4j.Level;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
|
@ -114,7 +115,8 @@ public class TestDFSHAAdminMiniCluster {
|
|||
|
||||
@Test
|
||||
public void testTryFailoverToSafeMode() throws Exception {
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
|
||||
TestDFSHAAdmin.getFencerTrueCommand());
|
||||
tool.setConf(conf);
|
||||
|
||||
NameNodeAdapter.enterSafeMode(cluster.getNameNode(0), false);
|
||||
|
@ -136,10 +138,17 @@ public class TestDFSHAAdminMiniCluster {
|
|||
// tmp file, so we can verify that the args were substituted right
|
||||
File tmpFile = File.createTempFile("testFencer", ".txt");
|
||||
tmpFile.deleteOnExit();
|
||||
if (Shell.WINDOWS) {
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
|
||||
"shell(echo %target_nameserviceid%.%target_namenodeid% " +
|
||||
"%target_port% %dfs_ha_namenode_id% > " +
|
||||
tmpFile.getAbsolutePath() + ")");
|
||||
} else {
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
|
||||
"shell(echo -n $target_nameserviceid.$target_namenodeid " +
|
||||
"$target_port $dfs_ha_namenode_id > " +
|
||||
tmpFile.getAbsolutePath() + ")");
|
||||
}
|
||||
|
||||
// Test failover with fencer
|
||||
tool.setConf(conf);
|
||||
|
@ -156,9 +165,11 @@ public class TestDFSHAAdminMiniCluster {
|
|||
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
|
||||
// The fence script should run with the configuration from the target
|
||||
// node, rather than the configuration from the fencing node
|
||||
assertEquals("minidfs-ns.nn1 " + nn1Port + " nn1",
|
||||
Files.toString(tmpFile, Charsets.UTF_8));
|
||||
// node, rather than the configuration from the fencing node. Strip
|
||||
// out any trailing spaces and CR/LFs which may be present on Windows.
|
||||
String fenceCommandOutput =Files.toString(tmpFile, Charsets.UTF_8).
|
||||
replaceAll(" *[\r\n]+", "");
|
||||
assertEquals("minidfs-ns.nn1 " + nn1Port + " nn1", fenceCommandOutput);
|
||||
tmpFile.delete();
|
||||
|
||||
// Test failover with forceactive option
|
||||
|
@ -181,7 +192,8 @@ public class TestDFSHAAdminMiniCluster {
|
|||
assertFalse(tmpFile.exists());
|
||||
|
||||
// Test failover with force fence listed before the other arguments
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
|
||||
TestDFSHAAdmin.getFencerTrueCommand());
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2"));
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue