HADOOP-8077. HA: fencing method should be able to be configured on a per-NN or per-NS basis. Contributed by Todd Lipcon.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1310172 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
37c3d51954
commit
65cca13ec4
|
@ -136,6 +136,9 @@ Release 2.0.0 - UNRELEASED
|
|||
|
||||
HADOOP-8007. Use substitution tokens for fencing argument (todd)
|
||||
|
||||
HADOOP-8077. HA: fencing method should be able to be configured on
|
||||
a per-NN or per-NS basis (todd)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
|
|
@ -53,9 +53,6 @@ import com.google.common.collect.Lists;
|
|||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Evolving
|
||||
public class NodeFencer {
|
||||
public static final String CONF_METHODS_KEY =
|
||||
"dfs.ha.fencing.methods";
|
||||
|
||||
private static final String CLASS_RE = "([a-zA-Z0-9\\.\\$]+)";
|
||||
private static final Pattern CLASS_WITH_ARGUMENT =
|
||||
Pattern.compile(CLASS_RE + "\\((.+?)\\)");
|
||||
|
@ -76,18 +73,18 @@ public class NodeFencer {
|
|||
|
||||
private final List<FenceMethodWithArg> methods;
|
||||
|
||||
public NodeFencer(Configuration conf)
|
||||
NodeFencer(Configuration conf, String spec)
|
||||
throws BadFencingConfigurationException {
|
||||
this.methods = parseMethods(conf);
|
||||
this.methods = parseMethods(conf, spec);
|
||||
}
|
||||
|
||||
public static NodeFencer create(Configuration conf)
|
||||
public static NodeFencer create(Configuration conf, String confKey)
|
||||
throws BadFencingConfigurationException {
|
||||
String confStr = conf.get(CONF_METHODS_KEY);
|
||||
String confStr = conf.get(confKey);
|
||||
if (confStr == null) {
|
||||
return null;
|
||||
}
|
||||
return new NodeFencer(conf);
|
||||
return new NodeFencer(conf, confStr);
|
||||
}
|
||||
|
||||
public boolean fence(HAServiceTarget fromSvc) {
|
||||
|
@ -115,10 +112,10 @@ public class NodeFencer {
|
|||
return false;
|
||||
}
|
||||
|
||||
private static List<FenceMethodWithArg> parseMethods(Configuration conf)
|
||||
private static List<FenceMethodWithArg> parseMethods(Configuration conf,
|
||||
String spec)
|
||||
throws BadFencingConfigurationException {
|
||||
String confStr = conf.get(CONF_METHODS_KEY);
|
||||
String[] lines = confStr.split("\\s*\n\\s*");
|
||||
String[] lines = spec.split("\\s*\n\\s*");
|
||||
|
||||
List<FenceMethodWithArg> methods = Lists.newArrayList();
|
||||
for (String line : lines) {
|
||||
|
|
|
@ -132,8 +132,7 @@ public class TestNodeFencer {
|
|||
throws BadFencingConfigurationException {
|
||||
System.err.println("Testing configuration:\n" + confStr);
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, confStr);
|
||||
return new NodeFencer(conf);
|
||||
return new NodeFencer(conf, confStr);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -71,8 +71,7 @@ public class TestShellCommandFencer {
|
|||
public void testCheckNoArgs() {
|
||||
try {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, "shell");
|
||||
new NodeFencer(conf);
|
||||
new NodeFencer(conf, "shell");
|
||||
fail("Didn't throw when passing no args to shell");
|
||||
} catch (BadFencingConfigurationException confe) {
|
||||
assertTrue(
|
||||
|
@ -85,8 +84,7 @@ public class TestShellCommandFencer {
|
|||
public void testCheckParensNoArgs() {
|
||||
try {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, "shell()");
|
||||
new NodeFencer(conf);
|
||||
new NodeFencer(conf, "shell()");
|
||||
fail("Didn't throw when passing no args to shell");
|
||||
} catch (BadFencingConfigurationException confe) {
|
||||
assertTrue(
|
||||
|
|
|
@ -343,4 +343,5 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
|
|||
public static final int DFS_HA_LOGROLL_PERIOD_DEFAULT = 2 * 60; // 2m
|
||||
public static final String DFS_HA_TAILEDITS_PERIOD_KEY = "dfs.ha.tail-edits.period";
|
||||
public static final int DFS_HA_TAILEDITS_PERIOD_DEFAULT = 60; // 1m
|
||||
public static final String DFS_HA_FENCE_METHODS_KEY = "dfs.ha.fencing.methods";
|
||||
}
|
||||
|
|
|
@ -40,6 +40,8 @@ import org.apache.hadoop.fs.FileSystem;
|
|||
import org.apache.hadoop.fs.FileUtil;
|
||||
import org.apache.hadoop.fs.Trash;
|
||||
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
|
||||
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.hdfs.DFSUtil;
|
||||
import org.apache.hadoop.hdfs.HAUtil;
|
||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||
|
@ -160,7 +162,8 @@ public class NameNode {
|
|||
DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY,
|
||||
DFS_NAMENODE_BACKUP_ADDRESS_KEY,
|
||||
DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY,
|
||||
DFS_NAMENODE_BACKUP_SERVICE_RPC_ADDRESS_KEY
|
||||
DFS_NAMENODE_BACKUP_SERVICE_RPC_ADDRESS_KEY,
|
||||
DFS_HA_FENCE_METHODS_KEY
|
||||
};
|
||||
|
||||
public long getProtocolVersion(String protocol,
|
||||
|
|
|
@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
|||
import org.apache.hadoop.ha.BadFencingConfigurationException;
|
||||
import org.apache.hadoop.ha.HAServiceTarget;
|
||||
import org.apache.hadoop.ha.NodeFencer;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.hdfs.DFSUtil;
|
||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||
import org.apache.hadoop.hdfs.server.namenode.NameNode;
|
||||
|
@ -75,7 +76,8 @@ public class NNHAServiceTarget extends HAServiceTarget {
|
|||
this.addr = NetUtils.createSocketAddr(serviceAddr,
|
||||
NameNode.DEFAULT_PORT);
|
||||
try {
|
||||
this.fencer = NodeFencer.create(targetConf);
|
||||
this.fencer = NodeFencer.create(targetConf,
|
||||
DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY);
|
||||
} catch (BadFencingConfigurationException e) {
|
||||
this.fenceConfigError = e;
|
||||
}
|
||||
|
|
|
@ -158,7 +158,7 @@ public class TestDFSHAAdmin {
|
|||
public void testFailoverWithFencerConfigured() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "nn1", "nn2"));
|
||||
}
|
||||
|
@ -167,7 +167,7 @@ public class TestDFSHAAdmin {
|
|||
public void testFailoverWithFencerAndNameservice() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-ns", "ns1", "-failover", "nn1", "nn2"));
|
||||
}
|
||||
|
@ -176,7 +176,7 @@ public class TestDFSHAAdmin {
|
|||
public void testFailoverWithFencerConfiguredAndForce() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
}
|
||||
|
@ -185,7 +185,7 @@ public class TestDFSHAAdmin {
|
|||
public void testFailoverWithForceActive() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forceactive"));
|
||||
}
|
||||
|
@ -194,7 +194,7 @@ public class TestDFSHAAdmin {
|
|||
public void testFailoverWithInvalidFenceArg() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
tool.setConf(conf);
|
||||
assertEquals(-1, runTool("-failover", "nn1", "nn2", "notforcefence"));
|
||||
}
|
||||
|
@ -209,7 +209,7 @@ public class TestDFSHAAdmin {
|
|||
public void testFailoverWithFenceAndBadFencer() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, "foobar!");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "foobar!");
|
||||
tool.setConf(conf);
|
||||
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
}
|
||||
|
@ -218,7 +218,7 @@ public class TestDFSHAAdmin {
|
|||
public void testForceFenceOptionListedBeforeArgs() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2"));
|
||||
}
|
||||
|
@ -240,7 +240,41 @@ public class TestDFSHAAdmin {
|
|||
assertEquals(-1, runTool("-checkHealth", "nn1"));
|
||||
assertOutputContains("Health check failed: fake health check failure");
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that the fencing configuration can be overridden per-nameservice
|
||||
* or per-namenode
|
||||
*/
|
||||
@Test
|
||||
public void testFencingConfigPerNameNode() throws Exception {
|
||||
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
|
||||
|
||||
final String nsSpecificKey = DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY + "." + NSID;
|
||||
final String nnSpecificKey = nsSpecificKey + ".nn1";
|
||||
|
||||
HdfsConfiguration conf = getHAConf();
|
||||
// Set the default fencer to succeed
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
|
||||
// Set the NN-specific fencer to fail. Should fail to fence.
|
||||
conf.set(nnSpecificKey, "shell(false)");
|
||||
tool.setConf(conf);
|
||||
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
conf.unset(nnSpecificKey);
|
||||
|
||||
// Set an NS-specific fencer to fail. Should fail.
|
||||
conf.set(nsSpecificKey, "shell(false)");
|
||||
tool.setConf(conf);
|
||||
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
|
||||
// Set the NS-specific fencer to succeed. Should succeed
|
||||
conf.set(nsSpecificKey, "shell(true)");
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
}
|
||||
|
||||
private Object runTool(String ... args) throws Exception {
|
||||
errOutBytes.reset();
|
||||
LOG.info("Running: DFSHAAdmin " + Joiner.on(" ").join(args));
|
||||
|
|
|
@ -28,6 +28,7 @@ import org.apache.commons.logging.LogFactory;
|
|||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.impl.Log4JLogger;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
|
||||
import org.apache.hadoop.hdfs.server.namenode.NameNode;
|
||||
|
@ -113,7 +114,7 @@ public class TestDFSHAAdminMiniCluster {
|
|||
|
||||
@Test
|
||||
public void testTryFailoverToSafeMode() throws Exception {
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
tool.setConf(conf);
|
||||
|
||||
NameNodeAdapter.enterSafeMode(cluster.getNameNode(0), false);
|
||||
|
@ -135,7 +136,7 @@ public class TestDFSHAAdminMiniCluster {
|
|||
// tmp file, so we can verify that the args were substituted right
|
||||
File tmpFile = File.createTempFile("testFencer", ".txt");
|
||||
tmpFile.deleteOnExit();
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY,
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
|
||||
"shell(echo -n $target_nameserviceid.$target_namenodeid " +
|
||||
"$target_port $dfs_ha_namenode_id > " +
|
||||
tmpFile.getAbsolutePath() + ")");
|
||||
|
@ -168,19 +169,19 @@ public class TestDFSHAAdminMiniCluster {
|
|||
|
||||
|
||||
// Test failover with not fencer and forcefence option
|
||||
conf.unset(NodeFencer.CONF_METHODS_KEY);
|
||||
conf.unset(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY);
|
||||
tool.setConf(conf);
|
||||
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
assertFalse(tmpFile.exists());
|
||||
|
||||
// Test failover with bad fencer and forcefence option
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, "foobar!");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "foobar!");
|
||||
tool.setConf(conf);
|
||||
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
|
||||
assertFalse(tmpFile.exists());
|
||||
|
||||
// Test failover with force fence listed before the other arguments
|
||||
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
|
||||
conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
|
||||
tool.setConf(conf);
|
||||
assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2"));
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue