Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-8193. Refactor FailoverController/HAAdmin code to add an abstract class for "target" services. Contributed by Todd Lipcon. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1304967 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Todd Lipcon 2012-03-24 23:48:07 +00:00
parent 4cb809bf04
commit 90ea037d21
18 changed files with 524 additions and 348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -220,6 +220,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8163. Improve ActiveStandbyElector to provide hooks for
fencing old active. (todd)
HADOOP-8193. Refactor FailoverController/HAAdmin code to add an abstract
class for "target" services. (todd)
OPTIMIZATIONS
BUG FIXES

7
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/BadFencingConfigurationException.java
View File

@ -19,11 +19,16 @@ package org.apache.hadoop.ha;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* Indicates that the operator has specified an invalid configuration
* for fencing methods.
*/
class BadFencingConfigurationException extends IOException {
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class BadFencingConfigurationException extends IOException {
private static final long serialVersionUID = 1L;
public BadFencingConfigurationException(String msg) {

50
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.ha;
import java.io.IOException;
import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -51,21 +50,21 @@ public class FailoverController {
* allow it to become active, eg because it triggers a log roll
* so the standby can learn about new blocks and leave safemode.
*
* @param toSvc service to make active
* @param toSvcName name of service to make active
* @param target service to make active
* @param forceActive ignore toSvc if it reports that it is not ready
* @throws FailoverFailedException if we should avoid failover
*/
private static void preFailoverChecks(HAServiceProtocol toSvc,
InetSocketAddress toSvcAddr,
private static void preFailoverChecks(HAServiceTarget target,
boolean forceActive)
throws FailoverFailedException {
HAServiceStatus toSvcStatus;
HAServiceProtocol toSvc;
try {
toSvc = target.getProxy();
toSvcStatus = toSvc.getServiceStatus();
} catch (IOException e) {
String msg = "Unable to get service state for " + toSvcAddr;
String msg = "Unable to get service state for " + target;
LOG.error(msg, e);
throw new FailoverFailedException(msg, e);
}
@ -79,7 +78,7 @@ public class FailoverController {
String notReadyReason = toSvcStatus.getNotReadyReason();
if (!forceActive) {
throw new FailoverFailedException(
toSvcAddr + " is not ready to become active: " +
target + " is not ready to become active: " +
notReadyReason);
} else {
LOG.warn("Service is not ready to become active, but forcing: " +
@ -103,44 +102,39 @@ public class FailoverController {
* then try to failback.
*
* @param fromSvc currently active service
* @param fromSvcAddr addr of the currently active service
* @param toSvc service to make active
* @param toSvcAddr addr of the service to make active
* @param fencer for fencing fromSvc
* @param forceFence to fence fromSvc even if not strictly necessary
* @param forceActive try to make toSvc active even if it is not ready
* @throws FailoverFailedException if the failover fails
*/
public static void failover(HAServiceProtocol fromSvc,
InetSocketAddress fromSvcAddr,
HAServiceProtocol toSvc,
InetSocketAddress toSvcAddr,
NodeFencer fencer,
public static void failover(HAServiceTarget fromSvc,
HAServiceTarget toSvc,
boolean forceFence,
boolean forceActive)
throws FailoverFailedException {
Preconditions.checkArgument(fencer != null, "failover requires a fencer");
preFailoverChecks(toSvc, toSvcAddr, forceActive);
Preconditions.checkArgument(fromSvc.getFencer() != null,
"failover requires a fencer");
preFailoverChecks(toSvc, forceActive);
// Try to make fromSvc standby
boolean tryFence = true;
try {
HAServiceProtocolHelper.transitionToStandby(fromSvc);
HAServiceProtocolHelper.transitionToStandby(fromSvc.getProxy());
// We should try to fence if we failed or it was forced
tryFence = forceFence ? true : false;
} catch (ServiceFailedException sfe) {
LOG.warn("Unable to make " + fromSvcAddr + " standby (" +
LOG.warn("Unable to make " + fromSvc + " standby (" +
sfe.getMessage() + ")");
} catch (IOException ioe) {
LOG.warn("Unable to make " + fromSvcAddr +
LOG.warn("Unable to make " + fromSvc +
" standby (unable to connect)", ioe);
}
// Fence fromSvc if it's required or forced by the user
if (tryFence) {
if (!fencer.fence(fromSvcAddr)) {
if (!fromSvc.getFencer().fence(fromSvc)) {
throw new FailoverFailedException("Unable to fence " +
fromSvcAddr + ". Fencing failed.");
fromSvc + ". Fencing failed.");
}
}
@ -148,14 +142,14 @@ public class FailoverController {
boolean failed = false;
Throwable cause = null;
try {
HAServiceProtocolHelper.transitionToActive(toSvc);
HAServiceProtocolHelper.transitionToActive(toSvc.getProxy());
} catch (ServiceFailedException sfe) {
LOG.error("Unable to make " + toSvcAddr + " active (" +
LOG.error("Unable to make " + toSvc + " active (" +
sfe.getMessage() + "). Failing back.");
failed = true;
cause = sfe;
} catch (IOException ioe) {
LOG.error("Unable to make " + toSvcAddr +
LOG.error("Unable to make " + toSvc +
" active (unable to connect). Failing back.", ioe);
failed = true;
cause = ioe;
@ -163,7 +157,7 @@ public class FailoverController {
// We failed to make toSvc active
if (failed) {
String msg = "Unable to failover to " + toSvcAddr;
String msg = "Unable to failover to " + toSvc;
// Only try to failback if we didn't fence fromSvc
if (!tryFence) {
try {
@ -171,9 +165,9 @@ public class FailoverController {
// become active, eg we timed out waiting for its response.
// Unconditionally force fromSvc to become active since it
// was previously active when we initiated failover.
failover(toSvc, toSvcAddr, fromSvc, fromSvcAddr, fencer, true, true);
failover(toSvc, fromSvc, true, true);
} catch (FailoverFailedException ffe) {
msg += ". Failback to " + fromSvcAddr +
msg += ". Failback to " + fromSvc +
" failed (" + ffe.getMessage() + ")";
LOG.fatal(msg);
}

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FenceMethod.java
View File

@ -17,8 +17,6 @@
*/
package org.apache.hadoop.ha;
import java.net.InetSocketAddress;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configurable;
@ -62,6 +60,6 @@ public interface FenceMethod {
* @throws BadFencingConfigurationException if the configuration was
* determined to be invalid only at runtime
*/
public boolean tryFence(InetSocketAddress serviceAddr, String args)
public boolean tryFence(HAServiceTarget target, String args)
throws BadFencingConfigurationException;
}

50
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
View File

@ -19,7 +19,6 @@ package org.apache.hadoop.ha;
import java.io.IOException;
import java.io.PrintStream;
import java.net.InetSocketAddress;
import java.util.Map;
import org.apache.commons.cli.Options;
@ -28,11 +27,8 @@ import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.ha.protocolPB.HAServiceProtocolClientSideTranslatorPB;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@ -77,6 +73,8 @@ public abstract class HAAdmin extends Configured implements Tool {
protected PrintStream errOut = System.err;
PrintStream out = System.out;
protected abstract HAServiceTarget resolveTarget(String string);
protected String getUsageString() {
return "Usage: HAAdmin";
}
@ -109,7 +107,7 @@ public abstract class HAAdmin extends Configured implements Tool {
return -1;
}
HAServiceProtocol proto = getProtocol(argv[1]);
HAServiceProtocol proto = resolveTarget(argv[1]).getProxy();
HAServiceProtocolHelper.transitionToActive(proto);
return 0;
}
@ -122,14 +120,13 @@ public abstract class HAAdmin extends Configured implements Tool {
return -1;
}
HAServiceProtocol proto = getProtocol(argv[1]);
HAServiceProtocol proto = resolveTarget(argv[1]).getProxy();
HAServiceProtocolHelper.transitionToStandby(proto);
return 0;
}
private int failover(final String[] argv)
throws IOException, ServiceFailedException {
Configuration conf = getConf();
boolean forceFence = false;
boolean forceActive = false;
@ -162,29 +159,12 @@ public abstract class HAAdmin extends Configured implements Tool {
return -1;
}
NodeFencer fencer;
try {
fencer = NodeFencer.create(conf);
} catch (BadFencingConfigurationException bfce) {
errOut.println("failover: incorrect fencing configuration: " +
bfce.getLocalizedMessage());
return -1;
}
if (fencer == null) {
errOut.println("failover: no fencer configured");
return -1;
}
InetSocketAddress addr1 =
NetUtils.createSocketAddr(getServiceAddr(args[0]));
InetSocketAddress addr2 =
NetUtils.createSocketAddr(getServiceAddr(args[1]));
HAServiceProtocol proto1 = getProtocol(args[0]);
HAServiceProtocol proto2 = getProtocol(args[1]);
HAServiceTarget fromNode = resolveTarget(args[0]);
HAServiceTarget toNode = resolveTarget(args[1]);
try {
FailoverController.failover(proto1, addr1, proto2, addr2,
fencer, forceFence, forceActive);
FailoverController.failover(fromNode, toNode,
forceFence, forceActive);
out.println("Failover from "+args[0]+" to "+args[1]+" successful");
} catch (FailoverFailedException ffe) {
errOut.println("Failover failed: " + ffe.getLocalizedMessage());
@ -201,7 +181,7 @@ public abstract class HAAdmin extends Configured implements Tool {
return -1;
}
HAServiceProtocol proto = getProtocol(argv[1]);
HAServiceProtocol proto = resolveTarget(argv[1]).getProxy();
try {
HAServiceProtocolHelper.monitorHealth(proto);
} catch (HealthCheckFailedException e) {
@ -219,7 +199,7 @@ public abstract class HAAdmin extends Configured implements Tool {
return -1;
}
HAServiceProtocol proto = getProtocol(argv[1]);
HAServiceProtocol proto = resolveTarget(argv[1]).getProxy();
out.println(proto.getServiceStatus().getState());
return 0;
}
@ -232,16 +212,6 @@ public abstract class HAAdmin extends Configured implements Tool {
return serviceId;
}
/**
* Return a proxy to the specified target service.
*/
protected HAServiceProtocol getProtocol(String serviceId)
throws IOException {
String serviceAddr = getServiceAddr(serviceId);
InetSocketAddress addr = NetUtils.createSocketAddr(serviceAddr);
return new HAServiceProtocolClientSideTranslatorPB(addr, getConf());
}
@Override
public int run(String[] argv) throws Exception {
try {

74
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java Normal file
View File

@ -0,0 +1,74 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ha;
import java.io.IOException;
import java.net.InetSocketAddress;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ha.protocolPB.HAServiceProtocolClientSideTranslatorPB;
/**
* Represents a target of the client side HA administration commands.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public abstract class HAServiceTarget {
/**
* @return the IPC address of the target node.
*/
public abstract InetSocketAddress getAddress();
/**
* @return a Fencer implementation configured for this target node
*/
public abstract NodeFencer getFencer();
/**
* @throws BadFencingConfigurationException if the fencing configuration
* appears to be invalid. This is divorced from the above
* {@link #getFencer()} method so that the configuration can be checked
* during the pre-flight phase of failover.
*/
public abstract void checkFencingConfigured()
throws BadFencingConfigurationException;
/**
* @return a proxy to connect to the target HA Service.
*/
public HAServiceProtocol getProxy(Configuration conf, int timeoutMs)
throws IOException {
Configuration confCopy = new Configuration(conf);
// Lower the timeout so we quickly fail to connect
confCopy.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1);
return new HAServiceProtocolClientSideTranslatorPB(
getAddress(),
confCopy, null, timeoutMs);
}
/**
* @return a proxy to connect to the target HA Service.
*/
public final HAServiceProtocol getProxy() throws IOException {
return getProxy(new Configuration(), 0); // default conf, timeout
}
}

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
View File

@ -17,7 +17,6 @@
*/
package org.apache.hadoop.ha;
import java.net.InetSocketAddress;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
@ -91,14 +90,14 @@ public class NodeFencer {
return new NodeFencer(conf);
}
public boolean fence(InetSocketAddress serviceAddr) {
public boolean fence(HAServiceTarget fromSvc) {
LOG.info("====== Beginning Service Fencing Process... ======");
int i = 0;
for (FenceMethodWithArg method : methods) {
LOG.info("Trying method " + (++i) + "/" + methods.size() +": " + method);
try {
if (method.method.tryFence(serviceAddr, method.arg)) {
if (method.method.tryFence(fromSvc, method.arg)) {
LOG.info("====== Fencing successful by method " + method + " ======");
return true;
}

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java
View File

@ -75,7 +75,8 @@ public class ShellCommandFencer
}
@Override
public boolean tryFence(InetSocketAddress serviceAddr, String cmd) {
public boolean tryFence(HAServiceTarget target, String cmd) {
InetSocketAddress serviceAddr = target.getAddress();
List<String> cmdList = Arrays.asList(cmd.split("\\s+"));
// Create arg list with service as the first argument

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
View File

@ -79,10 +79,11 @@ public class SshFenceByTcpPort extends Configured
}
@Override
public boolean tryFence(InetSocketAddress serviceAddr, String argsStr)
public boolean tryFence(HAServiceTarget target, String argsStr)
throws BadFencingConfigurationException {
Args args = new Args(argsStr);
InetSocketAddress serviceAddr = target.getAddress();
String host = serviceAddr.getHostName();
Session session;

94
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java Normal file
View File

@ -0,0 +1,94 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ha;
import java.io.IOException;
import java.net.InetSocketAddress;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.security.AccessControlException;
import org.mockito.Mockito;
/**
* Test-only implementation of {@link HAServiceTarget}, which returns
* a mock implementation.
*/
class DummyHAService extends HAServiceTarget {
HAServiceState state;
HAServiceProtocol proxy;
NodeFencer fencer;
InetSocketAddress address;
DummyHAService(HAServiceState state, InetSocketAddress address) {
this.state = state;
this.proxy = makeMock();
this.fencer = Mockito.mock(NodeFencer.class);
this.address = address;
}
private HAServiceProtocol makeMock() {
return Mockito.spy(new HAServiceProtocol() {
@Override
public void monitorHealth() throws HealthCheckFailedException,
AccessControlException, IOException {
}
@Override
public void transitionToActive() throws ServiceFailedException,
AccessControlException, IOException {
state = HAServiceState.ACTIVE;
}
@Override
public void transitionToStandby() throws ServiceFailedException,
AccessControlException, IOException {
state = HAServiceState.STANDBY;
}
@Override
public HAServiceStatus getServiceStatus() throws IOException {
HAServiceStatus ret = new HAServiceStatus(state);
if (state == HAServiceState.STANDBY) {
ret.setReadyToBecomeActive();
}
return ret;
}
});
}
@Override
public InetSocketAddress getAddress() {
return address;
}
@Override
public HAServiceProtocol getProxy(Configuration conf, int timeout)
throws IOException {
return proxy;
}
@Override
public NodeFencer getFencer() {
return fencer;
}
@Override
public void checkFencingConfigured() throws BadFencingConfigurationException {
}
}

356
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestFailoverController.java
View File

@ -24,124 +24,85 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.protocolPB.HAServiceProtocolClientSideTranslatorPB;
import org.apache.hadoop.ha.TestNodeFencer.AlwaysSucceedFencer;
import org.apache.hadoop.ha.TestNodeFencer.AlwaysFailFencer;
import static org.apache.hadoop.ha.TestNodeFencer.setupFencer;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.internal.stubbing.answers.ThrowsException;
import org.mockito.stubbing.Answer;
import static org.junit.Assert.*;
public class TestFailoverController {
private InetSocketAddress svc1Addr = new InetSocketAddress("svc1", 1234);
private InetSocketAddress svc2Addr = new InetSocketAddress("svc2", 5678);
private class DummyService implements HAServiceProtocol {
HAServiceState state;
DummyService(HAServiceState state) {
this.state = state;
}
@Override
public void monitorHealth() throws HealthCheckFailedException, IOException {
// Do nothing
}
@Override
public void transitionToActive() throws ServiceFailedException, IOException {
state = HAServiceState.ACTIVE;
}
@Override
public void transitionToStandby() throws ServiceFailedException, IOException {
state = HAServiceState.STANDBY;
}
@Override
public HAServiceStatus getServiceStatus() throws IOException {
HAServiceStatus ret = new HAServiceStatus(state);
if (state == HAServiceState.STANDBY) {
ret.setReadyToBecomeActive();
}
return ret;
}
private HAServiceState getServiceState() {
return state;
}
}
HAServiceStatus STATE_NOT_READY = new HAServiceStatus(HAServiceState.STANDBY)
.setNotReadyToBecomeActive("injected not ready");
@Test
public void testFailoverAndFailback() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE);
DummyService svc2 = new DummyService(HAServiceState.STANDBY);
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
AlwaysSucceedFencer.fenceCalled = 0;
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled);
assertEquals(HAServiceState.STANDBY, svc1.getServiceState());
assertEquals(HAServiceState.ACTIVE, svc2.getServiceState());
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(HAServiceState.ACTIVE, svc2.state);
AlwaysSucceedFencer.fenceCalled = 0;
FailoverController.failover(svc2, svc2Addr, svc1, svc1Addr, fencer, false, false);
FailoverController.failover(svc2, svc1, false, false);
assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled);
assertEquals(HAServiceState.ACTIVE, svc1.getServiceState());
assertEquals(HAServiceState.STANDBY, svc2.getServiceState());
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testFailoverFromStandbyToStandby() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.STANDBY);
DummyService svc2 = new DummyService(HAServiceState.STANDBY);
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.STANDBY, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
assertEquals(HAServiceState.STANDBY, svc1.getServiceState());
assertEquals(HAServiceState.ACTIVE, svc2.getServiceState());
FailoverController.failover(svc1, svc2, false, false);
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(HAServiceState.ACTIVE, svc2.state);
}
@Test
public void testFailoverFromActiveToActive() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE);
DummyService svc2 = new DummyService(HAServiceState.ACTIVE);
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.ACTIVE, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
fail("Can't failover to an already active service");
} catch (FailoverFailedException ffe) {
// Expected
}
assertEquals(HAServiceState.ACTIVE, svc1.getServiceState());
assertEquals(HAServiceState.ACTIVE, svc2.getServiceState());
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.ACTIVE, svc2.state);
}
@Test
public void testFailoverWithoutPermission() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE) {
@Override
public HAServiceStatus getServiceStatus() throws IOException {
throw new AccessControlException("Access denied");
}
};
DummyService svc2 = new DummyService(HAServiceState.STANDBY) {
@Override
public HAServiceStatus getServiceStatus() throws IOException {
throw new AccessControlException("Access denied");
}
};
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
Mockito.doThrow(new AccessControlException("Access denied"))
.when(svc1.proxy).getServiceStatus();
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new AccessControlException("Access denied"))
.when(svc2.proxy).getServiceStatus();
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
fail("Can't failover when access is denied");
} catch (FailoverFailedException ffe) {
assertTrue(ffe.getCause().getMessage().contains("Access denied"));
@ -151,19 +112,13 @@ public class TestFailoverController {
@Test
public void testFailoverToUnreadyService() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE);
DummyService svc2 = new DummyService(HAServiceState.STANDBY) {
@Override
public HAServiceStatus getServiceStatus() throws IOException {
HAServiceStatus ret = new HAServiceStatus(HAServiceState.STANDBY);
ret.setNotReadyToBecomeActive("injected not ready");
return ret;
}
};
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doReturn(STATE_NOT_READY).when(svc2.proxy).getServiceStatus();
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
fail("Can't failover to a service that's not ready");
} catch (FailoverFailedException ffe) {
// Expected
@ -172,95 +127,88 @@ public class TestFailoverController {
}
}
assertEquals(HAServiceState.ACTIVE, svc1.getServiceState());
assertEquals(HAServiceState.STANDBY, svc2.getServiceState());
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
// Forcing it means we ignore readyToBecomeActive
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, true);
assertEquals(HAServiceState.STANDBY, svc1.getServiceState());
assertEquals(HAServiceState.ACTIVE, svc2.getServiceState());
FailoverController.failover(svc1, svc2, false, true);
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(HAServiceState.ACTIVE, svc2.state);
}
@Test
public void testFailoverToUnhealthyServiceFailsAndFailsback() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE);
DummyService svc2 = new DummyService(HAServiceState.STANDBY) {
@Override
public void monitorHealth() throws HealthCheckFailedException {
throw new HealthCheckFailedException("Failed!");
}
};
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new HealthCheckFailedException("Failed!"))
.when(svc2.proxy).monitorHealth();
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
fail("Failover to unhealthy service");
} catch (FailoverFailedException ffe) {
// Expected
}
assertEquals(HAServiceState.ACTIVE, svc1.getServiceState());
assertEquals(HAServiceState.STANDBY, svc2.getServiceState());
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testFailoverFromFaultyServiceSucceeds() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE) {
@Override
public void transitionToStandby() throws ServiceFailedException {
throw new ServiceFailedException("Failed!");
}
};
DummyService svc2 = new DummyService(HAServiceState.STANDBY);
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc1.proxy).transitionToStandby();
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
AlwaysSucceedFencer.fenceCalled = 0;
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
} catch (FailoverFailedException ffe) {
fail("Faulty active prevented failover");
}
// svc1 still thinks it's active, that's OK, it was fenced
assertEquals(1, AlwaysSucceedFencer.fenceCalled);
assertEquals("svc1:1234", AlwaysSucceedFencer.fencedSvc);
assertEquals(HAServiceState.ACTIVE, svc1.getServiceState());
assertEquals(HAServiceState.ACTIVE, svc2.getServiceState());
assertSame(svc1, AlwaysSucceedFencer.fencedSvc);
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.ACTIVE, svc2.state);
}
@Test
public void testFailoverFromFaultyServiceFencingFailure() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE) {
@Override
public void transitionToStandby() throws ServiceFailedException {
throw new ServiceFailedException("Failed!");
}
};
DummyService svc2 = new DummyService(HAServiceState.STANDBY);
NodeFencer fencer = setupFencer(AlwaysFailFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc1.proxy).transitionToStandby();
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());
AlwaysFailFencer.fenceCalled = 0;
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
fail("Failed over even though fencing failed");
} catch (FailoverFailedException ffe) {
// Expected
}
assertEquals(1, AlwaysFailFencer.fenceCalled);
assertEquals("svc1:1234", AlwaysFailFencer.fencedSvc);
assertEquals(HAServiceState.ACTIVE, svc1.getServiceState());
assertEquals(HAServiceState.STANDBY, svc2.getServiceState());
assertSame(svc1, AlwaysFailFencer.fencedSvc);
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testFencingFailureDuringFailover() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE);
DummyService svc2 = new DummyService(HAServiceState.STANDBY);
NodeFencer fencer = setupFencer(AlwaysFailFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());
AlwaysFailFencer.fenceCalled = 0;
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, true, false);
FailoverController.failover(svc1, svc2, true, false);
fail("Failed over even though fencing requested and failed");
} catch (FailoverFailedException ffe) {
// Expected
@ -269,90 +217,83 @@ public class TestFailoverController {
// If fencing was requested and it failed we don't try to make
// svc2 active anyway, and we don't failback to svc1.
assertEquals(1, AlwaysFailFencer.fenceCalled);
assertEquals("svc1:1234", AlwaysFailFencer.fencedSvc);
assertEquals(HAServiceState.STANDBY, svc1.getServiceState());
assertEquals(HAServiceState.STANDBY, svc2.getServiceState());
}
private HAServiceProtocol getProtocol(String target)
throws IOException {
InetSocketAddress addr = NetUtils.createSocketAddr(target);
Configuration conf = new Configuration();
// Lower the timeout so we quickly fail to connect
conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1);
return new HAServiceProtocolClientSideTranslatorPB(addr, conf);
assertSame(svc1, AlwaysFailFencer.fencedSvc);
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testFailoverFromNonExistantServiceWithFencer() throws Exception {
HAServiceProtocol svc1 = getProtocol("localhost:1234");
DummyService svc2 = new DummyService(HAServiceState.STANDBY);
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = spy(new DummyHAService(null, svc1Addr));
// Getting a proxy to a dead server will throw IOException on call,
// not on creation of the proxy.
HAServiceProtocol errorThrowingProxy = Mockito.mock(HAServiceProtocol.class,
new ThrowsException(new IOException("Could not connect to host")));
Mockito.doReturn(errorThrowingProxy).when(svc1).getProxy();
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
} catch (FailoverFailedException ffe) {
fail("Non-existant active prevented failover");
}
// Don't check svc1 because we can't reach it, but that's OK, it's been fenced.
assertEquals(HAServiceState.ACTIVE, svc2.getServiceState());
assertEquals(HAServiceState.ACTIVE, svc2.state);
}
@Test
public void testFailoverToNonExistantServiceFails() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE);
HAServiceProtocol svc2 = getProtocol("localhost:1234");
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = spy(new DummyHAService(null, svc2Addr));
Mockito.doThrow(new IOException("Failed to connect"))
.when(svc2).getProxy(Mockito.<Configuration>any(),
Mockito.anyInt());
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
fail("Failed over to a non-existant standby");
} catch (FailoverFailedException ffe) {
// Expected
}
assertEquals(HAServiceState.ACTIVE, svc1.getServiceState());
assertEquals(HAServiceState.ACTIVE, svc1.state);
}
@Test
public void testFailoverToFaultyServiceFailsbackOK() throws Exception {
DummyService svc1 = spy(new DummyService(HAServiceState.ACTIVE));
DummyService svc2 = new DummyService(HAServiceState.STANDBY) {
@Override
public void transitionToActive() throws ServiceFailedException {
throw new ServiceFailedException("Failed!");
}
};
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = spy(new DummyHAService(HAServiceState.ACTIVE, svc1Addr));
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc2.proxy).transitionToActive();
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
fail("Failover to already active service");
} catch (FailoverFailedException ffe) {
// Expected
}
// svc1 went standby then back to active
verify(svc1).transitionToStandby();
verify(svc1).transitionToActive();
assertEquals(HAServiceState.ACTIVE, svc1.getServiceState());
assertEquals(HAServiceState.STANDBY, svc2.getServiceState());
verify(svc1.proxy).transitionToStandby();
verify(svc1.proxy).transitionToActive();
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testWeDontFailbackIfActiveWasFenced() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE);
DummyService svc2 = new DummyService(HAServiceState.STANDBY) {
@Override
public void transitionToActive() throws ServiceFailedException {
throw new ServiceFailedException("Failed!");
}
};
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc2.proxy).transitionToActive();
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, true, false);
FailoverController.failover(svc1, svc2, true, false);
fail("Failed over to service that won't transition to active");
} catch (FailoverFailedException ffe) {
// Expected
@ -360,24 +301,21 @@ public class TestFailoverController {
// We failed to failover and did not failback because we fenced
// svc1 (we forced it), therefore svc1 and svc2 should be standby.
assertEquals(HAServiceState.STANDBY, svc1.getServiceState());
assertEquals(HAServiceState.STANDBY, svc2.getServiceState());
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testWeFenceOnFailbackIfTransitionToActiveFails() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE);
DummyService svc2 = new DummyService(HAServiceState.STANDBY) {
@Override
public void transitionToActive() throws ServiceFailedException, IOException {
throw new IOException("Failed!");
}
};
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc2.proxy).transitionToActive();
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
AlwaysSucceedFencer.fenceCalled = 0;
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
fail("Failed over to service that won't transition to active");
} catch (FailoverFailedException ffe) {
// Expected
@ -386,25 +324,22 @@ public class TestFailoverController {
// We failed to failover. We did not fence svc1 because it cooperated
// and we didn't force it, so we failed back to svc1 and fenced svc2.
// Note svc2 still thinks it's active, that's OK, we fenced it.
assertEquals(HAServiceState.ACTIVE, svc1.getServiceState());
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(1, AlwaysSucceedFencer.fenceCalled);
assertEquals("svc2:5678", AlwaysSucceedFencer.fencedSvc);
assertSame(svc2, AlwaysSucceedFencer.fencedSvc);
}
@Test
public void testFailureToFenceOnFailbackFailsTheFailback() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE);
DummyService svc2 = new DummyService(HAServiceState.STANDBY) {
@Override
public void transitionToActive() throws ServiceFailedException, IOException {
throw new IOException("Failed!");
}
};
NodeFencer fencer = setupFencer(AlwaysFailFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new IOException("Failed!"))
.when(svc2.proxy).transitionToActive();
svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());
AlwaysFailFencer.fenceCalled = 0;
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
fail("Failed over to service that won't transition to active");
} catch (FailoverFailedException ffe) {
// Expected
@ -413,35 +348,30 @@ public class TestFailoverController {
// We did not fence svc1 because it cooperated and we didn't force it,
// we failed to failover so we fenced svc2, we failed to fence svc2
// so we did not failback to svc1, ie it's still standby.
assertEquals(HAServiceState.STANDBY, svc1.getServiceState());
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(1, AlwaysFailFencer.fenceCalled);
assertEquals("svc2:5678", AlwaysFailFencer.fencedSvc);
assertSame(svc2, AlwaysFailFencer.fencedSvc);
}
@Test
public void testFailbackToFaultyServiceFails() throws Exception {
DummyService svc1 = new DummyService(HAServiceState.ACTIVE) {
@Override
public void transitionToActive() throws ServiceFailedException {
throw new ServiceFailedException("Failed!");
}
};
DummyService svc2 = new DummyService(HAServiceState.STANDBY) {
@Override
public void transitionToActive() throws ServiceFailedException {
throw new ServiceFailedException("Failed!");
}
};
NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName());
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc1.proxy).transitionToActive();
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc2.proxy).transitionToActive();
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false);
FailoverController.failover(svc1, svc2, false, false);
fail("Failover to already active service");
} catch (FailoverFailedException ffe) {
// Expected
}
assertEquals(HAServiceState.STANDBY, svc1.getServiceState());
assertEquals(HAServiceState.STANDBY, svc2.getServiceState());
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
}

10
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java
View File

@ -22,14 +22,15 @@ import static org.junit.Assert.*;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.InetSocketAddress;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.Log;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
@ -40,15 +41,14 @@ public class TestHAAdmin {
private HAAdmin tool;
private ByteArrayOutputStream errOutBytes = new ByteArrayOutputStream();
private String errOutput;
private HAServiceProtocol mockProtocol;
@Before
public void setup() throws IOException {
mockProtocol = Mockito.mock(HAServiceProtocol.class);
tool = new HAAdmin() {
@Override
protected HAServiceProtocol getProtocol(String target) throws IOException {
return mockProtocol;
protected HAServiceTarget resolveTarget(String target) {
return new DummyHAService(HAServiceState.STANDBY,
new InetSocketAddress("dummy", 12345));
}
};
tool.setConf(new Configuration());

47
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java
View File

@ -26,26 +26,35 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import com.google.common.collect.Lists;
public class TestNodeFencer {
private HAServiceTarget MOCK_TARGET;
@Before
public void clearMockState() {
AlwaysSucceedFencer.fenceCalled = 0;
AlwaysSucceedFencer.callArgs.clear();
AlwaysFailFencer.fenceCalled = 0;
AlwaysFailFencer.callArgs.clear();
MOCK_TARGET = Mockito.mock(HAServiceTarget.class);
Mockito.doReturn("my mock").when(MOCK_TARGET).toString();
Mockito.doReturn(new InetSocketAddress("host", 1234))
.when(MOCK_TARGET).getAddress();
}
@Test
public void testSingleFencer() throws BadFencingConfigurationException {
NodeFencer fencer = setupFencer(
AlwaysSucceedFencer.class.getName() + "(foo)");
assertTrue(fencer.fence(new InetSocketAddress("host", 1234)));
assertTrue(fencer.fence(MOCK_TARGET));
assertEquals(1, AlwaysSucceedFencer.fenceCalled);
assertEquals("host:1234", AlwaysSucceedFencer.fencedSvc);
assertSame(MOCK_TARGET, AlwaysSucceedFencer.fencedSvc);
assertEquals("foo", AlwaysSucceedFencer.callArgs.get(0));
}
@ -54,7 +63,7 @@ public class TestNodeFencer {
NodeFencer fencer = setupFencer(
AlwaysSucceedFencer.class.getName() + "(foo)\n" +
AlwaysSucceedFencer.class.getName() + "(bar)\n");
assertTrue(fencer.fence(new InetSocketAddress("host", 1234)));
assertTrue(fencer.fence(MOCK_TARGET));
// Only one call, since the first fencer succeeds
assertEquals(1, AlwaysSucceedFencer.fenceCalled);
assertEquals("foo", AlwaysSucceedFencer.callArgs.get(0));
@ -68,12 +77,12 @@ public class TestNodeFencer {
" # the next one will always fail\n" +
" " + AlwaysFailFencer.class.getName() + "(foo) # <- fails\n" +
AlwaysSucceedFencer.class.getName() + "(bar) \n");
assertTrue(fencer.fence(new InetSocketAddress("host", 1234)));
assertTrue(fencer.fence(MOCK_TARGET));
// One call to each, since top fencer fails
assertEquals(1, AlwaysFailFencer.fenceCalled);
assertEquals("host:1234", AlwaysFailFencer.fencedSvc);
assertSame(MOCK_TARGET, AlwaysFailFencer.fencedSvc);
assertEquals(1, AlwaysSucceedFencer.fenceCalled);
assertEquals("host:1234", AlwaysSucceedFencer.fencedSvc);
assertSame(MOCK_TARGET, AlwaysSucceedFencer.fencedSvc);
assertEquals("foo", AlwaysFailFencer.callArgs.get(0));
assertEquals("bar", AlwaysSucceedFencer.callArgs.get(0));
}
@ -82,41 +91,41 @@ public class TestNodeFencer {
public void testArglessFencer() throws BadFencingConfigurationException {
NodeFencer fencer = setupFencer(
AlwaysSucceedFencer.class.getName());
assertTrue(fencer.fence(new InetSocketAddress("host", 1234)));
assertTrue(fencer.fence(MOCK_TARGET));
// One call to each, since top fencer fails
assertEquals(1, AlwaysSucceedFencer.fenceCalled);
assertEquals("host:1234", AlwaysSucceedFencer.fencedSvc);
assertSame(MOCK_TARGET, AlwaysSucceedFencer.fencedSvc);
assertEquals(null, AlwaysSucceedFencer.callArgs.get(0));
}
@Test
public void testShortNameShell() throws BadFencingConfigurationException {
NodeFencer fencer = setupFencer("shell(true)");
assertTrue(fencer.fence(new InetSocketAddress("host", 1234)));
assertTrue(fencer.fence(MOCK_TARGET));
}
@Test
public void testShortNameSsh() throws BadFencingConfigurationException {
NodeFencer fencer = setupFencer("sshfence");
assertFalse(fencer.fence(new InetSocketAddress("host", 1234)));
assertFalse(fencer.fence(MOCK_TARGET));
}
@Test
public void testShortNameSshWithUser() throws BadFencingConfigurationException {
NodeFencer fencer = setupFencer("sshfence(user)");
assertFalse(fencer.fence(new InetSocketAddress("host", 1234)));
assertFalse(fencer.fence(MOCK_TARGET));
}
@Test
public void testShortNameSshWithPort() throws BadFencingConfigurationException {
NodeFencer fencer = setupFencer("sshfence(:123)");
assertFalse(fencer.fence(new InetSocketAddress("host", 1234)));
assertFalse(fencer.fence(MOCK_TARGET));
}
@Test
public void testShortNameSshWithUserPort() throws BadFencingConfigurationException {
NodeFencer fencer = setupFencer("sshfence(user:123)");
assertFalse(fencer.fence(new InetSocketAddress("host", 1234)));
assertFalse(fencer.fence(MOCK_TARGET));
}
public static NodeFencer setupFencer(String confStr)
@ -133,12 +142,12 @@ public class TestNodeFencer {
public static class AlwaysSucceedFencer extends Configured
implements FenceMethod {
static int fenceCalled = 0;
static String fencedSvc;
static HAServiceTarget fencedSvc;
static List<String> callArgs = Lists.newArrayList();
@Override
public boolean tryFence(InetSocketAddress serviceAddr, String args) {
fencedSvc = serviceAddr.getHostName() + ":" + serviceAddr.getPort();
public boolean tryFence(HAServiceTarget target, String args) {
fencedSvc = target;
callArgs.add(args);
fenceCalled++;
return true;
@ -155,12 +164,12 @@ public class TestNodeFencer {
public static class AlwaysFailFencer extends Configured
implements FenceMethod {
static int fenceCalled = 0;
static String fencedSvc;
static HAServiceTarget fencedSvc;
static List<String> callArgs = Lists.newArrayList();
@Override
public boolean tryFence(InetSocketAddress serviceAddr, String args) {
fencedSvc = serviceAddr.getHostName() + ":" + serviceAddr.getPort();
public boolean tryFence(HAServiceTarget target, String args) {
fencedSvc = target;
callArgs.add(args);
fenceCalled++;
return false;

23
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java
View File

@ -22,6 +22,7 @@ import static org.junit.Assert.*;
import java.net.InetSocketAddress;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.BeforeClass;
@ -32,6 +33,9 @@ import static org.mockito.Mockito.spy;
public class TestShellCommandFencer {
private ShellCommandFencer fencer = createFencer();
private static final HAServiceTarget TEST_TARGET =
new DummyHAService(HAServiceState.ACTIVE,
new InetSocketAddress("host", 1234));
@BeforeClass
public static void setupLogSpy() {
@ -57,11 +61,10 @@ public class TestShellCommandFencer {
*/
@Test
public void testBasicSuccessFailure() {
InetSocketAddress addr = new InetSocketAddress("host", 1234);
assertTrue(fencer.tryFence(addr, "echo"));
assertFalse(fencer.tryFence(addr, "exit 1"));
assertTrue(fencer.tryFence(TEST_TARGET, "echo"));
assertFalse(fencer.tryFence(TEST_TARGET, "exit 1"));
// bad path should also fail
assertFalse(fencer.tryFence(addr, "xxxxxxxxxxxx"));
assertFalse(fencer.tryFence(TEST_TARGET, "xxxxxxxxxxxx"));
}
@Test
@ -98,8 +101,7 @@ public class TestShellCommandFencer {
*/
@Test
public void testStdoutLogging() {
InetSocketAddress addr = new InetSocketAddress("host", 1234);
assertTrue(fencer.tryFence(addr, "echo hello"));
assertTrue(fencer.tryFence(TEST_TARGET, "echo hello"));
Mockito.verify(ShellCommandFencer.LOG).info(
Mockito.endsWith("echo hello: host:1234 hello"));
}
@ -110,8 +112,7 @@ public class TestShellCommandFencer {
*/
@Test
public void testStderrLogging() {
InetSocketAddress addr = new InetSocketAddress("host", 1234);
assertTrue(fencer.tryFence(addr, "echo hello >&2"));
assertTrue(fencer.tryFence(TEST_TARGET, "echo hello >&2"));
Mockito.verify(ShellCommandFencer.LOG).warn(
Mockito.endsWith("echo hello >&2: host:1234 hello"));
}
@ -122,8 +123,7 @@ public class TestShellCommandFencer {
*/
@Test
public void testConfAsEnvironment() {
InetSocketAddress addr = new InetSocketAddress("host", 1234);
fencer.tryFence(addr, "echo $in_fencing_tests");
fencer.tryFence(TEST_TARGET, "echo $in_fencing_tests");
Mockito.verify(ShellCommandFencer.LOG).info(
Mockito.endsWith("echo $in...ing_tests: host:1234 yessir"));
}
@ -136,8 +136,7 @@ public class TestShellCommandFencer {
*/
@Test(timeout=10000)
public void testSubprocessInputIsClosed() {
InetSocketAddress addr = new InetSocketAddress("host", 1234);
assertFalse(fencer.tryFence(addr, "read"));
assertFalse(fencer.tryFence(TEST_TARGET, "read"));
}
@Test

26
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestSshFenceByTcpPort.java
View File

@ -23,6 +23,7 @@ import java.net.InetSocketAddress;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.SshFenceByTcpPort.Args;
import org.apache.log4j.Level;
import org.junit.Assume;
@ -34,13 +35,26 @@ public class TestSshFenceByTcpPort {
((Log4JLogger)SshFenceByTcpPort.LOG).getLogger().setLevel(Level.ALL);
}
private String TEST_FENCING_HOST = System.getProperty(
private static String TEST_FENCING_HOST = System.getProperty(
"test.TestSshFenceByTcpPort.host", "localhost");
private String TEST_FENCING_PORT = System.getProperty(
private static final String TEST_FENCING_PORT = System.getProperty(
"test.TestSshFenceByTcpPort.port", "8020");
private final String TEST_KEYFILE = System.getProperty(
private static final String TEST_KEYFILE = System.getProperty(
"test.TestSshFenceByTcpPort.key");
private static final InetSocketAddress TEST_ADDR =
new InetSocketAddress(TEST_FENCING_HOST,
Integer.valueOf(TEST_FENCING_PORT));
private static final HAServiceTarget TEST_TARGET =
new DummyHAService(HAServiceState.ACTIVE, TEST_ADDR);
/**
* Connect to Google's DNS server - not running ssh!
*/
private static final HAServiceTarget UNFENCEABLE_TARGET =
new DummyHAService(HAServiceState.ACTIVE,
new InetSocketAddress("8.8.8.8", 1234));
@Test(timeout=20000)
public void testFence() throws BadFencingConfigurationException {
Assume.assumeTrue(isConfigured());
@ -49,8 +63,7 @@ public class TestSshFenceByTcpPort {
SshFenceByTcpPort fence = new SshFenceByTcpPort();
fence.setConf(conf);
assertTrue(fence.tryFence(
new InetSocketAddress(TEST_FENCING_HOST,
Integer.valueOf(TEST_FENCING_PORT)),
TEST_TARGET,
null));
}
@ -65,8 +78,7 @@ public class TestSshFenceByTcpPort {
conf.setInt(SshFenceByTcpPort.CONF_CONNECT_TIMEOUT_KEY, 3000);
SshFenceByTcpPort fence = new SshFenceByTcpPort();
fence.setConf(conf);
// Connect to Google's DNS server - not running ssh!
assertFalse(fence.tryFence(new InetSocketAddress("8.8.8.8", 1234), ""));
assertFalse(fence.tryFence(UNFENCEABLE_TARGET, ""));
}
@Test

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
View File

@ -25,8 +25,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ha.HAAdmin;
import org.apache.hadoop.ha.HAServiceTarget;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.util.ToolRunner;
@ -65,15 +65,9 @@ public class DFSHAAdmin extends HAAdmin {
* Try to map the given namenode ID to its service address.
*/
@Override
protected String getServiceAddr(String nnId) {
protected HAServiceTarget resolveTarget(String nnId) {
HdfsConfiguration conf = (HdfsConfiguration)getConf();
String serviceAddr =
DFSUtil.getNamenodeServiceAddr(conf, nameserviceId, nnId);
if (serviceAddr == null) {
throw new IllegalArgumentException(
"Unable to determine service address for namenode '" + nnId + "'");
}
return serviceAddr;
return new NNHAServiceTarget(conf, nameserviceId, nnId);
}
@Override

84
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java Normal file
View File

@ -0,0 +1,84 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.tools;
import java.net.InetSocketAddress;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.ha.BadFencingConfigurationException;
import org.apache.hadoop.ha.HAServiceTarget;
import org.apache.hadoop.ha.NodeFencer;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.net.NetUtils;
/**
* One of the NN NameNodes acting as the target of an administrative command
* (e.g. failover).
*/
@InterfaceAudience.Private
public class NNHAServiceTarget extends HAServiceTarget {
private final InetSocketAddress addr;
private NodeFencer fencer;
private BadFencingConfigurationException fenceConfigError;
public NNHAServiceTarget(HdfsConfiguration conf,
String nsId, String nnId) {
String serviceAddr =
DFSUtil.getNamenodeServiceAddr(conf, nsId, nnId);
if (serviceAddr == null) {
throw new IllegalArgumentException(
"Unable to determine service address for namenode '" + nnId + "'");
}
this.addr = NetUtils.createSocketAddr(serviceAddr,
NameNode.DEFAULT_PORT);
try {
this.fencer = NodeFencer.create(conf);
} catch (BadFencingConfigurationException e) {
this.fenceConfigError = e;
}
}
/**
* @return the NN's IPC address.
*/
@Override
public InetSocketAddress getAddress() {
return addr;
}
@Override
public void checkFencingConfigured() throws BadFencingConfigurationException {
if (fenceConfigError != null) {
throw fenceConfigError;
}
}
@Override
public NodeFencer getFencer() {
return fencer;
}
@Override
public String toString() {
return "NameNode at " + addr;
}
}

15
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
View File

@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.ha.HAServiceTarget;
import org.apache.hadoop.ha.HealthCheckFailedException;
import org.apache.hadoop.ha.NodeFencer;
@ -79,10 +80,18 @@ public class TestDFSHAAdmin {
public void setup() throws IOException {
mockProtocol = Mockito.mock(HAServiceProtocol.class);
tool = new DFSHAAdmin() {
@Override
protected HAServiceProtocol getProtocol(String serviceId) throws IOException {
getServiceAddr(serviceId);
return mockProtocol;
protected HAServiceTarget resolveTarget(String nnId) {
HAServiceTarget target = super.resolveTarget(nnId);
HAServiceTarget spy = Mockito.spy(target);
// OVerride the target to return our mock protocol
try {
Mockito.doReturn(mockProtocol).when(spy).getProxy();
} catch (IOException e) {
throw new AssertionError(e); // mock setup doesn't really throw
}
return spy;
}
};
tool.setConf(getHAConf());