From 90ea037d21be08821fc9c95b80bec1e1078d79cc Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Sat, 24 Mar 2012 23:48:07 +0000 Subject: [PATCH] HADOOP-8193. Refactor FailoverController/HAAdmin code to add an abstract class for "target" services. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1304967 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../ha/BadFencingConfigurationException.java | 7 +- .../apache/hadoop/ha/FailoverController.java | 50 ++- .../org/apache/hadoop/ha/FenceMethod.java | 4 +- .../java/org/apache/hadoop/ha/HAAdmin.java | 52 +-- .../org/apache/hadoop/ha/HAServiceTarget.java | 74 ++++ .../java/org/apache/hadoop/ha/NodeFencer.java | 5 +- .../apache/hadoop/ha/ShellCommandFencer.java | 3 +- .../apache/hadoop/ha/SshFenceByTcpPort.java | 3 +- .../org/apache/hadoop/ha/DummyHAService.java | 94 +++++ .../hadoop/ha/TestFailoverController.java | 358 +++++++----------- .../org/apache/hadoop/ha/TestHAAdmin.java | 12 +- .../org/apache/hadoop/ha/TestNodeFencer.java | 47 ++- .../hadoop/ha/TestShellCommandFencer.java | 23 +- .../hadoop/ha/TestSshFenceByTcpPort.java | 26 +- .../apache/hadoop/hdfs/tools/DFSHAAdmin.java | 12 +- .../hadoop/hdfs/tools/NNHAServiceTarget.java | 84 ++++ .../hadoop/hdfs/tools/TestDFSHAAdmin.java | 15 +- 18 files changed, 524 insertions(+), 348 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f3fda7668c5..f0b9b945c07 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -220,6 +220,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8163. Improve ActiveStandbyElector to provide hooks for fencing old active. (todd) + HADOOP-8193. Refactor FailoverController/HAAdmin code to add an abstract + class for "target" services. (todd) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/BadFencingConfigurationException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/BadFencingConfigurationException.java index 3d3b1ba53cc..a5641cf5066 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/BadFencingConfigurationException.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/BadFencingConfigurationException.java @@ -19,11 +19,16 @@ package org.apache.hadoop.ha; import java.io.IOException; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + /** * Indicates that the operator has specified an invalid configuration * for fencing methods. */ -class BadFencingConfigurationException extends IOException { +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class BadFencingConfigurationException extends IOException { private static final long serialVersionUID = 1L; public BadFencingConfigurationException(String msg) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java index c8878e8b739..9cff2a50c45 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ha; import java.io.IOException; -import java.net.InetSocketAddress; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -51,21 +50,21 @@ public class FailoverController { * allow it to become active, eg because it triggers a log roll * so the standby can learn about new blocks and leave safemode. * - * @param toSvc service to make active - * @param toSvcName name of service to make active + * @param target service to make active * @param forceActive ignore toSvc if it reports that it is not ready * @throws FailoverFailedException if we should avoid failover */ - private static void preFailoverChecks(HAServiceProtocol toSvc, - InetSocketAddress toSvcAddr, + private static void preFailoverChecks(HAServiceTarget target, boolean forceActive) throws FailoverFailedException { HAServiceStatus toSvcStatus; + HAServiceProtocol toSvc; try { + toSvc = target.getProxy(); toSvcStatus = toSvc.getServiceStatus(); } catch (IOException e) { - String msg = "Unable to get service state for " + toSvcAddr; + String msg = "Unable to get service state for " + target; LOG.error(msg, e); throw new FailoverFailedException(msg, e); } @@ -79,7 +78,7 @@ public class FailoverController { String notReadyReason = toSvcStatus.getNotReadyReason(); if (!forceActive) { throw new FailoverFailedException( - toSvcAddr + " is not ready to become active: " + + target + " is not ready to become active: " + notReadyReason); } else { LOG.warn("Service is not ready to become active, but forcing: " + @@ -103,44 +102,39 @@ public class FailoverController { * then try to failback. * * @param fromSvc currently active service - * @param fromSvcAddr addr of the currently active service * @param toSvc service to make active - * @param toSvcAddr addr of the service to make active - * @param fencer for fencing fromSvc * @param forceFence to fence fromSvc even if not strictly necessary * @param forceActive try to make toSvc active even if it is not ready * @throws FailoverFailedException if the failover fails */ - public static void failover(HAServiceProtocol fromSvc, - InetSocketAddress fromSvcAddr, - HAServiceProtocol toSvc, - InetSocketAddress toSvcAddr, - NodeFencer fencer, + public static void failover(HAServiceTarget fromSvc, + HAServiceTarget toSvc, boolean forceFence, boolean forceActive) throws FailoverFailedException { - Preconditions.checkArgument(fencer != null, "failover requires a fencer"); - preFailoverChecks(toSvc, toSvcAddr, forceActive); + Preconditions.checkArgument(fromSvc.getFencer() != null, + "failover requires a fencer"); + preFailoverChecks(toSvc, forceActive); // Try to make fromSvc standby boolean tryFence = true; try { - HAServiceProtocolHelper.transitionToStandby(fromSvc); + HAServiceProtocolHelper.transitionToStandby(fromSvc.getProxy()); // We should try to fence if we failed or it was forced tryFence = forceFence ? true : false; } catch (ServiceFailedException sfe) { - LOG.warn("Unable to make " + fromSvcAddr + " standby (" + + LOG.warn("Unable to make " + fromSvc + " standby (" + sfe.getMessage() + ")"); } catch (IOException ioe) { - LOG.warn("Unable to make " + fromSvcAddr + + LOG.warn("Unable to make " + fromSvc + " standby (unable to connect)", ioe); } // Fence fromSvc if it's required or forced by the user if (tryFence) { - if (!fencer.fence(fromSvcAddr)) { + if (!fromSvc.getFencer().fence(fromSvc)) { throw new FailoverFailedException("Unable to fence " + - fromSvcAddr + ". Fencing failed."); + fromSvc + ". Fencing failed."); } } @@ -148,14 +142,14 @@ public class FailoverController { boolean failed = false; Throwable cause = null; try { - HAServiceProtocolHelper.transitionToActive(toSvc); + HAServiceProtocolHelper.transitionToActive(toSvc.getProxy()); } catch (ServiceFailedException sfe) { - LOG.error("Unable to make " + toSvcAddr + " active (" + + LOG.error("Unable to make " + toSvc + " active (" + sfe.getMessage() + "). Failing back."); failed = true; cause = sfe; } catch (IOException ioe) { - LOG.error("Unable to make " + toSvcAddr + + LOG.error("Unable to make " + toSvc + " active (unable to connect). Failing back.", ioe); failed = true; cause = ioe; @@ -163,7 +157,7 @@ public class FailoverController { // We failed to make toSvc active if (failed) { - String msg = "Unable to failover to " + toSvcAddr; + String msg = "Unable to failover to " + toSvc; // Only try to failback if we didn't fence fromSvc if (!tryFence) { try { @@ -171,9 +165,9 @@ public class FailoverController { // become active, eg we timed out waiting for its response. // Unconditionally force fromSvc to become active since it // was previously active when we initiated failover. - failover(toSvc, toSvcAddr, fromSvc, fromSvcAddr, fencer, true, true); + failover(toSvc, fromSvc, true, true); } catch (FailoverFailedException ffe) { - msg += ". Failback to " + fromSvcAddr + + msg += ". Failback to " + fromSvc + " failed (" + ffe.getMessage() + ")"; LOG.fatal(msg); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FenceMethod.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FenceMethod.java index d8bda1402fa..ac343fe3478 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FenceMethod.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FenceMethod.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.ha; -import java.net.InetSocketAddress; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; @@ -62,6 +60,6 @@ public interface FenceMethod { * @throws BadFencingConfigurationException if the configuration was * determined to be invalid only at runtime */ - public boolean tryFence(InetSocketAddress serviceAddr, String args) + public boolean tryFence(HAServiceTarget target, String args) throws BadFencingConfigurationException; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java index a16ffb4c400..3839eb3b999 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java @@ -19,7 +19,6 @@ package org.apache.hadoop.ha; import java.io.IOException; import java.io.PrintStream; -import java.net.InetSocketAddress; import java.util.Map; import org.apache.commons.cli.Options; @@ -28,11 +27,8 @@ import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.ParseException; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.ha.protocolPB.HAServiceProtocolClientSideTranslatorPB; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -77,6 +73,8 @@ public abstract class HAAdmin extends Configured implements Tool { protected PrintStream errOut = System.err; PrintStream out = System.out; + protected abstract HAServiceTarget resolveTarget(String string); + protected String getUsageString() { return "Usage: HAAdmin"; } @@ -109,7 +107,7 @@ public abstract class HAAdmin extends Configured implements Tool { return -1; } - HAServiceProtocol proto = getProtocol(argv[1]); + HAServiceProtocol proto = resolveTarget(argv[1]).getProxy(); HAServiceProtocolHelper.transitionToActive(proto); return 0; } @@ -122,14 +120,13 @@ public abstract class HAAdmin extends Configured implements Tool { return -1; } - HAServiceProtocol proto = getProtocol(argv[1]); + HAServiceProtocol proto = resolveTarget(argv[1]).getProxy(); HAServiceProtocolHelper.transitionToStandby(proto); return 0; } private int failover(final String[] argv) throws IOException, ServiceFailedException { - Configuration conf = getConf(); boolean forceFence = false; boolean forceActive = false; @@ -162,29 +159,12 @@ public abstract class HAAdmin extends Configured implements Tool { return -1; } - NodeFencer fencer; + HAServiceTarget fromNode = resolveTarget(args[0]); + HAServiceTarget toNode = resolveTarget(args[1]); + try { - fencer = NodeFencer.create(conf); - } catch (BadFencingConfigurationException bfce) { - errOut.println("failover: incorrect fencing configuration: " + - bfce.getLocalizedMessage()); - return -1; - } - if (fencer == null) { - errOut.println("failover: no fencer configured"); - return -1; - } - - InetSocketAddress addr1 = - NetUtils.createSocketAddr(getServiceAddr(args[0])); - InetSocketAddress addr2 = - NetUtils.createSocketAddr(getServiceAddr(args[1])); - HAServiceProtocol proto1 = getProtocol(args[0]); - HAServiceProtocol proto2 = getProtocol(args[1]); - - try { - FailoverController.failover(proto1, addr1, proto2, addr2, - fencer, forceFence, forceActive); + FailoverController.failover(fromNode, toNode, + forceFence, forceActive); out.println("Failover from "+args[0]+" to "+args[1]+" successful"); } catch (FailoverFailedException ffe) { errOut.println("Failover failed: " + ffe.getLocalizedMessage()); @@ -201,7 +181,7 @@ public abstract class HAAdmin extends Configured implements Tool { return -1; } - HAServiceProtocol proto = getProtocol(argv[1]); + HAServiceProtocol proto = resolveTarget(argv[1]).getProxy(); try { HAServiceProtocolHelper.monitorHealth(proto); } catch (HealthCheckFailedException e) { @@ -219,7 +199,7 @@ public abstract class HAAdmin extends Configured implements Tool { return -1; } - HAServiceProtocol proto = getProtocol(argv[1]); + HAServiceProtocol proto = resolveTarget(argv[1]).getProxy(); out.println(proto.getServiceStatus().getState()); return 0; } @@ -232,16 +212,6 @@ public abstract class HAAdmin extends Configured implements Tool { return serviceId; } - /** - * Return a proxy to the specified target service. - */ - protected HAServiceProtocol getProtocol(String serviceId) - throws IOException { - String serviceAddr = getServiceAddr(serviceId); - InetSocketAddress addr = NetUtils.createSocketAddr(serviceAddr); - return new HAServiceProtocolClientSideTranslatorPB(addr, getConf()); - } - @Override public int run(String[] argv) throws Exception { try { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java new file mode 100644 index 00000000000..78a2f2e4d98 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ha; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.ha.protocolPB.HAServiceProtocolClientSideTranslatorPB; + +/** + * Represents a target of the client side HA administration commands. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class HAServiceTarget { + + /** + * @return the IPC address of the target node. + */ + public abstract InetSocketAddress getAddress(); + + /** + * @return a Fencer implementation configured for this target node + */ + public abstract NodeFencer getFencer(); + + /** + * @throws BadFencingConfigurationException if the fencing configuration + * appears to be invalid. This is divorced from the above + * {@link #getFencer()} method so that the configuration can be checked + * during the pre-flight phase of failover. + */ + public abstract void checkFencingConfigured() + throws BadFencingConfigurationException; + + /** + * @return a proxy to connect to the target HA Service. + */ + public HAServiceProtocol getProxy(Configuration conf, int timeoutMs) + throws IOException { + Configuration confCopy = new Configuration(conf); + // Lower the timeout so we quickly fail to connect + confCopy.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1); + return new HAServiceProtocolClientSideTranslatorPB( + getAddress(), + confCopy, null, timeoutMs); + } + + /** + * @return a proxy to connect to the target HA Service. + */ + public final HAServiceProtocol getProxy() throws IOException { + return getProxy(new Configuration(), 0); // default conf, timeout + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java index 34a2c8b823a..f77df089dbf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.ha; -import java.net.InetSocketAddress; import java.util.List; import java.util.Map; import java.util.regex.Matcher; @@ -91,14 +90,14 @@ public class NodeFencer { return new NodeFencer(conf); } - public boolean fence(InetSocketAddress serviceAddr) { + public boolean fence(HAServiceTarget fromSvc) { LOG.info("====== Beginning Service Fencing Process... ======"); int i = 0; for (FenceMethodWithArg method : methods) { LOG.info("Trying method " + (++i) + "/" + methods.size() +": " + method); try { - if (method.method.tryFence(serviceAddr, method.arg)) { + if (method.method.tryFence(fromSvc, method.arg)) { LOG.info("====== Fencing successful by method " + method + " ======"); return true; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java index ca81f23a187..db5676dfc0e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java @@ -75,7 +75,8 @@ public class ShellCommandFencer } @Override - public boolean tryFence(InetSocketAddress serviceAddr, String cmd) { + public boolean tryFence(HAServiceTarget target, String cmd) { + InetSocketAddress serviceAddr = target.getAddress(); List cmdList = Arrays.asList(cmd.split("\\s+")); // Create arg list with service as the first argument diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java index 00b9a83a572..537fba942de 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java @@ -79,10 +79,11 @@ public class SshFenceByTcpPort extends Configured } @Override - public boolean tryFence(InetSocketAddress serviceAddr, String argsStr) + public boolean tryFence(HAServiceTarget target, String argsStr) throws BadFencingConfigurationException { Args args = new Args(argsStr); + InetSocketAddress serviceAddr = target.getAddress(); String host = serviceAddr.getHostName(); Session session; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java new file mode 100644 index 00000000000..69c4a6fde41 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ha; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; +import org.apache.hadoop.security.AccessControlException; +import org.mockito.Mockito; + +/** + * Test-only implementation of {@link HAServiceTarget}, which returns + * a mock implementation. + */ +class DummyHAService extends HAServiceTarget { + HAServiceState state; + HAServiceProtocol proxy; + NodeFencer fencer; + InetSocketAddress address; + + DummyHAService(HAServiceState state, InetSocketAddress address) { + this.state = state; + this.proxy = makeMock(); + this.fencer = Mockito.mock(NodeFencer.class); + this.address = address; + } + + private HAServiceProtocol makeMock() { + return Mockito.spy(new HAServiceProtocol() { + @Override + public void monitorHealth() throws HealthCheckFailedException, + AccessControlException, IOException { + } + + @Override + public void transitionToActive() throws ServiceFailedException, + AccessControlException, IOException { + state = HAServiceState.ACTIVE; + } + + @Override + public void transitionToStandby() throws ServiceFailedException, + AccessControlException, IOException { + state = HAServiceState.STANDBY; + } + + @Override + public HAServiceStatus getServiceStatus() throws IOException { + HAServiceStatus ret = new HAServiceStatus(state); + if (state == HAServiceState.STANDBY) { + ret.setReadyToBecomeActive(); + } + return ret; + } + }); + } + + @Override + public InetSocketAddress getAddress() { + return address; + } + + @Override + public HAServiceProtocol getProxy(Configuration conf, int timeout) + throws IOException { + return proxy; + } + + @Override + public NodeFencer getFencer() { + return fencer; + } + + @Override + public void checkFencingConfigured() throws BadFencingConfigurationException { + } +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestFailoverController.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestFailoverController.java index 6dec32c636d..218bfec4ee9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestFailoverController.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestFailoverController.java @@ -24,124 +24,85 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; -import org.apache.hadoop.ha.protocolPB.HAServiceProtocolClientSideTranslatorPB; import org.apache.hadoop.ha.TestNodeFencer.AlwaysSucceedFencer; import org.apache.hadoop.ha.TestNodeFencer.AlwaysFailFencer; import static org.apache.hadoop.ha.TestNodeFencer.setupFencer; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.internal.stubbing.answers.ThrowsException; +import org.mockito.stubbing.Answer; + import static org.junit.Assert.*; public class TestFailoverController { - private InetSocketAddress svc1Addr = new InetSocketAddress("svc1", 1234); - private InetSocketAddress svc2Addr = new InetSocketAddress("svc2", 5678); + private InetSocketAddress svc2Addr = new InetSocketAddress("svc2", 5678); - private class DummyService implements HAServiceProtocol { - HAServiceState state; + HAServiceStatus STATE_NOT_READY = new HAServiceStatus(HAServiceState.STANDBY) + .setNotReadyToBecomeActive("injected not ready"); - DummyService(HAServiceState state) { - this.state = state; - } - - @Override - public void monitorHealth() throws HealthCheckFailedException, IOException { - // Do nothing - } - - @Override - public void transitionToActive() throws ServiceFailedException, IOException { - state = HAServiceState.ACTIVE; - } - - @Override - public void transitionToStandby() throws ServiceFailedException, IOException { - state = HAServiceState.STANDBY; - } - - @Override - public HAServiceStatus getServiceStatus() throws IOException { - HAServiceStatus ret = new HAServiceStatus(state); - if (state == HAServiceState.STANDBY) { - ret.setReadyToBecomeActive(); - } - return ret; - } - - private HAServiceState getServiceState() { - return state; - } - } - @Test public void testFailoverAndFailback() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE); - DummyService svc2 = new DummyService(HAServiceState.STANDBY); - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); AlwaysSucceedFencer.fenceCalled = 0; - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled); - assertEquals(HAServiceState.STANDBY, svc1.getServiceState()); - assertEquals(HAServiceState.ACTIVE, svc2.getServiceState()); + assertEquals(HAServiceState.STANDBY, svc1.state); + assertEquals(HAServiceState.ACTIVE, svc2.state); AlwaysSucceedFencer.fenceCalled = 0; - FailoverController.failover(svc2, svc2Addr, svc1, svc1Addr, fencer, false, false); + FailoverController.failover(svc2, svc1, false, false); assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled); - assertEquals(HAServiceState.ACTIVE, svc1.getServiceState()); - assertEquals(HAServiceState.STANDBY, svc2.getServiceState()); + assertEquals(HAServiceState.ACTIVE, svc1.state); + assertEquals(HAServiceState.STANDBY, svc2.state); } @Test public void testFailoverFromStandbyToStandby() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.STANDBY); - DummyService svc2 = new DummyService(HAServiceState.STANDBY); - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.STANDBY, svc1Addr); + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); - assertEquals(HAServiceState.STANDBY, svc1.getServiceState()); - assertEquals(HAServiceState.ACTIVE, svc2.getServiceState()); + FailoverController.failover(svc1, svc2, false, false); + assertEquals(HAServiceState.STANDBY, svc1.state); + assertEquals(HAServiceState.ACTIVE, svc2.state); } @Test public void testFailoverFromActiveToActive() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE); - DummyService svc2 = new DummyService(HAServiceState.ACTIVE); - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + DummyHAService svc2 = new DummyHAService(HAServiceState.ACTIVE, svc2Addr); + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); fail("Can't failover to an already active service"); } catch (FailoverFailedException ffe) { // Expected } - assertEquals(HAServiceState.ACTIVE, svc1.getServiceState()); - assertEquals(HAServiceState.ACTIVE, svc2.getServiceState()); + assertEquals(HAServiceState.ACTIVE, svc1.state); + assertEquals(HAServiceState.ACTIVE, svc2.state); } @Test public void testFailoverWithoutPermission() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE) { - @Override - public HAServiceStatus getServiceStatus() throws IOException { - throw new AccessControlException("Access denied"); - } - }; - DummyService svc2 = new DummyService(HAServiceState.STANDBY) { - @Override - public HAServiceStatus getServiceStatus() throws IOException { - throw new AccessControlException("Access denied"); - } - }; - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + Mockito.doThrow(new AccessControlException("Access denied")) + .when(svc1.proxy).getServiceStatus(); + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + Mockito.doThrow(new AccessControlException("Access denied")) + .when(svc2.proxy).getServiceStatus(); + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); fail("Can't failover when access is denied"); } catch (FailoverFailedException ffe) { assertTrue(ffe.getCause().getMessage().contains("Access denied")); @@ -151,19 +112,13 @@ public class TestFailoverController { @Test public void testFailoverToUnreadyService() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE); - DummyService svc2 = new DummyService(HAServiceState.STANDBY) { - @Override - public HAServiceStatus getServiceStatus() throws IOException { - HAServiceStatus ret = new HAServiceStatus(HAServiceState.STANDBY); - ret.setNotReadyToBecomeActive("injected not ready"); - return ret; - } - }; - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + Mockito.doReturn(STATE_NOT_READY).when(svc2.proxy).getServiceStatus(); + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); fail("Can't failover to a service that's not ready"); } catch (FailoverFailedException ffe) { // Expected @@ -172,95 +127,88 @@ public class TestFailoverController { } } - assertEquals(HAServiceState.ACTIVE, svc1.getServiceState()); - assertEquals(HAServiceState.STANDBY, svc2.getServiceState()); + assertEquals(HAServiceState.ACTIVE, svc1.state); + assertEquals(HAServiceState.STANDBY, svc2.state); // Forcing it means we ignore readyToBecomeActive - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, true); - assertEquals(HAServiceState.STANDBY, svc1.getServiceState()); - assertEquals(HAServiceState.ACTIVE, svc2.getServiceState()); + FailoverController.failover(svc1, svc2, false, true); + assertEquals(HAServiceState.STANDBY, svc1.state); + assertEquals(HAServiceState.ACTIVE, svc2.state); } @Test public void testFailoverToUnhealthyServiceFailsAndFailsback() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE); - DummyService svc2 = new DummyService(HAServiceState.STANDBY) { - @Override - public void monitorHealth() throws HealthCheckFailedException { - throw new HealthCheckFailedException("Failed!"); - } - }; - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + Mockito.doThrow(new HealthCheckFailedException("Failed!")) + .when(svc2.proxy).monitorHealth(); + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); fail("Failover to unhealthy service"); } catch (FailoverFailedException ffe) { // Expected } - assertEquals(HAServiceState.ACTIVE, svc1.getServiceState()); - assertEquals(HAServiceState.STANDBY, svc2.getServiceState()); + assertEquals(HAServiceState.ACTIVE, svc1.state); + assertEquals(HAServiceState.STANDBY, svc2.state); } @Test public void testFailoverFromFaultyServiceSucceeds() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE) { - @Override - public void transitionToStandby() throws ServiceFailedException { - throw new ServiceFailedException("Failed!"); - } - }; - DummyService svc2 = new DummyService(HAServiceState.STANDBY); - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + Mockito.doThrow(new ServiceFailedException("Failed!")) + .when(svc1.proxy).transitionToStandby(); + + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); AlwaysSucceedFencer.fenceCalled = 0; try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); } catch (FailoverFailedException ffe) { fail("Faulty active prevented failover"); } // svc1 still thinks it's active, that's OK, it was fenced assertEquals(1, AlwaysSucceedFencer.fenceCalled); - assertEquals("svc1:1234", AlwaysSucceedFencer.fencedSvc); - assertEquals(HAServiceState.ACTIVE, svc1.getServiceState()); - assertEquals(HAServiceState.ACTIVE, svc2.getServiceState()); + assertSame(svc1, AlwaysSucceedFencer.fencedSvc); + assertEquals(HAServiceState.ACTIVE, svc1.state); + assertEquals(HAServiceState.ACTIVE, svc2.state); } @Test public void testFailoverFromFaultyServiceFencingFailure() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE) { - @Override - public void transitionToStandby() throws ServiceFailedException { - throw new ServiceFailedException("Failed!"); - } - }; - DummyService svc2 = new DummyService(HAServiceState.STANDBY); - NodeFencer fencer = setupFencer(AlwaysFailFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + Mockito.doThrow(new ServiceFailedException("Failed!")) + .when(svc1.proxy).transitionToStandby(); + + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName()); AlwaysFailFencer.fenceCalled = 0; try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); fail("Failed over even though fencing failed"); } catch (FailoverFailedException ffe) { // Expected } assertEquals(1, AlwaysFailFencer.fenceCalled); - assertEquals("svc1:1234", AlwaysFailFencer.fencedSvc); - assertEquals(HAServiceState.ACTIVE, svc1.getServiceState()); - assertEquals(HAServiceState.STANDBY, svc2.getServiceState()); + assertSame(svc1, AlwaysFailFencer.fencedSvc); + assertEquals(HAServiceState.ACTIVE, svc1.state); + assertEquals(HAServiceState.STANDBY, svc2.state); } @Test public void testFencingFailureDuringFailover() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE); - DummyService svc2 = new DummyService(HAServiceState.STANDBY); - NodeFencer fencer = setupFencer(AlwaysFailFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName()); AlwaysFailFencer.fenceCalled = 0; try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, true, false); + FailoverController.failover(svc1, svc2, true, false); fail("Failed over even though fencing requested and failed"); } catch (FailoverFailedException ffe) { // Expected @@ -269,90 +217,83 @@ public class TestFailoverController { // If fencing was requested and it failed we don't try to make // svc2 active anyway, and we don't failback to svc1. assertEquals(1, AlwaysFailFencer.fenceCalled); - assertEquals("svc1:1234", AlwaysFailFencer.fencedSvc); - assertEquals(HAServiceState.STANDBY, svc1.getServiceState()); - assertEquals(HAServiceState.STANDBY, svc2.getServiceState()); + assertSame(svc1, AlwaysFailFencer.fencedSvc); + assertEquals(HAServiceState.STANDBY, svc1.state); + assertEquals(HAServiceState.STANDBY, svc2.state); } - private HAServiceProtocol getProtocol(String target) - throws IOException { - InetSocketAddress addr = NetUtils.createSocketAddr(target); - Configuration conf = new Configuration(); - // Lower the timeout so we quickly fail to connect - conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1); - return new HAServiceProtocolClientSideTranslatorPB(addr, conf); - } - @Test public void testFailoverFromNonExistantServiceWithFencer() throws Exception { - HAServiceProtocol svc1 = getProtocol("localhost:1234"); - DummyService svc2 = new DummyService(HAServiceState.STANDBY); - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = spy(new DummyHAService(null, svc1Addr)); + // Getting a proxy to a dead server will throw IOException on call, + // not on creation of the proxy. + HAServiceProtocol errorThrowingProxy = Mockito.mock(HAServiceProtocol.class, + new ThrowsException(new IOException("Could not connect to host"))); + Mockito.doReturn(errorThrowingProxy).when(svc1).getProxy(); + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); } catch (FailoverFailedException ffe) { fail("Non-existant active prevented failover"); } // Don't check svc1 because we can't reach it, but that's OK, it's been fenced. - assertEquals(HAServiceState.ACTIVE, svc2.getServiceState()); + assertEquals(HAServiceState.ACTIVE, svc2.state); } @Test public void testFailoverToNonExistantServiceFails() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE); - HAServiceProtocol svc2 = getProtocol("localhost:1234"); - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + DummyHAService svc2 = spy(new DummyHAService(null, svc2Addr)); + Mockito.doThrow(new IOException("Failed to connect")) + .when(svc2).getProxy(Mockito.any(), + Mockito.anyInt()); + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); fail("Failed over to a non-existant standby"); } catch (FailoverFailedException ffe) { // Expected } - assertEquals(HAServiceState.ACTIVE, svc1.getServiceState()); + assertEquals(HAServiceState.ACTIVE, svc1.state); } @Test public void testFailoverToFaultyServiceFailsbackOK() throws Exception { - DummyService svc1 = spy(new DummyService(HAServiceState.ACTIVE)); - DummyService svc2 = new DummyService(HAServiceState.STANDBY) { - @Override - public void transitionToActive() throws ServiceFailedException { - throw new ServiceFailedException("Failed!"); - } - }; - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = spy(new DummyHAService(HAServiceState.ACTIVE, svc1Addr)); + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + Mockito.doThrow(new ServiceFailedException("Failed!")) + .when(svc2.proxy).transitionToActive(); + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); fail("Failover to already active service"); } catch (FailoverFailedException ffe) { // Expected } // svc1 went standby then back to active - verify(svc1).transitionToStandby(); - verify(svc1).transitionToActive(); - assertEquals(HAServiceState.ACTIVE, svc1.getServiceState()); - assertEquals(HAServiceState.STANDBY, svc2.getServiceState()); + verify(svc1.proxy).transitionToStandby(); + verify(svc1.proxy).transitionToActive(); + assertEquals(HAServiceState.ACTIVE, svc1.state); + assertEquals(HAServiceState.STANDBY, svc2.state); } @Test public void testWeDontFailbackIfActiveWasFenced() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE); - DummyService svc2 = new DummyService(HAServiceState.STANDBY) { - @Override - public void transitionToActive() throws ServiceFailedException { - throw new ServiceFailedException("Failed!"); - } - }; - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + Mockito.doThrow(new ServiceFailedException("Failed!")) + .when(svc2.proxy).transitionToActive(); + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, true, false); + FailoverController.failover(svc1, svc2, true, false); fail("Failed over to service that won't transition to active"); } catch (FailoverFailedException ffe) { // Expected @@ -360,24 +301,21 @@ public class TestFailoverController { // We failed to failover and did not failback because we fenced // svc1 (we forced it), therefore svc1 and svc2 should be standby. - assertEquals(HAServiceState.STANDBY, svc1.getServiceState()); - assertEquals(HAServiceState.STANDBY, svc2.getServiceState()); + assertEquals(HAServiceState.STANDBY, svc1.state); + assertEquals(HAServiceState.STANDBY, svc2.state); } @Test public void testWeFenceOnFailbackIfTransitionToActiveFails() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE); - DummyService svc2 = new DummyService(HAServiceState.STANDBY) { - @Override - public void transitionToActive() throws ServiceFailedException, IOException { - throw new IOException("Failed!"); - } - }; - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + Mockito.doThrow(new ServiceFailedException("Failed!")) + .when(svc2.proxy).transitionToActive(); + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); AlwaysSucceedFencer.fenceCalled = 0; try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); fail("Failed over to service that won't transition to active"); } catch (FailoverFailedException ffe) { // Expected @@ -386,25 +324,22 @@ public class TestFailoverController { // We failed to failover. We did not fence svc1 because it cooperated // and we didn't force it, so we failed back to svc1 and fenced svc2. // Note svc2 still thinks it's active, that's OK, we fenced it. - assertEquals(HAServiceState.ACTIVE, svc1.getServiceState()); + assertEquals(HAServiceState.ACTIVE, svc1.state); assertEquals(1, AlwaysSucceedFencer.fenceCalled); - assertEquals("svc2:5678", AlwaysSucceedFencer.fencedSvc); + assertSame(svc2, AlwaysSucceedFencer.fencedSvc); } @Test public void testFailureToFenceOnFailbackFailsTheFailback() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE); - DummyService svc2 = new DummyService(HAServiceState.STANDBY) { - @Override - public void transitionToActive() throws ServiceFailedException, IOException { - throw new IOException("Failed!"); - } - }; - NodeFencer fencer = setupFencer(AlwaysFailFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + Mockito.doThrow(new IOException("Failed!")) + .when(svc2.proxy).transitionToActive(); + svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName()); AlwaysFailFencer.fenceCalled = 0; try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); fail("Failed over to service that won't transition to active"); } catch (FailoverFailedException ffe) { // Expected @@ -413,35 +348,30 @@ public class TestFailoverController { // We did not fence svc1 because it cooperated and we didn't force it, // we failed to failover so we fenced svc2, we failed to fence svc2 // so we did not failback to svc1, ie it's still standby. - assertEquals(HAServiceState.STANDBY, svc1.getServiceState()); + assertEquals(HAServiceState.STANDBY, svc1.state); assertEquals(1, AlwaysFailFencer.fenceCalled); - assertEquals("svc2:5678", AlwaysFailFencer.fencedSvc); + assertSame(svc2, AlwaysFailFencer.fencedSvc); } @Test public void testFailbackToFaultyServiceFails() throws Exception { - DummyService svc1 = new DummyService(HAServiceState.ACTIVE) { - @Override - public void transitionToActive() throws ServiceFailedException { - throw new ServiceFailedException("Failed!"); - } - }; - DummyService svc2 = new DummyService(HAServiceState.STANDBY) { - @Override - public void transitionToActive() throws ServiceFailedException { - throw new ServiceFailedException("Failed!"); - } - }; - NodeFencer fencer = setupFencer(AlwaysSucceedFencer.class.getName()); + DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr); + Mockito.doThrow(new ServiceFailedException("Failed!")) + .when(svc1.proxy).transitionToActive(); + DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr); + Mockito.doThrow(new ServiceFailedException("Failed!")) + .when(svc2.proxy).transitionToActive(); + + svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName()); try { - FailoverController.failover(svc1, svc1Addr, svc2, svc2Addr, fencer, false, false); + FailoverController.failover(svc1, svc2, false, false); fail("Failover to already active service"); } catch (FailoverFailedException ffe) { // Expected } - assertEquals(HAServiceState.STANDBY, svc1.getServiceState()); - assertEquals(HAServiceState.STANDBY, svc2.getServiceState()); + assertEquals(HAServiceState.STANDBY, svc1.state); + assertEquals(HAServiceState.STANDBY, svc2.state); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java index 7f885d8bc25..a3cac1b96f0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAAdmin.java @@ -22,14 +22,15 @@ import static org.junit.Assert.*; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; +import java.net.InetSocketAddress; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.Log; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; import com.google.common.base.Charsets; import com.google.common.base.Joiner; @@ -40,15 +41,14 @@ public class TestHAAdmin { private HAAdmin tool; private ByteArrayOutputStream errOutBytes = new ByteArrayOutputStream(); private String errOutput; - private HAServiceProtocol mockProtocol; - + @Before public void setup() throws IOException { - mockProtocol = Mockito.mock(HAServiceProtocol.class); tool = new HAAdmin() { @Override - protected HAServiceProtocol getProtocol(String target) throws IOException { - return mockProtocol; + protected HAServiceTarget resolveTarget(String target) { + return new DummyHAService(HAServiceState.STANDBY, + new InetSocketAddress("dummy", 12345)); } }; tool.setConf(new Configuration()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java index 5508547c0a5..853a0e34464 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java @@ -26,26 +26,35 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import com.google.common.collect.Lists; public class TestNodeFencer { + private HAServiceTarget MOCK_TARGET; + + @Before public void clearMockState() { AlwaysSucceedFencer.fenceCalled = 0; AlwaysSucceedFencer.callArgs.clear(); AlwaysFailFencer.fenceCalled = 0; AlwaysFailFencer.callArgs.clear(); + + MOCK_TARGET = Mockito.mock(HAServiceTarget.class); + Mockito.doReturn("my mock").when(MOCK_TARGET).toString(); + Mockito.doReturn(new InetSocketAddress("host", 1234)) + .when(MOCK_TARGET).getAddress(); } @Test public void testSingleFencer() throws BadFencingConfigurationException { NodeFencer fencer = setupFencer( AlwaysSucceedFencer.class.getName() + "(foo)"); - assertTrue(fencer.fence(new InetSocketAddress("host", 1234))); + assertTrue(fencer.fence(MOCK_TARGET)); assertEquals(1, AlwaysSucceedFencer.fenceCalled); - assertEquals("host:1234", AlwaysSucceedFencer.fencedSvc); + assertSame(MOCK_TARGET, AlwaysSucceedFencer.fencedSvc); assertEquals("foo", AlwaysSucceedFencer.callArgs.get(0)); } @@ -54,7 +63,7 @@ public class TestNodeFencer { NodeFencer fencer = setupFencer( AlwaysSucceedFencer.class.getName() + "(foo)\n" + AlwaysSucceedFencer.class.getName() + "(bar)\n"); - assertTrue(fencer.fence(new InetSocketAddress("host", 1234))); + assertTrue(fencer.fence(MOCK_TARGET)); // Only one call, since the first fencer succeeds assertEquals(1, AlwaysSucceedFencer.fenceCalled); assertEquals("foo", AlwaysSucceedFencer.callArgs.get(0)); @@ -68,12 +77,12 @@ public class TestNodeFencer { " # the next one will always fail\n" + " " + AlwaysFailFencer.class.getName() + "(foo) # <- fails\n" + AlwaysSucceedFencer.class.getName() + "(bar) \n"); - assertTrue(fencer.fence(new InetSocketAddress("host", 1234))); + assertTrue(fencer.fence(MOCK_TARGET)); // One call to each, since top fencer fails assertEquals(1, AlwaysFailFencer.fenceCalled); - assertEquals("host:1234", AlwaysFailFencer.fencedSvc); + assertSame(MOCK_TARGET, AlwaysFailFencer.fencedSvc); assertEquals(1, AlwaysSucceedFencer.fenceCalled); - assertEquals("host:1234", AlwaysSucceedFencer.fencedSvc); + assertSame(MOCK_TARGET, AlwaysSucceedFencer.fencedSvc); assertEquals("foo", AlwaysFailFencer.callArgs.get(0)); assertEquals("bar", AlwaysSucceedFencer.callArgs.get(0)); } @@ -82,41 +91,41 @@ public class TestNodeFencer { public void testArglessFencer() throws BadFencingConfigurationException { NodeFencer fencer = setupFencer( AlwaysSucceedFencer.class.getName()); - assertTrue(fencer.fence(new InetSocketAddress("host", 1234))); + assertTrue(fencer.fence(MOCK_TARGET)); // One call to each, since top fencer fails assertEquals(1, AlwaysSucceedFencer.fenceCalled); - assertEquals("host:1234", AlwaysSucceedFencer.fencedSvc); + assertSame(MOCK_TARGET, AlwaysSucceedFencer.fencedSvc); assertEquals(null, AlwaysSucceedFencer.callArgs.get(0)); } @Test public void testShortNameShell() throws BadFencingConfigurationException { NodeFencer fencer = setupFencer("shell(true)"); - assertTrue(fencer.fence(new InetSocketAddress("host", 1234))); + assertTrue(fencer.fence(MOCK_TARGET)); } @Test public void testShortNameSsh() throws BadFencingConfigurationException { NodeFencer fencer = setupFencer("sshfence"); - assertFalse(fencer.fence(new InetSocketAddress("host", 1234))); + assertFalse(fencer.fence(MOCK_TARGET)); } @Test public void testShortNameSshWithUser() throws BadFencingConfigurationException { NodeFencer fencer = setupFencer("sshfence(user)"); - assertFalse(fencer.fence(new InetSocketAddress("host", 1234))); + assertFalse(fencer.fence(MOCK_TARGET)); } @Test public void testShortNameSshWithPort() throws BadFencingConfigurationException { NodeFencer fencer = setupFencer("sshfence(:123)"); - assertFalse(fencer.fence(new InetSocketAddress("host", 1234))); + assertFalse(fencer.fence(MOCK_TARGET)); } @Test public void testShortNameSshWithUserPort() throws BadFencingConfigurationException { NodeFencer fencer = setupFencer("sshfence(user:123)"); - assertFalse(fencer.fence(new InetSocketAddress("host", 1234))); + assertFalse(fencer.fence(MOCK_TARGET)); } public static NodeFencer setupFencer(String confStr) @@ -133,12 +142,12 @@ public class TestNodeFencer { public static class AlwaysSucceedFencer extends Configured implements FenceMethod { static int fenceCalled = 0; - static String fencedSvc; + static HAServiceTarget fencedSvc; static List callArgs = Lists.newArrayList(); @Override - public boolean tryFence(InetSocketAddress serviceAddr, String args) { - fencedSvc = serviceAddr.getHostName() + ":" + serviceAddr.getPort(); + public boolean tryFence(HAServiceTarget target, String args) { + fencedSvc = target; callArgs.add(args); fenceCalled++; return true; @@ -155,12 +164,12 @@ public class TestNodeFencer { public static class AlwaysFailFencer extends Configured implements FenceMethod { static int fenceCalled = 0; - static String fencedSvc; + static HAServiceTarget fencedSvc; static List callArgs = Lists.newArrayList(); @Override - public boolean tryFence(InetSocketAddress serviceAddr, String args) { - fencedSvc = serviceAddr.getHostName() + ":" + serviceAddr.getPort(); + public boolean tryFence(HAServiceTarget target, String args) { + fencedSvc = target; callArgs.add(args); fenceCalled++; return false; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java index 49bae039ecc..e95ba59af27 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java @@ -22,6 +22,7 @@ import static org.junit.Assert.*; import java.net.InetSocketAddress; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.util.StringUtils; import org.junit.Before; import org.junit.BeforeClass; @@ -32,6 +33,9 @@ import static org.mockito.Mockito.spy; public class TestShellCommandFencer { private ShellCommandFencer fencer = createFencer(); + private static final HAServiceTarget TEST_TARGET = + new DummyHAService(HAServiceState.ACTIVE, + new InetSocketAddress("host", 1234)); @BeforeClass public static void setupLogSpy() { @@ -57,11 +61,10 @@ public class TestShellCommandFencer { */ @Test public void testBasicSuccessFailure() { - InetSocketAddress addr = new InetSocketAddress("host", 1234); - assertTrue(fencer.tryFence(addr, "echo")); - assertFalse(fencer.tryFence(addr, "exit 1")); + assertTrue(fencer.tryFence(TEST_TARGET, "echo")); + assertFalse(fencer.tryFence(TEST_TARGET, "exit 1")); // bad path should also fail - assertFalse(fencer.tryFence(addr, "xxxxxxxxxxxx")); + assertFalse(fencer.tryFence(TEST_TARGET, "xxxxxxxxxxxx")); } @Test @@ -98,8 +101,7 @@ public class TestShellCommandFencer { */ @Test public void testStdoutLogging() { - InetSocketAddress addr = new InetSocketAddress("host", 1234); - assertTrue(fencer.tryFence(addr, "echo hello")); + assertTrue(fencer.tryFence(TEST_TARGET, "echo hello")); Mockito.verify(ShellCommandFencer.LOG).info( Mockito.endsWith("echo hello: host:1234 hello")); } @@ -110,8 +112,7 @@ public class TestShellCommandFencer { */ @Test public void testStderrLogging() { - InetSocketAddress addr = new InetSocketAddress("host", 1234); - assertTrue(fencer.tryFence(addr, "echo hello >&2")); + assertTrue(fencer.tryFence(TEST_TARGET, "echo hello >&2")); Mockito.verify(ShellCommandFencer.LOG).warn( Mockito.endsWith("echo hello >&2: host:1234 hello")); } @@ -122,8 +123,7 @@ public class TestShellCommandFencer { */ @Test public void testConfAsEnvironment() { - InetSocketAddress addr = new InetSocketAddress("host", 1234); - fencer.tryFence(addr, "echo $in_fencing_tests"); + fencer.tryFence(TEST_TARGET, "echo $in_fencing_tests"); Mockito.verify(ShellCommandFencer.LOG).info( Mockito.endsWith("echo $in...ing_tests: host:1234 yessir")); } @@ -136,8 +136,7 @@ public class TestShellCommandFencer { */ @Test(timeout=10000) public void testSubprocessInputIsClosed() { - InetSocketAddress addr = new InetSocketAddress("host", 1234); - assertFalse(fencer.tryFence(addr, "read")); + assertFalse(fencer.tryFence(TEST_TARGET, "read")); } @Test diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestSshFenceByTcpPort.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestSshFenceByTcpPort.java index 554a7abca5f..4796fe6d081 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestSshFenceByTcpPort.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestSshFenceByTcpPort.java @@ -23,6 +23,7 @@ import java.net.InetSocketAddress; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.SshFenceByTcpPort.Args; import org.apache.log4j.Level; import org.junit.Assume; @@ -34,12 +35,25 @@ public class TestSshFenceByTcpPort { ((Log4JLogger)SshFenceByTcpPort.LOG).getLogger().setLevel(Level.ALL); } - private String TEST_FENCING_HOST = System.getProperty( + private static String TEST_FENCING_HOST = System.getProperty( "test.TestSshFenceByTcpPort.host", "localhost"); - private String TEST_FENCING_PORT = System.getProperty( + private static final String TEST_FENCING_PORT = System.getProperty( "test.TestSshFenceByTcpPort.port", "8020"); - private final String TEST_KEYFILE = System.getProperty( + private static final String TEST_KEYFILE = System.getProperty( "test.TestSshFenceByTcpPort.key"); + + private static final InetSocketAddress TEST_ADDR = + new InetSocketAddress(TEST_FENCING_HOST, + Integer.valueOf(TEST_FENCING_PORT)); + private static final HAServiceTarget TEST_TARGET = + new DummyHAService(HAServiceState.ACTIVE, TEST_ADDR); + + /** + * Connect to Google's DNS server - not running ssh! + */ + private static final HAServiceTarget UNFENCEABLE_TARGET = + new DummyHAService(HAServiceState.ACTIVE, + new InetSocketAddress("8.8.8.8", 1234)); @Test(timeout=20000) public void testFence() throws BadFencingConfigurationException { @@ -49,8 +63,7 @@ public class TestSshFenceByTcpPort { SshFenceByTcpPort fence = new SshFenceByTcpPort(); fence.setConf(conf); assertTrue(fence.tryFence( - new InetSocketAddress(TEST_FENCING_HOST, - Integer.valueOf(TEST_FENCING_PORT)), + TEST_TARGET, null)); } @@ -65,8 +78,7 @@ public class TestSshFenceByTcpPort { conf.setInt(SshFenceByTcpPort.CONF_CONNECT_TIMEOUT_KEY, 3000); SshFenceByTcpPort fence = new SshFenceByTcpPort(); fence.setConf(conf); - // Connect to Google's DNS server - not running ssh! - assertFalse(fence.tryFence(new InetSocketAddress("8.8.8.8", 1234), "")); + assertFalse(fence.tryFence(UNFENCEABLE_TARGET, "")); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java index 13bde2ae533..4db5a86b93d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java @@ -25,8 +25,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.ha.HAAdmin; +import org.apache.hadoop.ha.HAServiceTarget; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.util.ToolRunner; @@ -65,15 +65,9 @@ public class DFSHAAdmin extends HAAdmin { * Try to map the given namenode ID to its service address. */ @Override - protected String getServiceAddr(String nnId) { + protected HAServiceTarget resolveTarget(String nnId) { HdfsConfiguration conf = (HdfsConfiguration)getConf(); - String serviceAddr = - DFSUtil.getNamenodeServiceAddr(conf, nameserviceId, nnId); - if (serviceAddr == null) { - throw new IllegalArgumentException( - "Unable to determine service address for namenode '" + nnId + "'"); - } - return serviceAddr; + return new NNHAServiceTarget(conf, nameserviceId, nnId); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java new file mode 100644 index 00000000000..9e8c239e7e8 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools; + +import java.net.InetSocketAddress; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.ha.BadFencingConfigurationException; +import org.apache.hadoop.ha.HAServiceTarget; +import org.apache.hadoop.ha.NodeFencer; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.net.NetUtils; + +/** + * One of the NN NameNodes acting as the target of an administrative command + * (e.g. failover). + */ +@InterfaceAudience.Private +public class NNHAServiceTarget extends HAServiceTarget { + + private final InetSocketAddress addr; + private NodeFencer fencer; + private BadFencingConfigurationException fenceConfigError; + + public NNHAServiceTarget(HdfsConfiguration conf, + String nsId, String nnId) { + String serviceAddr = + DFSUtil.getNamenodeServiceAddr(conf, nsId, nnId); + if (serviceAddr == null) { + throw new IllegalArgumentException( + "Unable to determine service address for namenode '" + nnId + "'"); + } + this.addr = NetUtils.createSocketAddr(serviceAddr, + NameNode.DEFAULT_PORT); + try { + this.fencer = NodeFencer.create(conf); + } catch (BadFencingConfigurationException e) { + this.fenceConfigError = e; + } + } + + /** + * @return the NN's IPC address. + */ + @Override + public InetSocketAddress getAddress() { + return addr; + } + + @Override + public void checkFencingConfigured() throws BadFencingConfigurationException { + if (fenceConfigError != null) { + throw fenceConfigError; + } + } + + @Override + public NodeFencer getFencer() { + return fencer; + } + + @Override + public String toString() { + return "NameNode at " + addr; + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java index c5ba0eb7e58..50f0e0b47fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.HAServiceStatus; +import org.apache.hadoop.ha.HAServiceTarget; import org.apache.hadoop.ha.HealthCheckFailedException; import org.apache.hadoop.ha.NodeFencer; @@ -79,10 +80,18 @@ public class TestDFSHAAdmin { public void setup() throws IOException { mockProtocol = Mockito.mock(HAServiceProtocol.class); tool = new DFSHAAdmin() { + @Override - protected HAServiceProtocol getProtocol(String serviceId) throws IOException { - getServiceAddr(serviceId); - return mockProtocol; + protected HAServiceTarget resolveTarget(String nnId) { + HAServiceTarget target = super.resolveTarget(nnId); + HAServiceTarget spy = Mockito.spy(target); + // OVerride the target to return our mock protocol + try { + Mockito.doReturn(mockProtocol).when(spy).getProxy(); + } catch (IOException e) { + throw new AssertionError(e); // mock setup doesn't really throw + } + return spy; } }; tool.setConf(getHAConf());