From 0600e834be8a148c8727ce4520e48b7f3235a738 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Thu, 17 May 2012 00:36:13 +0000 Subject: [PATCH] HDFS-3432. TestDFSZKFailoverController tries to fail over too early. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-3042@1339434 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/ha/HealthMonitor.java | 7 ++-- .../org/apache/hadoop/ha/MiniZKFCCluster.java | 6 +--- .../org/apache/hadoop/ha/ZKFCTestUtil.java | 34 +++++++++++++++++++ .../hadoop-hdfs/CHANGES.HDFS-3042.txt | 2 ++ .../ha/TestDFSZKFailoverController.java | 14 ++++++-- 5 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ZKFCTestUtil.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java index 753352945d4..a349626ece9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -43,7 +44,8 @@ import com.google.common.base.Preconditions; * Classes which need callbacks should implement the {@link Callback} * interface. */ -class HealthMonitor { +@InterfaceAudience.Private +public class HealthMonitor { private static final Log LOG = LogFactory.getLog( HealthMonitor.class); @@ -75,7 +77,8 @@ class HealthMonitor { private HAServiceStatus lastServiceState = new HAServiceStatus( HAServiceState.INITIALIZING); - enum State { + @InterfaceAudience.Private + public enum State { /** * The health monitor is still starting up. */ diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java index 76d273552dd..1db792453a2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java @@ -170,11 +170,7 @@ public class MiniZKFCCluster { */ public void waitForHealthState(int idx, State state) throws Exception { - ZKFailoverController zkfc = thrs[idx].zkfc; - while (zkfc.getLastHealthState() != state) { - ctx.checkException(); - Thread.sleep(50); - } + ZKFCTestUtil.waitForHealthState(thrs[idx].zkfc, state, ctx); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ZKFCTestUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ZKFCTestUtil.java new file mode 100644 index 00000000000..4a5eacd4baa --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ZKFCTestUtil.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ha; + +import org.apache.hadoop.test.MultithreadedTestUtil; + +public class ZKFCTestUtil { + + public static void waitForHealthState(ZKFailoverController zkfc, + HealthMonitor.State state, + MultithreadedTestUtil.TestContext ctx) throws Exception { + while (zkfc.getLastHealthState() != state) { + if (ctx != null) { + ctx.checkException(); + } + Thread.sleep(50); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3042.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3042.txt index 678766cbdd0..394064f36e5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3042.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3042.txt @@ -15,3 +15,5 @@ HDFS-3261. TestHASafeMode fails on HDFS-3042 branch (todd) HDFS-3159. Document NN auto-failover setup and configuration (todd) HDFS-3412. Fix findbugs warnings in auto-HA branch (todd) + +HDFS-3432. TestDFSZKFailoverController tries to fail over too early (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java index d1d1991218a..52e136940de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java @@ -26,6 +26,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.ha.ClientBaseWithFixes; +import org.apache.hadoop.ha.HealthMonitor; +import org.apache.hadoop.ha.ZKFCTestUtil; import org.apache.hadoop.ha.ZKFailoverController; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.TestNodeFencer.AlwaysSucceedFencer; @@ -91,6 +93,12 @@ public class TestDFSZKFailoverController extends ClientBaseWithFixes { ctx.addThread(thr2 = new ZKFCThread(ctx, 1)); thr2.start(); + // Wait for the ZKFCs to fully start up + ZKFCTestUtil.waitForHealthState(thr1.zkfc, + HealthMonitor.State.SERVICE_HEALTHY, ctx); + ZKFCTestUtil.waitForHealthState(thr2.zkfc, + HealthMonitor.State.SERVICE_HEALTHY, ctx); + fs = HATestUtil.configureFailoverFs(cluster, conf); } @@ -160,10 +168,12 @@ public class TestDFSZKFailoverController extends ClientBaseWithFixes { public void testManualFailoverWithDFSHAAdmin() throws Exception { DFSHAAdmin tool = new DFSHAAdmin(); tool.setConf(conf); - tool.run(new String[]{"-failover", "nn1", "nn2"}); + assertEquals(0, + tool.run(new String[]{"-failover", "nn1", "nn2"})); waitForHAState(0, HAServiceState.STANDBY); waitForHAState(1, HAServiceState.ACTIVE); - tool.run(new String[]{"-failover", "nn2", "nn1"}); + assertEquals(0, + tool.run(new String[]{"-failover", "nn2", "nn1"})); waitForHAState(0, HAServiceState.ACTIVE); waitForHAState(1, HAServiceState.STANDBY); }