From 16a25b74dbd1ee9f0a6c8c9d965f729d2be57976 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Sat, 27 Jun 2020 10:36:07 +0800 Subject: [PATCH] HBASE-24635 Split TestMetaWithReplicas (#1980) Signed-off-by: Guanghao Zhang --- .../hbase/IntegrationTestMetaReplicas.java | 4 +- .../client/MetaWithReplicasTestBase.java | 123 +++++ .../TestFailedMetaReplicaAssigment.java | 135 ++++++ .../client/TestMetaReplicasAddressChange.java | 92 ++++ .../hbase/client/TestMetaWithReplicas.java | 428 ------------------ .../client/TestMetaWithReplicasBasic.java | 91 ++++ .../TestMetaWithReplicasShutdownHandling.java | 164 +++++++ .../TestShutdownOfMetaReplicaHolder.java | 67 +++ 8 files changed, 674 insertions(+), 430 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/MetaWithReplicasTestBase.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaReplicasAddressChange.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasBasic.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasShutdownHandling.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestShutdownOfMetaReplicaHolder.java diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMetaReplicas.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMetaReplicas.java index 6146fd9404a..d906bfd8420 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMetaReplicas.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMetaReplicas.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hbase; import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.TestMetaWithReplicas; +import org.apache.hadoop.hbase.client.TestMetaWithReplicasShutdownHandling; import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore; import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.zookeeper.ZKUtil; @@ -96,7 +96,7 @@ public class IntegrationTestMetaReplicas { // server holding the primary meta replica. Then it does a put/get into/from // the test table. The put/get operations would use the replicas to locate the // location of the test table's region - TestMetaWithReplicas.shutdownMetaAndDoValidations(util); + TestMetaWithReplicasShutdownHandling.shutdownMetaAndDoValidations(util); } public static void main(String[] args) throws Exception { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/MetaWithReplicasTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/MetaWithReplicasTestBase.java new file mode 100644 index 00000000000..78e3e541e89 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/MetaWithReplicasTestBase.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.StartMiniClusterOption; +import org.apache.hadoop.hbase.TableNameTestRule; +import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil; +import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore; +import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +import org.junit.AfterClass; +import org.junit.Rule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base class for testing the scenarios where replicas are enabled for the meta table. + */ +public class MetaWithReplicasTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(MetaWithReplicasTestBase.class); + + protected static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + protected static final int REGIONSERVERS_COUNT = 3; + + @Rule + public TableNameTestRule name = new TableNameTestRule(); + + protected static void startCluster() throws Exception { + TEST_UTIL.getConfiguration().setInt("zookeeper.session.timeout", 30000); + TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, 3); + TEST_UTIL.getConfiguration() + .setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 1000); + StartMiniClusterOption option = StartMiniClusterOption.builder().numAlwaysStandByMasters(1) + .numMasters(1).numRegionServers(REGIONSERVERS_COUNT).build(); + TEST_UTIL.startMiniCluster(option); + AssignmentManager am = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager(); + Set sns = new HashSet(); + ServerName hbaseMetaServerName = + MetaTableLocator.getMetaRegionLocation(TEST_UTIL.getZooKeeperWatcher()); + LOG.info("HBASE:META DEPLOY: on " + hbaseMetaServerName); + sns.add(hbaseMetaServerName); + for (int replicaId = 1; replicaId < 3; replicaId++) { + RegionInfo h = RegionReplicaUtil + .getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId); + AssignmentTestingUtil.waitForAssignment(am, h); + ServerName sn = am.getRegionStates().getRegionServerOfRegion(h); + assertNotNull(sn); + LOG.info("HBASE:META DEPLOY: " + h.getRegionNameAsString() + " on " + sn); + sns.add(sn); + } + // Fun. All meta region replicas have ended up on the one server. This will cause this test + // to fail ... sometimes. + if (sns.size() == 1) { + int count = TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size(); + assertTrue("count=" + count, count == REGIONSERVERS_COUNT); + LOG.warn("All hbase:meta replicas are on the one server; moving hbase:meta: " + sns); + int metaServerIndex = TEST_UTIL.getHBaseCluster().getServerWithMeta(); + int newServerIndex = metaServerIndex; + while (newServerIndex == metaServerIndex) { + newServerIndex = (newServerIndex + 1) % REGIONSERVERS_COUNT; + } + assertNotEquals(metaServerIndex, newServerIndex); + ServerName destinationServerName = + TEST_UTIL.getHBaseCluster().getRegionServer(newServerIndex).getServerName(); + ServerName metaServerName = + TEST_UTIL.getHBaseCluster().getRegionServer(metaServerIndex).getServerName(); + assertNotEquals(destinationServerName, metaServerName); + TEST_UTIL.getAdmin().move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), + destinationServerName); + } + // Disable the balancer + LoadBalancerTracker l = + new LoadBalancerTracker(TEST_UTIL.getZooKeeperWatcher(), new Abortable() { + AtomicBoolean aborted = new AtomicBoolean(false); + + @Override + public boolean isAborted() { + return aborted.get(); + } + + @Override + public void abort(String why, Throwable e) { + aborted.set(true); + } + }); + l.setBalancerOn(false); + LOG.debug("All meta replicas assigned"); + } + + @AfterClass + public static void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java new file mode 100644 index 00000000000..0c26d7934ff --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.concurrent.Future; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.StartMiniClusterOption; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.assignment.RegionStateNode; +import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MiscTests.class, MediumTests.class }) +public class TestFailedMetaReplicaAssigment { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestFailedMetaReplicaAssigment.class); + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setUp() throws Exception { + // using our rigged master, to force a failed meta replica assignment + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setInt(HConstants.META_REPLICAS_NUM, 3); + StartMiniClusterOption option = StartMiniClusterOption.builder().numAlwaysStandByMasters(1) + .numMasters(1).numRegionServers(1).masterClass(BrokenMetaReplicaMaster.class).build(); + TEST_UTIL.startMiniCluster(option); + } + + @AfterClass + public static void tearDown() throws IOException { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testFailedReplicaAssignment() throws InterruptedException { + HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); + // waiting for master to come up + TEST_UTIL.waitFor(30000, () -> master.isInitialized()); + + AssignmentManager am = master.getAssignmentManager(); + // showing one of the replicas got assigned + RegionInfo metaReplicaHri = + RegionReplicaUtil.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, 1); + // we use assignAsync so we need to wait a bit + TEST_UTIL.waitFor(30000, () -> { + RegionStateNode metaReplicaRegionNode = + am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri); + return metaReplicaRegionNode.getRegionLocation() != null; + }); + // showing one of the replicas failed to be assigned + RegionInfo metaReplicaHri2 = + RegionReplicaUtil.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, 2); + RegionStateNode metaReplicaRegionNode2 = + am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri2); + // wait for several seconds to make sure that it is not assigned + for (int i = 0; i < 3; i++) { + Thread.sleep(2000); + assertNull(metaReplicaRegionNode2.getRegionLocation()); + } + + // showing master is active and running + assertFalse(master.isStopping()); + assertFalse(master.isStopped()); + assertTrue(master.isActiveMaster()); + } + + public static class BrokenTransitRegionStateProcedure extends TransitRegionStateProcedure { + protected BrokenTransitRegionStateProcedure() { + super(null, null, null, false, TransitionType.ASSIGN); + } + } + + public static class BrokenMetaReplicaMaster extends HMaster { + public BrokenMetaReplicaMaster(final Configuration conf) throws IOException { + super(conf); + } + + @Override + public AssignmentManager createAssignmentManager(MasterServices master) { + return new BrokenMasterMetaAssignmentManager(master); + } + } + + public static class BrokenMasterMetaAssignmentManager extends AssignmentManager { + MasterServices master; + + public BrokenMasterMetaAssignmentManager(final MasterServices master) { + super(master); + this.master = master; + } + + public Future assignAsync(RegionInfo regionInfo, ServerName sn) throws IOException { + RegionStateNode regionNode = getRegionStates().getOrCreateRegionStateNode(regionInfo); + if (regionNode.getRegionInfo().getReplicaId() == 2) { + regionNode.setProcedure(new BrokenTransitRegionStateProcedure()); + } + return super.assignAsync(regionInfo, sn); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaReplicasAddressChange.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaReplicasAddressChange.java new file mode 100644 index 00000000000..fe105848c86 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaReplicasAddressChange.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Collection; +import java.util.EnumSet; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClusterMetrics.Option; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; +import org.apache.hadoop.hbase.zookeeper.ZNodePaths; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; + +@Category({ MiscTests.class, MediumTests.class }) +public class TestMetaReplicasAddressChange extends MetaWithReplicasTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMetaReplicasAddressChange.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestMetaReplicasAddressChange.class); + + @BeforeClass + public static void setUp() throws Exception { + startCluster(); + } + + @Test + public void testMetaAddressChange() throws Exception { + // checks that even when the meta's location changes, the various + // caches update themselves. Uses the master operations to test + // this + Configuration conf = TEST_UTIL.getConfiguration(); + ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher(); + String baseZNode = + conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); + String primaryMetaZnode = + ZNodePaths.joinZNode(baseZNode, conf.get("zookeeper.znode.metaserver", "meta-region-server")); + // check that the data in the znode is parseable (this would also mean the znode exists) + byte[] data = ZKUtil.getData(zkw, primaryMetaZnode); + ServerName currentServer = ProtobufUtil.toServerName(data); + Collection liveServers = TEST_UTIL.getAdmin() + .getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet(); + ServerName moveToServer = + liveServers.stream().filter(s -> !currentServer.equals(s)).findAny().get(); + final TableName tableName = name.getTableName(); + TEST_UTIL.createTable(tableName, "f"); + assertTrue(TEST_UTIL.getAdmin().tableExists(tableName)); + TEST_UTIL.getAdmin().move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), + moveToServer); + assertNotEquals(currentServer, moveToServer); + LOG.debug("CurrentServer={}, moveToServer={}", currentServer, moveToServer); + TEST_UTIL.waitFor(60000, () -> { + byte[] bytes = ZKUtil.getData(zkw, primaryMetaZnode); + ServerName actualServer = ProtobufUtil.toServerName(bytes); + return moveToServer.equals(actualServer); + }); + TEST_UTIL.getAdmin().disableTable(tableName); + assertTrue(TEST_UTIL.getAdmin().isTableDisabled(tableName)); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java deleted file mode 100644 index 65dc38ebea4..00000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java +++ /dev/null @@ -1,428 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Abortable; -import org.apache.hadoop.hbase.ClusterMetrics.Option; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.StartMiniClusterOption; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.master.assignment.AssignmentManager; -import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil; -import org.apache.hadoop.hbase.master.assignment.RegionStateNode; -import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; -import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; -import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; -import org.apache.hadoop.hbase.zookeeper.ZKUtil; -import org.apache.hadoop.hbase.zookeeper.ZKWatcher; -import org.apache.hadoop.hbase.zookeeper.ZNodePaths; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; - -/** - * Tests the scenarios where replicas are enabled for the meta table - */ -@Category(LargeTests.class) -public class TestMetaWithReplicas { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetaWithReplicas.class); - - private static final Logger LOG = LoggerFactory.getLogger(TestMetaWithReplicas.class); - private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static final int REGIONSERVERS_COUNT = 3; - - @Rule - public TestName name = new TestName(); - - @Before - public void setup() throws Exception { - TEST_UTIL.getConfiguration().setInt("zookeeper.session.timeout", 30000); - TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, 3); - TEST_UTIL.getConfiguration().setInt( - StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 1000); - StartMiniClusterOption option = StartMiniClusterOption.builder(). - numAlwaysStandByMasters(1).numMasters(1).numRegionServers(REGIONSERVERS_COUNT).build(); - TEST_UTIL.startMiniCluster(option); - AssignmentManager am = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager(); - Set sns = new HashSet(); - ServerName hbaseMetaServerName = - MetaTableLocator.getMetaRegionLocation(TEST_UTIL.getZooKeeperWatcher()); - LOG.info("HBASE:META DEPLOY: on " + hbaseMetaServerName); - sns.add(hbaseMetaServerName); - for (int replicaId = 1; replicaId < 3; replicaId++) { - RegionInfo h = RegionReplicaUtil - .getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId); - AssignmentTestingUtil.waitForAssignment(am, h); - ServerName sn = am.getRegionStates().getRegionServerOfRegion(h); - assertNotNull(sn); - LOG.info("HBASE:META DEPLOY: " + h.getRegionNameAsString() + " on " + sn); - sns.add(sn); - } - // Fun. All meta region replicas have ended up on the one server. This will cause this test - // to fail ... sometimes. - if (sns.size() == 1) { - int count = TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size(); - assertTrue("count=" + count, count == REGIONSERVERS_COUNT); - LOG.warn("All hbase:meta replicas are on the one server; moving hbase:meta: " + sns); - int metaServerIndex = TEST_UTIL.getHBaseCluster().getServerWithMeta(); - int newServerIndex = metaServerIndex; - while (newServerIndex == metaServerIndex) { - newServerIndex = (newServerIndex + 1) % REGIONSERVERS_COUNT; - } - assertNotEquals(metaServerIndex, newServerIndex); - ServerName destinationServerName = - TEST_UTIL.getHBaseCluster().getRegionServer(newServerIndex).getServerName(); - ServerName metaServerName = - TEST_UTIL.getHBaseCluster().getRegionServer(metaServerIndex).getServerName(); - assertNotEquals(destinationServerName, metaServerName); - TEST_UTIL.getAdmin().move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), - destinationServerName); - } - // Disable the balancer - LoadBalancerTracker l = new LoadBalancerTracker(TEST_UTIL.getZooKeeperWatcher(), - new Abortable() { - AtomicBoolean aborted = new AtomicBoolean(false); - @Override - public boolean isAborted() { - return aborted.get(); - } - @Override - public void abort(String why, Throwable e) { - aborted.set(true); - } - }); - l.setBalancerOn(false); - LOG.debug("All meta replicas assigned"); - } - - @After - public void tearDown() throws Exception { - TEST_UTIL.shutdownMiniCluster(); - } - - @Test - public void testMetaHTDReplicaCount() throws Exception { - assertEquals(3, - TEST_UTIL.getAdmin().getDescriptor(TableName.META_TABLE_NAME).getRegionReplication()); - } - - @Test - public void testZookeeperNodesForReplicas() throws Exception { - // Checks all the znodes exist when meta's replicas are enabled - ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher(); - Configuration conf = TEST_UTIL.getConfiguration(); - String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, - HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); - String primaryMetaZnode = ZNodePaths.joinZNode(baseZNode, - conf.get("zookeeper.znode.metaserver", "meta-region-server")); - // check that the data in the znode is parseable (this would also mean the znode exists) - byte[] data = ZKUtil.getData(zkw, primaryMetaZnode); - ProtobufUtil.toServerName(data); - for (int i = 1; i < 3; i++) { - String secZnode = ZNodePaths.joinZNode(baseZNode, - conf.get("zookeeper.znode.metaserver", "meta-region-server") + "-" + i); - String str = zkw.getZNodePaths().getZNodeForReplica(i); - assertTrue(str.equals(secZnode)); - // check that the data in the znode is parseable (this would also mean the znode exists) - data = ZKUtil.getData(zkw, secZnode); - ProtobufUtil.toServerName(data); - } - } - - @Test - public void testShutdownHandling() throws Exception { - // This test creates a table, flushes the meta (with 3 replicas), kills the - // server holding the primary meta replica. Then it does a put/get into/from - // the test table. The put/get operations would use the replicas to locate the - // location of the test table's region - shutdownMetaAndDoValidations(TEST_UTIL); - } - - public static void shutdownMetaAndDoValidations(HBaseTestingUtility util) throws Exception { - // This test creates a table, flushes the meta (with 3 replicas), kills the - // server holding the primary meta replica. Then it does a put/get into/from - // the test table. The put/get operations would use the replicas to locate the - // location of the test table's region - ZKWatcher zkw = util.getZooKeeperWatcher(); - Configuration conf = util.getConfiguration(); - conf.setBoolean(HConstants.USE_META_REPLICAS, true); - - String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, - HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); - String primaryMetaZnode = ZNodePaths.joinZNode(baseZNode, - conf.get("zookeeper.znode.metaserver", "meta-region-server")); - byte[] data = ZKUtil.getData(zkw, primaryMetaZnode); - ServerName primary = ProtobufUtil.toServerName(data); - LOG.info("Primary=" + primary.toString()); - - TableName TABLE = TableName.valueOf("testShutdownHandling"); - byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") }; - if (util.getAdmin().tableExists(TABLE)) { - util.getAdmin().disableTable(TABLE); - util.getAdmin().deleteTable(TABLE); - } - byte[] row = Bytes.toBytes("test"); - ServerName master = null; - try (Connection c = ConnectionFactory.createConnection(util.getConfiguration())) { - try (Table htable = util.createTable(TABLE, FAMILIES)) { - util.getAdmin().flush(TableName.META_TABLE_NAME); - Thread.sleep( - conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 30000) * 6); - List regions = MetaTableAccessor.getTableRegions(c, TABLE); - HRegionLocation hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0)); - // Ensure that the primary server for test table is not the same one as the primary - // of the meta region since we will be killing the srv holding the meta's primary... - // We want to be able to write to the test table even when the meta is not present .. - // If the servers are the same, then move the test table's region out of the server - // to another random server - if (hrl.getServerName().equals(primary)) { - util.getAdmin().move(hrl.getRegion().getEncodedNameAsBytes()); - // wait for the move to complete - do { - Thread.sleep(10); - hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0)); - } while (primary.equals(hrl.getServerName())); - util.getAdmin().flush(TableName.META_TABLE_NAME); - Thread.sleep(conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, - 30000) * 3); - } - // Ensure all metas are not on same hbase:meta replica=0 server! - - master = util.getHBaseClusterInterface().getClusterMetrics().getMasterName(); - // kill the master so that regionserver recovery is not triggered at all - // for the meta server - LOG.info("Stopping master=" + master.toString()); - util.getHBaseClusterInterface().stopMaster(master); - util.getHBaseClusterInterface().waitForMasterToStop(master, 60000); - LOG.info("Master " + master + " stopped!"); - if (!master.equals(primary)) { - util.getHBaseClusterInterface().killRegionServer(primary); - util.getHBaseClusterInterface().waitForRegionServerToStop(primary, 60000); - } - c.clearRegionLocationCache(); - } - LOG.info("Running GETs"); - try (Table htable = c.getTable(TABLE)) { - Put put = new Put(row); - put.addColumn(Bytes.toBytes("foo"), row, row); - BufferedMutator m = c.getBufferedMutator(TABLE); - m.mutate(put); - m.flush(); - // Try to do a get of the row that was just put - Result r = htable.get(new Get(row)); - assertTrue(Arrays.equals(r.getRow(), row)); - // now start back the killed servers and disable use of replicas. That would mean - // calls go to the primary - LOG.info("Starting Master"); - util.getHBaseClusterInterface().startMaster(master.getHostname(), 0); - util.getHBaseClusterInterface().startRegionServer(primary.getHostname(), 0); - util.getHBaseClusterInterface().waitForActiveAndReadyMaster(); - LOG.info("Master active!"); - c.clearRegionLocationCache(); - } - } - conf.setBoolean(HConstants.USE_META_REPLICAS, false); - LOG.info("Running GETs no replicas"); - try (Connection c = ConnectionFactory.createConnection(conf)) { - try (Table htable = c.getTable(TABLE)) { - Result r = htable.get(new Get(row)); - assertTrue(Arrays.equals(r.getRow(), row)); - } - } - } - - @Test - public void testAccessingUnknownTables() throws Exception { - Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); - conf.setBoolean(HConstants.USE_META_REPLICAS, true); - Table table = TEST_UTIL.getConnection().getTable(TableName.valueOf(name.getMethodName())); - Get get = new Get(Bytes.toBytes("foo")); - try { - table.get(get); - } catch (TableNotFoundException t) { - return; - } - fail("Expected TableNotFoundException"); - } - - @Test - public void testMetaAddressChange() throws Exception { - // checks that even when the meta's location changes, the various - // caches update themselves. Uses the master operations to test - // this - Configuration conf = TEST_UTIL.getConfiguration(); - ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher(); - String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, - HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); - String primaryMetaZnode = ZNodePaths.joinZNode(baseZNode, - conf.get("zookeeper.znode.metaserver", "meta-region-server")); - // check that the data in the znode is parseable (this would also mean the znode exists) - byte[] data = ZKUtil.getData(zkw, primaryMetaZnode); - ServerName currentServer = ProtobufUtil.toServerName(data); - Collection liveServers = TEST_UTIL.getAdmin() - .getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet(); - ServerName moveToServer = null; - for (ServerName s : liveServers) { - if (!currentServer.equals(s)) { - moveToServer = s; - } - } - assertNotNull(moveToServer); - final TableName tableName = TableName.valueOf(name.getMethodName()); - TEST_UTIL.createTable(tableName, "f"); - assertTrue(TEST_UTIL.getAdmin().tableExists(tableName)); - TEST_UTIL.getAdmin().move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), - moveToServer); - int i = 0; - assertNotEquals(currentServer, moveToServer); - LOG.info("CurrentServer=" + currentServer + ", moveToServer=" + moveToServer); - final int max = 10000; - do { - Thread.sleep(10); - data = ZKUtil.getData(zkw, primaryMetaZnode); - currentServer = ProtobufUtil.toServerName(data); - i++; - } while (!moveToServer.equals(currentServer) && i < max); //wait for 10 seconds overall - assertNotEquals(max, i); - TEST_UTIL.getAdmin().disableTable(tableName); - assertTrue(TEST_UTIL.getAdmin().isTableDisabled(tableName)); - } - - @Test - public void testShutdownOfReplicaHolder() throws Exception { - // checks that the when the server holding meta replica is shut down, the meta replica - // can be recovered - try ( - Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); - RegionLocator locator = conn.getRegionLocator(TableName.META_TABLE_NAME)) { - HRegionLocation hrl = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1); - ServerName oldServer = hrl.getServerName(); - TEST_UTIL.getHBaseClusterInterface().killRegionServer(oldServer); - int i = 0; - do { - LOG.debug("Waiting for the replica " + hrl.getRegion() + " to come up"); - Thread.sleep(10000); // wait for the detection/recovery - hrl = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1); - i++; - } while ((hrl == null || hrl.getServerName().equals(oldServer)) && i < 3); - assertNotEquals(3, i); - } - } - - @Test - public void testFailedReplicaAssigment() throws InterruptedException, IOException { - //using our rigged master, to force a failed meta replica assignment - TEST_UTIL.getMiniHBaseCluster().getConfiguration().setClass(HConstants.MASTER_IMPL, BrokenMetaReplicaMaster.class, HMaster.class); - TEST_UTIL.getMiniHBaseCluster().stopMaster(0).join(); - HMaster newMaster = TEST_UTIL.getMiniHBaseCluster().startMaster().getMaster(); - //waiting for master to come up - TEST_UTIL.waitFor(30000, () -> newMaster.isInitialized()); - TEST_UTIL.getMiniHBaseCluster().getConfiguration().unset(HConstants.MASTER_IMPL); - - - AssignmentManager am = newMaster.getAssignmentManager(); - //showing one of the replicas got assigned - RegionInfo metaReplicaHri = RegionReplicaUtil.getRegionInfoForReplica( - RegionInfoBuilder.FIRST_META_REGIONINFO, 1); - RegionStateNode metaReplicaRegionNode = am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri); - Assert.assertNotNull(metaReplicaRegionNode.getRegionLocation()); - //showing one of the replicas failed to be assigned - RegionInfo metaReplicaHri2 = RegionReplicaUtil.getRegionInfoForReplica( - RegionInfoBuilder.FIRST_META_REGIONINFO, 2); - RegionStateNode metaReplicaRegionNode2 = am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri2); - Assert.assertNull(metaReplicaRegionNode2.getRegionLocation()); - - //showing master is active and running - Assert.assertFalse(newMaster.isStopping()); - Assert.assertFalse(newMaster.isStopped()); - Assert.assertTrue(newMaster.isActiveMaster()); - } - - public static class BrokenTransitRegionStateProcedure extends TransitRegionStateProcedure { - protected BrokenTransitRegionStateProcedure() { - //super(env, hri, assignCandidate, forceNewPlan, type); - super(null, null, null, false,TransitionType.ASSIGN); - } - } - - public static class BrokenMetaReplicaMaster extends HMaster{ - public BrokenMetaReplicaMaster(final Configuration conf) throws IOException { - super(conf); - } - - @Override - public AssignmentManager createAssignmentManager(MasterServices master) { - return new BrokenMasterMetaAssignmentManager(master); - } - } - - public static class BrokenMasterMetaAssignmentManager extends AssignmentManager{ - MasterServices master; - public BrokenMasterMetaAssignmentManager(final MasterServices master) { - super(master); - this.master = master; - } - - public Future assignAsync(RegionInfo regionInfo, ServerName sn) throws IOException { - RegionStateNode regionNode = getRegionStates().getOrCreateRegionStateNode(regionInfo); - if (regionNode.getRegionInfo().getReplicaId() == 2) { - regionNode.setProcedure(new BrokenTransitRegionStateProcedure()); - } - return super.assignAsync(regionInfo, sn); - } - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasBasic.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasBasic.java new file mode 100644 index 00000000000..8ffbe6bb47f --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasBasic.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; +import org.apache.hadoop.hbase.zookeeper.ZNodePaths; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; + +@Category({ MiscTests.class, MediumTests.class }) +public class TestMetaWithReplicasBasic extends MetaWithReplicasTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMetaWithReplicasBasic.class); + + @BeforeClass + public static void setUp() throws Exception { + startCluster(); + } + + @Test + public void testMetaHTDReplicaCount() throws Exception { + assertEquals(3, + TEST_UTIL.getAdmin().getDescriptor(TableName.META_TABLE_NAME).getRegionReplication()); + } + + @Test + public void testZookeeperNodesForReplicas() throws Exception { + // Checks all the znodes exist when meta's replicas are enabled + ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher(); + Configuration conf = TEST_UTIL.getConfiguration(); + String baseZNode = + conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); + String primaryMetaZnode = + ZNodePaths.joinZNode(baseZNode, conf.get("zookeeper.znode.metaserver", "meta-region-server")); + // check that the data in the znode is parseable (this would also mean the znode exists) + byte[] data = ZKUtil.getData(zkw, primaryMetaZnode); + ProtobufUtil.toServerName(data); + for (int i = 1; i < 3; i++) { + String secZnode = ZNodePaths.joinZNode(baseZNode, + conf.get("zookeeper.znode.metaserver", "meta-region-server") + "-" + i); + String str = zkw.getZNodePaths().getZNodeForReplica(i); + assertTrue(str.equals(secZnode)); + // check that the data in the znode is parseable (this would also mean the znode exists) + data = ZKUtil.getData(zkw, secZnode); + ProtobufUtil.toServerName(data); + } + } + + @Test + public void testAccessingUnknownTables() throws Exception { + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + conf.setBoolean(HConstants.USE_META_REPLICAS, true); + Table table = TEST_UTIL.getConnection().getTable(name.getTableName()); + Get get = new Get(Bytes.toBytes("foo")); + assertThrows(TableNotFoundException.class, () -> table.get(get)); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasShutdownHandling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasShutdownHandling.java new file mode 100644 index 00000000000..db08165a395 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasShutdownHandling.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; +import org.apache.hadoop.hbase.zookeeper.ZNodePaths; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; + +@Category({ MiscTests.class, MediumTests.class }) +public class TestMetaWithReplicasShutdownHandling extends MetaWithReplicasTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMetaWithReplicasShutdownHandling.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestMetaWithReplicasShutdownHandling.class); + + @BeforeClass + public static void setUp() throws Exception { + startCluster(); + } + + @Test + public void testShutdownHandling() throws Exception { + // This test creates a table, flushes the meta (with 3 replicas), kills the + // server holding the primary meta replica. Then it does a put/get into/from + // the test table. The put/get operations would use the replicas to locate the + // location of the test table's region + shutdownMetaAndDoValidations(TEST_UTIL); + } + + public static void shutdownMetaAndDoValidations(HBaseTestingUtility util) throws Exception { + // This test creates a table, flushes the meta (with 3 replicas), kills the + // server holding the primary meta replica. Then it does a put/get into/from + // the test table. The put/get operations would use the replicas to locate the + // location of the test table's region + ZKWatcher zkw = util.getZooKeeperWatcher(); + Configuration conf = util.getConfiguration(); + conf.setBoolean(HConstants.USE_META_REPLICAS, true); + + String baseZNode = + conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); + String primaryMetaZnode = + ZNodePaths.joinZNode(baseZNode, conf.get("zookeeper.znode.metaserver", "meta-region-server")); + byte[] data = ZKUtil.getData(zkw, primaryMetaZnode); + ServerName primary = ProtobufUtil.toServerName(data); + LOG.info("Primary=" + primary.toString()); + + TableName TABLE = TableName.valueOf("testShutdownHandling"); + byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") }; + if (util.getAdmin().tableExists(TABLE)) { + util.getAdmin().disableTable(TABLE); + util.getAdmin().deleteTable(TABLE); + } + byte[] row = Bytes.toBytes("test"); + ServerName master = null; + try (Connection c = ConnectionFactory.createConnection(util.getConfiguration())) { + try (Table htable = util.createTable(TABLE, FAMILIES)) { + util.getAdmin().flush(TableName.META_TABLE_NAME); + Thread.sleep( + conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 30000) * 6); + List regions = MetaTableAccessor.getTableRegions(c, TABLE); + HRegionLocation hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0)); + // Ensure that the primary server for test table is not the same one as the primary + // of the meta region since we will be killing the srv holding the meta's primary... + // We want to be able to write to the test table even when the meta is not present .. + // If the servers are the same, then move the test table's region out of the server + // to another random server + if (hrl.getServerName().equals(primary)) { + util.getAdmin().move(hrl.getRegion().getEncodedNameAsBytes()); + // wait for the move to complete + do { + Thread.sleep(10); + hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0)); + } while (primary.equals(hrl.getServerName())); + util.getAdmin().flush(TableName.META_TABLE_NAME); + Thread.sleep( + conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 30000) * 3); + } + // Ensure all metas are not on same hbase:meta replica=0 server! + + master = util.getHBaseClusterInterface().getClusterMetrics().getMasterName(); + // kill the master so that regionserver recovery is not triggered at all + // for the meta server + LOG.info("Stopping master=" + master.toString()); + util.getHBaseClusterInterface().stopMaster(master); + util.getHBaseClusterInterface().waitForMasterToStop(master, 60000); + LOG.info("Master " + master + " stopped!"); + if (!master.equals(primary)) { + util.getHBaseClusterInterface().killRegionServer(primary); + util.getHBaseClusterInterface().waitForRegionServerToStop(primary, 60000); + } + c.clearRegionLocationCache(); + } + LOG.info("Running GETs"); + try (Table htable = c.getTable(TABLE)) { + Put put = new Put(row); + put.addColumn(Bytes.toBytes("foo"), row, row); + BufferedMutator m = c.getBufferedMutator(TABLE); + m.mutate(put); + m.flush(); + // Try to do a get of the row that was just put + Result r = htable.get(new Get(row)); + assertTrue(Arrays.equals(r.getRow(), row)); + // now start back the killed servers and disable use of replicas. That would mean + // calls go to the primary + LOG.info("Starting Master"); + util.getHBaseClusterInterface().startMaster(master.getHostname(), 0); + util.getHBaseClusterInterface().startRegionServer(primary.getHostname(), 0); + util.getHBaseClusterInterface().waitForActiveAndReadyMaster(); + LOG.info("Master active!"); + c.clearRegionLocationCache(); + } + } + conf.setBoolean(HConstants.USE_META_REPLICAS, false); + LOG.info("Running GETs no replicas"); + try (Connection c = ConnectionFactory.createConnection(conf); + Table htable = c.getTable(TABLE)) { + Result r = htable.get(new Get(row)); + assertArrayEquals(row, r.getRow()); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestShutdownOfMetaReplicaHolder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestShutdownOfMetaReplicaHolder.java new file mode 100644 index 00000000000..f03ac2f5850 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestShutdownOfMetaReplicaHolder.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({ MiscTests.class, MediumTests.class }) +public class TestShutdownOfMetaReplicaHolder extends MetaWithReplicasTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestShutdownOfMetaReplicaHolder.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestShutdownOfMetaReplicaHolder.class); + + @BeforeClass + public static void setUp() throws Exception { + startCluster(); + } + + @Test + public void testShutdownOfReplicaHolder() throws Exception { + // checks that the when the server holding meta replica is shut down, the meta replica + // can be recovered + try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); + RegionLocator locator = conn.getRegionLocator(TableName.META_TABLE_NAME)) { + HRegionLocation hrl = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1); + ServerName oldServer = hrl.getServerName(); + TEST_UTIL.getHBaseClusterInterface().killRegionServer(oldServer); + LOG.debug("Waiting for the replica {} to come up", hrl.getRegion()); + TEST_UTIL.waitFor(30000, () -> { + HRegionLocation loc = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1); + return !loc.getServerName().equals(oldServer); + }); + LOG.debug("Replica {} is online on {}, old server is {}", hrl.getRegion(), + locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1).getServerName(), + oldServer); + } + } +}