HBASE-24635 Split TestMetaWithReplicas (#1980)
Signed-off-by: Guanghao Zhang <zghao@apache.org>
This commit is contained in:
Duo Zhang
GitHub
parent
83f27b5287
commit
16a25b74db
|
|
@ -19,7 +19,7 @@ package org.apache.hadoop.hbase;
|
|||
|
||||
import java.io.IOException;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.client.TestMetaWithReplicas;
|
||||
import org.apache.hadoop.hbase.client.TestMetaWithReplicasShutdownHandling;
|
||||
import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
|
||||
import org.apache.hadoop.hbase.testclassification.IntegrationTests;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
|
||||
|
|
@ -96,7 +96,7 @@ public class IntegrationTestMetaReplicas {
|
|||
// server holding the primary meta replica. Then it does a put/get into/from
|
||||
// the test table. The put/get operations would use the replicas to locate the
|
||||
// location of the test table's region
|
||||
TestMetaWithReplicas.shutdownMetaAndDoValidations(util);
|
||||
TestMetaWithReplicasShutdownHandling.shutdownMetaAndDoValidations(util);
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,123 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.client;
|
||||
|
||||
import static org.junit.Assert.assertNotEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.apache.hadoop.hbase.Abortable;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.StartMiniClusterOption;
|
||||
import org.apache.hadoop.hbase.TableNameTestRule;
|
||||
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
|
||||
import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
|
||||
import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
|
||||
import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
|
||||
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Rule;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Base class for testing the scenarios where replicas are enabled for the meta table.
|
||||
*/
|
||||
public class MetaWithReplicasTestBase {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(MetaWithReplicasTestBase.class);
|
||||
|
||||
protected static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
|
||||
|
||||
protected static final int REGIONSERVERS_COUNT = 3;
|
||||
|
||||
@Rule
|
||||
public TableNameTestRule name = new TableNameTestRule();
|
||||
|
||||
protected static void startCluster() throws Exception {
|
||||
TEST_UTIL.getConfiguration().setInt("zookeeper.session.timeout", 30000);
|
||||
TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, 3);
|
||||
TEST_UTIL.getConfiguration()
|
||||
.setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 1000);
|
||||
StartMiniClusterOption option = StartMiniClusterOption.builder().numAlwaysStandByMasters(1)
|
||||
.numMasters(1).numRegionServers(REGIONSERVERS_COUNT).build();
|
||||
TEST_UTIL.startMiniCluster(option);
|
||||
AssignmentManager am = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
|
||||
Set<ServerName> sns = new HashSet<ServerName>();
|
||||
ServerName hbaseMetaServerName =
|
||||
MetaTableLocator.getMetaRegionLocation(TEST_UTIL.getZooKeeperWatcher());
|
||||
LOG.info("HBASE:META DEPLOY: on " + hbaseMetaServerName);
|
||||
sns.add(hbaseMetaServerName);
|
||||
for (int replicaId = 1; replicaId < 3; replicaId++) {
|
||||
RegionInfo h = RegionReplicaUtil
|
||||
.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId);
|
||||
AssignmentTestingUtil.waitForAssignment(am, h);
|
||||
ServerName sn = am.getRegionStates().getRegionServerOfRegion(h);
|
||||
assertNotNull(sn);
|
||||
LOG.info("HBASE:META DEPLOY: " + h.getRegionNameAsString() + " on " + sn);
|
||||
sns.add(sn);
|
||||
}
|
||||
// Fun. All meta region replicas have ended up on the one server. This will cause this test
|
||||
// to fail ... sometimes.
|
||||
if (sns.size() == 1) {
|
||||
int count = TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size();
|
||||
assertTrue("count=" + count, count == REGIONSERVERS_COUNT);
|
||||
LOG.warn("All hbase:meta replicas are on the one server; moving hbase:meta: " + sns);
|
||||
int metaServerIndex = TEST_UTIL.getHBaseCluster().getServerWithMeta();
|
||||
int newServerIndex = metaServerIndex;
|
||||
while (newServerIndex == metaServerIndex) {
|
||||
newServerIndex = (newServerIndex + 1) % REGIONSERVERS_COUNT;
|
||||
}
|
||||
assertNotEquals(metaServerIndex, newServerIndex);
|
||||
ServerName destinationServerName =
|
||||
TEST_UTIL.getHBaseCluster().getRegionServer(newServerIndex).getServerName();
|
||||
ServerName metaServerName =
|
||||
TEST_UTIL.getHBaseCluster().getRegionServer(metaServerIndex).getServerName();
|
||||
assertNotEquals(destinationServerName, metaServerName);
|
||||
TEST_UTIL.getAdmin().move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
|
||||
destinationServerName);
|
||||
}
|
||||
// Disable the balancer
|
||||
LoadBalancerTracker l =
|
||||
new LoadBalancerTracker(TEST_UTIL.getZooKeeperWatcher(), new Abortable() {
|
||||
AtomicBoolean aborted = new AtomicBoolean(false);
|
||||
|
||||
@Override
|
||||
public boolean isAborted() {
|
||||
return aborted.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void abort(String why, Throwable e) {
|
||||
aborted.set(true);
|
||||
}
|
||||
});
|
||||
l.setBalancerOn(false);
|
||||
LOG.debug("All meta replicas assigned");
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDown() throws Exception {
|
||||
TEST_UTIL.shutdownMiniCluster();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.client;
|
||||
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.Future;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.StartMiniClusterOption;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.master.MasterServices;
|
||||
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
|
||||
import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
|
||||
import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.testclassification.MiscTests;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
|
||||
@Category({ MiscTests.class, MediumTests.class })
|
||||
public class TestFailedMetaReplicaAssigment {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestFailedMetaReplicaAssigment.class);
|
||||
|
||||
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
// using our rigged master, to force a failed meta replica assignment
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
conf.setInt(HConstants.META_REPLICAS_NUM, 3);
|
||||
StartMiniClusterOption option = StartMiniClusterOption.builder().numAlwaysStandByMasters(1)
|
||||
.numMasters(1).numRegionServers(1).masterClass(BrokenMetaReplicaMaster.class).build();
|
||||
TEST_UTIL.startMiniCluster(option);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDown() throws IOException {
|
||||
TEST_UTIL.shutdownMiniCluster();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFailedReplicaAssignment() throws InterruptedException {
|
||||
HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
|
||||
// waiting for master to come up
|
||||
TEST_UTIL.waitFor(30000, () -> master.isInitialized());
|
||||
|
||||
AssignmentManager am = master.getAssignmentManager();
|
||||
// showing one of the replicas got assigned
|
||||
RegionInfo metaReplicaHri =
|
||||
RegionReplicaUtil.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, 1);
|
||||
// we use assignAsync so we need to wait a bit
|
||||
TEST_UTIL.waitFor(30000, () -> {
|
||||
RegionStateNode metaReplicaRegionNode =
|
||||
am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri);
|
||||
return metaReplicaRegionNode.getRegionLocation() != null;
|
||||
});
|
||||
// showing one of the replicas failed to be assigned
|
||||
RegionInfo metaReplicaHri2 =
|
||||
RegionReplicaUtil.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, 2);
|
||||
RegionStateNode metaReplicaRegionNode2 =
|
||||
am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri2);
|
||||
// wait for several seconds to make sure that it is not assigned
|
||||
for (int i = 0; i < 3; i++) {
|
||||
Thread.sleep(2000);
|
||||
assertNull(metaReplicaRegionNode2.getRegionLocation());
|
||||
}
|
||||
|
||||
// showing master is active and running
|
||||
assertFalse(master.isStopping());
|
||||
assertFalse(master.isStopped());
|
||||
assertTrue(master.isActiveMaster());
|
||||
}
|
||||
|
||||
public static class BrokenTransitRegionStateProcedure extends TransitRegionStateProcedure {
|
||||
protected BrokenTransitRegionStateProcedure() {
|
||||
super(null, null, null, false, TransitionType.ASSIGN);
|
||||
}
|
||||
}
|
||||
|
||||
public static class BrokenMetaReplicaMaster extends HMaster {
|
||||
public BrokenMetaReplicaMaster(final Configuration conf) throws IOException {
|
||||
super(conf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AssignmentManager createAssignmentManager(MasterServices master) {
|
||||
return new BrokenMasterMetaAssignmentManager(master);
|
||||
}
|
||||
}
|
||||
|
||||
public static class BrokenMasterMetaAssignmentManager extends AssignmentManager {
|
||||
MasterServices master;
|
||||
|
||||
public BrokenMasterMetaAssignmentManager(final MasterServices master) {
|
||||
super(master);
|
||||
this.master = master;
|
||||
}
|
||||
|
||||
public Future<byte[]> assignAsync(RegionInfo regionInfo, ServerName sn) throws IOException {
|
||||
RegionStateNode regionNode = getRegionStates().getOrCreateRegionStateNode(regionInfo);
|
||||
if (regionNode.getRegionInfo().getReplicaId() == 2) {
|
||||
regionNode.setProcedure(new BrokenTransitRegionStateProcedure());
|
||||
}
|
||||
return super.assignAsync(regionInfo, sn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.client;
|
||||
|
||||
import static org.junit.Assert.assertNotEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.EnumSet;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.ClusterMetrics.Option;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.testclassification.MiscTests;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
||||
|
||||
@Category({ MiscTests.class, MediumTests.class })
|
||||
public class TestMetaReplicasAddressChange extends MetaWithReplicasTestBase {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestMetaReplicasAddressChange.class);
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(TestMetaReplicasAddressChange.class);
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
startCluster();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMetaAddressChange() throws Exception {
|
||||
// checks that even when the meta's location changes, the various
|
||||
// caches update themselves. Uses the master operations to test
|
||||
// this
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
|
||||
String baseZNode =
|
||||
conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
|
||||
String primaryMetaZnode =
|
||||
ZNodePaths.joinZNode(baseZNode, conf.get("zookeeper.znode.metaserver", "meta-region-server"));
|
||||
// check that the data in the znode is parseable (this would also mean the znode exists)
|
||||
byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
|
||||
ServerName currentServer = ProtobufUtil.toServerName(data);
|
||||
Collection<ServerName> liveServers = TEST_UTIL.getAdmin()
|
||||
.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet();
|
||||
ServerName moveToServer =
|
||||
liveServers.stream().filter(s -> !currentServer.equals(s)).findAny().get();
|
||||
final TableName tableName = name.getTableName();
|
||||
TEST_UTIL.createTable(tableName, "f");
|
||||
assertTrue(TEST_UTIL.getAdmin().tableExists(tableName));
|
||||
TEST_UTIL.getAdmin().move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
|
||||
moveToServer);
|
||||
assertNotEquals(currentServer, moveToServer);
|
||||
LOG.debug("CurrentServer={}, moveToServer={}", currentServer, moveToServer);
|
||||
TEST_UTIL.waitFor(60000, () -> {
|
||||
byte[] bytes = ZKUtil.getData(zkw, primaryMetaZnode);
|
||||
ServerName actualServer = ProtobufUtil.toServerName(bytes);
|
||||
return moveToServer.equals(actualServer);
|
||||
});
|
||||
TEST_UTIL.getAdmin().disableTable(tableName);
|
||||
assertTrue(TEST_UTIL.getAdmin().isTableDisabled(tableName));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,428 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.client;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.Abortable;
|
||||
import org.apache.hadoop.hbase.ClusterMetrics.Option;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.HRegionLocation;
|
||||
import org.apache.hadoop.hbase.MetaTableAccessor;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.StartMiniClusterOption;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.TableNotFoundException;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.master.MasterServices;
|
||||
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
|
||||
import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
|
||||
import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
|
||||
import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
|
||||
import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
|
||||
import org.apache.hadoop.hbase.testclassification.LargeTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
|
||||
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.junit.rules.TestName;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
||||
|
||||
/**
|
||||
* Tests the scenarios where replicas are enabled for the meta table
|
||||
*/
|
||||
@Category(LargeTests.class)
|
||||
public class TestMetaWithReplicas {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestMetaWithReplicas.class);
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(TestMetaWithReplicas.class);
|
||||
private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
|
||||
private static final int REGIONSERVERS_COUNT = 3;
|
||||
|
||||
@Rule
|
||||
public TestName name = new TestName();
|
||||
|
||||
@Before
|
||||
public void setup() throws Exception {
|
||||
TEST_UTIL.getConfiguration().setInt("zookeeper.session.timeout", 30000);
|
||||
TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, 3);
|
||||
TEST_UTIL.getConfiguration().setInt(
|
||||
StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 1000);
|
||||
StartMiniClusterOption option = StartMiniClusterOption.builder().
|
||||
numAlwaysStandByMasters(1).numMasters(1).numRegionServers(REGIONSERVERS_COUNT).build();
|
||||
TEST_UTIL.startMiniCluster(option);
|
||||
AssignmentManager am = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
|
||||
Set<ServerName> sns = new HashSet<ServerName>();
|
||||
ServerName hbaseMetaServerName =
|
||||
MetaTableLocator.getMetaRegionLocation(TEST_UTIL.getZooKeeperWatcher());
|
||||
LOG.info("HBASE:META DEPLOY: on " + hbaseMetaServerName);
|
||||
sns.add(hbaseMetaServerName);
|
||||
for (int replicaId = 1; replicaId < 3; replicaId++) {
|
||||
RegionInfo h = RegionReplicaUtil
|
||||
.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId);
|
||||
AssignmentTestingUtil.waitForAssignment(am, h);
|
||||
ServerName sn = am.getRegionStates().getRegionServerOfRegion(h);
|
||||
assertNotNull(sn);
|
||||
LOG.info("HBASE:META DEPLOY: " + h.getRegionNameAsString() + " on " + sn);
|
||||
sns.add(sn);
|
||||
}
|
||||
// Fun. All meta region replicas have ended up on the one server. This will cause this test
|
||||
// to fail ... sometimes.
|
||||
if (sns.size() == 1) {
|
||||
int count = TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size();
|
||||
assertTrue("count=" + count, count == REGIONSERVERS_COUNT);
|
||||
LOG.warn("All hbase:meta replicas are on the one server; moving hbase:meta: " + sns);
|
||||
int metaServerIndex = TEST_UTIL.getHBaseCluster().getServerWithMeta();
|
||||
int newServerIndex = metaServerIndex;
|
||||
while (newServerIndex == metaServerIndex) {
|
||||
newServerIndex = (newServerIndex + 1) % REGIONSERVERS_COUNT;
|
||||
}
|
||||
assertNotEquals(metaServerIndex, newServerIndex);
|
||||
ServerName destinationServerName =
|
||||
TEST_UTIL.getHBaseCluster().getRegionServer(newServerIndex).getServerName();
|
||||
ServerName metaServerName =
|
||||
TEST_UTIL.getHBaseCluster().getRegionServer(metaServerIndex).getServerName();
|
||||
assertNotEquals(destinationServerName, metaServerName);
|
||||
TEST_UTIL.getAdmin().move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
|
||||
destinationServerName);
|
||||
}
|
||||
// Disable the balancer
|
||||
LoadBalancerTracker l = new LoadBalancerTracker(TEST_UTIL.getZooKeeperWatcher(),
|
||||
new Abortable() {
|
||||
AtomicBoolean aborted = new AtomicBoolean(false);
|
||||
@Override
|
||||
public boolean isAborted() {
|
||||
return aborted.get();
|
||||
}
|
||||
@Override
|
||||
public void abort(String why, Throwable e) {
|
||||
aborted.set(true);
|
||||
}
|
||||
});
|
||||
l.setBalancerOn(false);
|
||||
LOG.debug("All meta replicas assigned");
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
TEST_UTIL.shutdownMiniCluster();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMetaHTDReplicaCount() throws Exception {
|
||||
assertEquals(3,
|
||||
TEST_UTIL.getAdmin().getDescriptor(TableName.META_TABLE_NAME).getRegionReplication());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testZookeeperNodesForReplicas() throws Exception {
|
||||
// Checks all the znodes exist when meta's replicas are enabled
|
||||
ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
|
||||
HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
|
||||
String primaryMetaZnode = ZNodePaths.joinZNode(baseZNode,
|
||||
conf.get("zookeeper.znode.metaserver", "meta-region-server"));
|
||||
// check that the data in the znode is parseable (this would also mean the znode exists)
|
||||
byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
|
||||
ProtobufUtil.toServerName(data);
|
||||
for (int i = 1; i < 3; i++) {
|
||||
String secZnode = ZNodePaths.joinZNode(baseZNode,
|
||||
conf.get("zookeeper.znode.metaserver", "meta-region-server") + "-" + i);
|
||||
String str = zkw.getZNodePaths().getZNodeForReplica(i);
|
||||
assertTrue(str.equals(secZnode));
|
||||
// check that the data in the znode is parseable (this would also mean the znode exists)
|
||||
data = ZKUtil.getData(zkw, secZnode);
|
||||
ProtobufUtil.toServerName(data);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShutdownHandling() throws Exception {
|
||||
// This test creates a table, flushes the meta (with 3 replicas), kills the
|
||||
// server holding the primary meta replica. Then it does a put/get into/from
|
||||
// the test table. The put/get operations would use the replicas to locate the
|
||||
// location of the test table's region
|
||||
shutdownMetaAndDoValidations(TEST_UTIL);
|
||||
}
|
||||
|
||||
public static void shutdownMetaAndDoValidations(HBaseTestingUtility util) throws Exception {
|
||||
// This test creates a table, flushes the meta (with 3 replicas), kills the
|
||||
// server holding the primary meta replica. Then it does a put/get into/from
|
||||
// the test table. The put/get operations would use the replicas to locate the
|
||||
// location of the test table's region
|
||||
ZKWatcher zkw = util.getZooKeeperWatcher();
|
||||
Configuration conf = util.getConfiguration();
|
||||
conf.setBoolean(HConstants.USE_META_REPLICAS, true);
|
||||
|
||||
String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
|
||||
HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
|
||||
String primaryMetaZnode = ZNodePaths.joinZNode(baseZNode,
|
||||
conf.get("zookeeper.znode.metaserver", "meta-region-server"));
|
||||
byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
|
||||
ServerName primary = ProtobufUtil.toServerName(data);
|
||||
LOG.info("Primary=" + primary.toString());
|
||||
|
||||
TableName TABLE = TableName.valueOf("testShutdownHandling");
|
||||
byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
|
||||
if (util.getAdmin().tableExists(TABLE)) {
|
||||
util.getAdmin().disableTable(TABLE);
|
||||
util.getAdmin().deleteTable(TABLE);
|
||||
}
|
||||
byte[] row = Bytes.toBytes("test");
|
||||
ServerName master = null;
|
||||
try (Connection c = ConnectionFactory.createConnection(util.getConfiguration())) {
|
||||
try (Table htable = util.createTable(TABLE, FAMILIES)) {
|
||||
util.getAdmin().flush(TableName.META_TABLE_NAME);
|
||||
Thread.sleep(
|
||||
conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 30000) * 6);
|
||||
List<RegionInfo> regions = MetaTableAccessor.getTableRegions(c, TABLE);
|
||||
HRegionLocation hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
|
||||
// Ensure that the primary server for test table is not the same one as the primary
|
||||
// of the meta region since we will be killing the srv holding the meta's primary...
|
||||
// We want to be able to write to the test table even when the meta is not present ..
|
||||
// If the servers are the same, then move the test table's region out of the server
|
||||
// to another random server
|
||||
if (hrl.getServerName().equals(primary)) {
|
||||
util.getAdmin().move(hrl.getRegion().getEncodedNameAsBytes());
|
||||
// wait for the move to complete
|
||||
do {
|
||||
Thread.sleep(10);
|
||||
hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
|
||||
} while (primary.equals(hrl.getServerName()));
|
||||
util.getAdmin().flush(TableName.META_TABLE_NAME);
|
||||
Thread.sleep(conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD,
|
||||
30000) * 3);
|
||||
}
|
||||
// Ensure all metas are not on same hbase:meta replica=0 server!
|
||||
|
||||
master = util.getHBaseClusterInterface().getClusterMetrics().getMasterName();
|
||||
// kill the master so that regionserver recovery is not triggered at all
|
||||
// for the meta server
|
||||
LOG.info("Stopping master=" + master.toString());
|
||||
util.getHBaseClusterInterface().stopMaster(master);
|
||||
util.getHBaseClusterInterface().waitForMasterToStop(master, 60000);
|
||||
LOG.info("Master " + master + " stopped!");
|
||||
if (!master.equals(primary)) {
|
||||
util.getHBaseClusterInterface().killRegionServer(primary);
|
||||
util.getHBaseClusterInterface().waitForRegionServerToStop(primary, 60000);
|
||||
}
|
||||
c.clearRegionLocationCache();
|
||||
}
|
||||
LOG.info("Running GETs");
|
||||
try (Table htable = c.getTable(TABLE)) {
|
||||
Put put = new Put(row);
|
||||
put.addColumn(Bytes.toBytes("foo"), row, row);
|
||||
BufferedMutator m = c.getBufferedMutator(TABLE);
|
||||
m.mutate(put);
|
||||
m.flush();
|
||||
// Try to do a get of the row that was just put
|
||||
Result r = htable.get(new Get(row));
|
||||
assertTrue(Arrays.equals(r.getRow(), row));
|
||||
// now start back the killed servers and disable use of replicas. That would mean
|
||||
// calls go to the primary
|
||||
LOG.info("Starting Master");
|
||||
util.getHBaseClusterInterface().startMaster(master.getHostname(), 0);
|
||||
util.getHBaseClusterInterface().startRegionServer(primary.getHostname(), 0);
|
||||
util.getHBaseClusterInterface().waitForActiveAndReadyMaster();
|
||||
LOG.info("Master active!");
|
||||
c.clearRegionLocationCache();
|
||||
}
|
||||
}
|
||||
conf.setBoolean(HConstants.USE_META_REPLICAS, false);
|
||||
LOG.info("Running GETs no replicas");
|
||||
try (Connection c = ConnectionFactory.createConnection(conf)) {
|
||||
try (Table htable = c.getTable(TABLE)) {
|
||||
Result r = htable.get(new Get(row));
|
||||
assertTrue(Arrays.equals(r.getRow(), row));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAccessingUnknownTables() throws Exception {
|
||||
Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
|
||||
conf.setBoolean(HConstants.USE_META_REPLICAS, true);
|
||||
Table table = TEST_UTIL.getConnection().getTable(TableName.valueOf(name.getMethodName()));
|
||||
Get get = new Get(Bytes.toBytes("foo"));
|
||||
try {
|
||||
table.get(get);
|
||||
} catch (TableNotFoundException t) {
|
||||
return;
|
||||
}
|
||||
fail("Expected TableNotFoundException");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMetaAddressChange() throws Exception {
|
||||
// checks that even when the meta's location changes, the various
|
||||
// caches update themselves. Uses the master operations to test
|
||||
// this
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
|
||||
String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
|
||||
HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
|
||||
String primaryMetaZnode = ZNodePaths.joinZNode(baseZNode,
|
||||
conf.get("zookeeper.znode.metaserver", "meta-region-server"));
|
||||
// check that the data in the znode is parseable (this would also mean the znode exists)
|
||||
byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
|
||||
ServerName currentServer = ProtobufUtil.toServerName(data);
|
||||
Collection<ServerName> liveServers = TEST_UTIL.getAdmin()
|
||||
.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet();
|
||||
ServerName moveToServer = null;
|
||||
for (ServerName s : liveServers) {
|
||||
if (!currentServer.equals(s)) {
|
||||
moveToServer = s;
|
||||
}
|
||||
}
|
||||
assertNotNull(moveToServer);
|
||||
final TableName tableName = TableName.valueOf(name.getMethodName());
|
||||
TEST_UTIL.createTable(tableName, "f");
|
||||
assertTrue(TEST_UTIL.getAdmin().tableExists(tableName));
|
||||
TEST_UTIL.getAdmin().move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
|
||||
moveToServer);
|
||||
int i = 0;
|
||||
assertNotEquals(currentServer, moveToServer);
|
||||
LOG.info("CurrentServer=" + currentServer + ", moveToServer=" + moveToServer);
|
||||
final int max = 10000;
|
||||
do {
|
||||
Thread.sleep(10);
|
||||
data = ZKUtil.getData(zkw, primaryMetaZnode);
|
||||
currentServer = ProtobufUtil.toServerName(data);
|
||||
i++;
|
||||
} while (!moveToServer.equals(currentServer) && i < max); //wait for 10 seconds overall
|
||||
assertNotEquals(max, i);
|
||||
TEST_UTIL.getAdmin().disableTable(tableName);
|
||||
assertTrue(TEST_UTIL.getAdmin().isTableDisabled(tableName));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShutdownOfReplicaHolder() throws Exception {
|
||||
// checks that the when the server holding meta replica is shut down, the meta replica
|
||||
// can be recovered
|
||||
try (
|
||||
Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
|
||||
RegionLocator locator = conn.getRegionLocator(TableName.META_TABLE_NAME)) {
|
||||
HRegionLocation hrl = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1);
|
||||
ServerName oldServer = hrl.getServerName();
|
||||
TEST_UTIL.getHBaseClusterInterface().killRegionServer(oldServer);
|
||||
int i = 0;
|
||||
do {
|
||||
LOG.debug("Waiting for the replica " + hrl.getRegion() + " to come up");
|
||||
Thread.sleep(10000); // wait for the detection/recovery
|
||||
hrl = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1);
|
||||
i++;
|
||||
} while ((hrl == null || hrl.getServerName().equals(oldServer)) && i < 3);
|
||||
assertNotEquals(3, i);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFailedReplicaAssigment() throws InterruptedException, IOException {
|
||||
//using our rigged master, to force a failed meta replica assignment
|
||||
TEST_UTIL.getMiniHBaseCluster().getConfiguration().setClass(HConstants.MASTER_IMPL, BrokenMetaReplicaMaster.class, HMaster.class);
|
||||
TEST_UTIL.getMiniHBaseCluster().stopMaster(0).join();
|
||||
HMaster newMaster = TEST_UTIL.getMiniHBaseCluster().startMaster().getMaster();
|
||||
//waiting for master to come up
|
||||
TEST_UTIL.waitFor(30000, () -> newMaster.isInitialized());
|
||||
TEST_UTIL.getMiniHBaseCluster().getConfiguration().unset(HConstants.MASTER_IMPL);
|
||||
|
||||
|
||||
AssignmentManager am = newMaster.getAssignmentManager();
|
||||
//showing one of the replicas got assigned
|
||||
RegionInfo metaReplicaHri = RegionReplicaUtil.getRegionInfoForReplica(
|
||||
RegionInfoBuilder.FIRST_META_REGIONINFO, 1);
|
||||
RegionStateNode metaReplicaRegionNode = am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri);
|
||||
Assert.assertNotNull(metaReplicaRegionNode.getRegionLocation());
|
||||
//showing one of the replicas failed to be assigned
|
||||
RegionInfo metaReplicaHri2 = RegionReplicaUtil.getRegionInfoForReplica(
|
||||
RegionInfoBuilder.FIRST_META_REGIONINFO, 2);
|
||||
RegionStateNode metaReplicaRegionNode2 = am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri2);
|
||||
Assert.assertNull(metaReplicaRegionNode2.getRegionLocation());
|
||||
|
||||
//showing master is active and running
|
||||
Assert.assertFalse(newMaster.isStopping());
|
||||
Assert.assertFalse(newMaster.isStopped());
|
||||
Assert.assertTrue(newMaster.isActiveMaster());
|
||||
}
|
||||
|
||||
public static class BrokenTransitRegionStateProcedure extends TransitRegionStateProcedure {
|
||||
protected BrokenTransitRegionStateProcedure() {
|
||||
//super(env, hri, assignCandidate, forceNewPlan, type);
|
||||
super(null, null, null, false,TransitionType.ASSIGN);
|
||||
}
|
||||
}
|
||||
|
||||
public static class BrokenMetaReplicaMaster extends HMaster{
|
||||
public BrokenMetaReplicaMaster(final Configuration conf) throws IOException {
|
||||
super(conf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AssignmentManager createAssignmentManager(MasterServices master) {
|
||||
return new BrokenMasterMetaAssignmentManager(master);
|
||||
}
|
||||
}
|
||||
|
||||
public static class BrokenMasterMetaAssignmentManager extends AssignmentManager{
|
||||
MasterServices master;
|
||||
public BrokenMasterMetaAssignmentManager(final MasterServices master) {
|
||||
super(master);
|
||||
this.master = master;
|
||||
}
|
||||
|
||||
public Future<byte[]> assignAsync(RegionInfo regionInfo, ServerName sn) throws IOException {
|
||||
RegionStateNode regionNode = getRegionStates().getOrCreateRegionStateNode(regionInfo);
|
||||
if (regionNode.getRegionInfo().getReplicaId() == 2) {
|
||||
regionNode.setProcedure(new BrokenTransitRegionStateProcedure());
|
||||
}
|
||||
return super.assignAsync(regionInfo, sn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.client;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertThrows;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.TableNotFoundException;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.testclassification.MiscTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
||||
|
||||
@Category({ MiscTests.class, MediumTests.class })
|
||||
public class TestMetaWithReplicasBasic extends MetaWithReplicasTestBase {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestMetaWithReplicasBasic.class);
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
startCluster();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMetaHTDReplicaCount() throws Exception {
|
||||
assertEquals(3,
|
||||
TEST_UTIL.getAdmin().getDescriptor(TableName.META_TABLE_NAME).getRegionReplication());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testZookeeperNodesForReplicas() throws Exception {
|
||||
// Checks all the znodes exist when meta's replicas are enabled
|
||||
ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
String baseZNode =
|
||||
conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
|
||||
String primaryMetaZnode =
|
||||
ZNodePaths.joinZNode(baseZNode, conf.get("zookeeper.znode.metaserver", "meta-region-server"));
|
||||
// check that the data in the znode is parseable (this would also mean the znode exists)
|
||||
byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
|
||||
ProtobufUtil.toServerName(data);
|
||||
for (int i = 1; i < 3; i++) {
|
||||
String secZnode = ZNodePaths.joinZNode(baseZNode,
|
||||
conf.get("zookeeper.znode.metaserver", "meta-region-server") + "-" + i);
|
||||
String str = zkw.getZNodePaths().getZNodeForReplica(i);
|
||||
assertTrue(str.equals(secZnode));
|
||||
// check that the data in the znode is parseable (this would also mean the znode exists)
|
||||
data = ZKUtil.getData(zkw, secZnode);
|
||||
ProtobufUtil.toServerName(data);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAccessingUnknownTables() throws Exception {
|
||||
Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
|
||||
conf.setBoolean(HConstants.USE_META_REPLICAS, true);
|
||||
Table table = TEST_UTIL.getConnection().getTable(name.getTableName());
|
||||
Get get = new Get(Bytes.toBytes("foo"));
|
||||
assertThrows(TableNotFoundException.class, () -> table.get(get));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,164 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.client;
|
||||
|
||||
import static org.junit.Assert.assertArrayEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.HRegionLocation;
|
||||
import org.apache.hadoop.hbase.MetaTableAccessor;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.testclassification.MiscTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
||||
|
||||
@Category({ MiscTests.class, MediumTests.class })
|
||||
public class TestMetaWithReplicasShutdownHandling extends MetaWithReplicasTestBase {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestMetaWithReplicasShutdownHandling.class);
|
||||
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(TestMetaWithReplicasShutdownHandling.class);
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
startCluster();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShutdownHandling() throws Exception {
|
||||
// This test creates a table, flushes the meta (with 3 replicas), kills the
|
||||
// server holding the primary meta replica. Then it does a put/get into/from
|
||||
// the test table. The put/get operations would use the replicas to locate the
|
||||
// location of the test table's region
|
||||
shutdownMetaAndDoValidations(TEST_UTIL);
|
||||
}
|
||||
|
||||
public static void shutdownMetaAndDoValidations(HBaseTestingUtility util) throws Exception {
|
||||
// This test creates a table, flushes the meta (with 3 replicas), kills the
|
||||
// server holding the primary meta replica. Then it does a put/get into/from
|
||||
// the test table. The put/get operations would use the replicas to locate the
|
||||
// location of the test table's region
|
||||
ZKWatcher zkw = util.getZooKeeperWatcher();
|
||||
Configuration conf = util.getConfiguration();
|
||||
conf.setBoolean(HConstants.USE_META_REPLICAS, true);
|
||||
|
||||
String baseZNode =
|
||||
conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
|
||||
String primaryMetaZnode =
|
||||
ZNodePaths.joinZNode(baseZNode, conf.get("zookeeper.znode.metaserver", "meta-region-server"));
|
||||
byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
|
||||
ServerName primary = ProtobufUtil.toServerName(data);
|
||||
LOG.info("Primary=" + primary.toString());
|
||||
|
||||
TableName TABLE = TableName.valueOf("testShutdownHandling");
|
||||
byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
|
||||
if (util.getAdmin().tableExists(TABLE)) {
|
||||
util.getAdmin().disableTable(TABLE);
|
||||
util.getAdmin().deleteTable(TABLE);
|
||||
}
|
||||
byte[] row = Bytes.toBytes("test");
|
||||
ServerName master = null;
|
||||
try (Connection c = ConnectionFactory.createConnection(util.getConfiguration())) {
|
||||
try (Table htable = util.createTable(TABLE, FAMILIES)) {
|
||||
util.getAdmin().flush(TableName.META_TABLE_NAME);
|
||||
Thread.sleep(
|
||||
conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 30000) * 6);
|
||||
List<RegionInfo> regions = MetaTableAccessor.getTableRegions(c, TABLE);
|
||||
HRegionLocation hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
|
||||
// Ensure that the primary server for test table is not the same one as the primary
|
||||
// of the meta region since we will be killing the srv holding the meta's primary...
|
||||
// We want to be able to write to the test table even when the meta is not present ..
|
||||
// If the servers are the same, then move the test table's region out of the server
|
||||
// to another random server
|
||||
if (hrl.getServerName().equals(primary)) {
|
||||
util.getAdmin().move(hrl.getRegion().getEncodedNameAsBytes());
|
||||
// wait for the move to complete
|
||||
do {
|
||||
Thread.sleep(10);
|
||||
hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
|
||||
} while (primary.equals(hrl.getServerName()));
|
||||
util.getAdmin().flush(TableName.META_TABLE_NAME);
|
||||
Thread.sleep(
|
||||
conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 30000) * 3);
|
||||
}
|
||||
// Ensure all metas are not on same hbase:meta replica=0 server!
|
||||
|
||||
master = util.getHBaseClusterInterface().getClusterMetrics().getMasterName();
|
||||
// kill the master so that regionserver recovery is not triggered at all
|
||||
// for the meta server
|
||||
LOG.info("Stopping master=" + master.toString());
|
||||
util.getHBaseClusterInterface().stopMaster(master);
|
||||
util.getHBaseClusterInterface().waitForMasterToStop(master, 60000);
|
||||
LOG.info("Master " + master + " stopped!");
|
||||
if (!master.equals(primary)) {
|
||||
util.getHBaseClusterInterface().killRegionServer(primary);
|
||||
util.getHBaseClusterInterface().waitForRegionServerToStop(primary, 60000);
|
||||
}
|
||||
c.clearRegionLocationCache();
|
||||
}
|
||||
LOG.info("Running GETs");
|
||||
try (Table htable = c.getTable(TABLE)) {
|
||||
Put put = new Put(row);
|
||||
put.addColumn(Bytes.toBytes("foo"), row, row);
|
||||
BufferedMutator m = c.getBufferedMutator(TABLE);
|
||||
m.mutate(put);
|
||||
m.flush();
|
||||
// Try to do a get of the row that was just put
|
||||
Result r = htable.get(new Get(row));
|
||||
assertTrue(Arrays.equals(r.getRow(), row));
|
||||
// now start back the killed servers and disable use of replicas. That would mean
|
||||
// calls go to the primary
|
||||
LOG.info("Starting Master");
|
||||
util.getHBaseClusterInterface().startMaster(master.getHostname(), 0);
|
||||
util.getHBaseClusterInterface().startRegionServer(primary.getHostname(), 0);
|
||||
util.getHBaseClusterInterface().waitForActiveAndReadyMaster();
|
||||
LOG.info("Master active!");
|
||||
c.clearRegionLocationCache();
|
||||
}
|
||||
}
|
||||
conf.setBoolean(HConstants.USE_META_REPLICAS, false);
|
||||
LOG.info("Running GETs no replicas");
|
||||
try (Connection c = ConnectionFactory.createConnection(conf);
|
||||
Table htable = c.getTable(TABLE)) {
|
||||
Result r = htable.get(new Get(row));
|
||||
assertArrayEquals(row, r.getRow());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.client;
|
||||
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.HRegionLocation;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.testclassification.MiscTests;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
@Category({ MiscTests.class, MediumTests.class })
|
||||
public class TestShutdownOfMetaReplicaHolder extends MetaWithReplicasTestBase {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestShutdownOfMetaReplicaHolder.class);
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(TestShutdownOfMetaReplicaHolder.class);
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
startCluster();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShutdownOfReplicaHolder() throws Exception {
|
||||
// checks that the when the server holding meta replica is shut down, the meta replica
|
||||
// can be recovered
|
||||
try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
|
||||
RegionLocator locator = conn.getRegionLocator(TableName.META_TABLE_NAME)) {
|
||||
HRegionLocation hrl = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1);
|
||||
ServerName oldServer = hrl.getServerName();
|
||||
TEST_UTIL.getHBaseClusterInterface().killRegionServer(oldServer);
|
||||
LOG.debug("Waiting for the replica {} to come up", hrl.getRegion());
|
||||
TEST_UTIL.waitFor(30000, () -> {
|
||||
HRegionLocation loc = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1);
|
||||
return !loc.getServerName().equals(oldServer);
|
||||
});
|
||||
LOG.debug("Replica {} is online on {}, old server is {}", hrl.getRegion(),
|
||||
locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1).getServerName(),
|
||||
oldServer);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue