Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-21770 Should deal with meta table in HRegionLocator.getAllRegionLocations

Browse Source 
Signed-off-by: Guanghao Zhang <zghao@apache.org>
This commit is contained in:
zhangduo 2019-01-24 22:34:46 +08:00 committed by Duo Zhang
parent 67ba050fce
commit de31d56c2d
4 changed files with 127 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hbase-client/src/main/java/org/apache/hadoop/hbase/client/HRegionLocator.java
View File

@ -21,7 +21,9 @@ package org.apache.hadoop.hbase.client;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.RegionLocations;
@ -72,7 +74,6 @@ public class HRegionLocator implements RegionLocator {
@Override
public List<HRegionLocation> getAllRegionLocations() throws IOException {
TableName tableName = getName();
ArrayList<HRegionLocation> regions = new ArrayList<>();
for (RegionLocations locations : listRegionLocations()) {
for (HRegionLocation location : locations.getRegionLocations()) {
@ -94,6 +95,10 @@ public class HRegionLocator implements RegionLocator {
}
private List<RegionLocations> listRegionLocations() throws IOException {
if (TableName.isMetaTableName(tableName)) {
return Collections
.singletonList(connection.locateRegion(tableName, HConstants.EMPTY_START_ROW, false, true));
}
final List<RegionLocations> regions = new ArrayList<>();
MetaTableAccessor.Visitor visitor = new MetaTableAccessor.TableVisitorBase(tableName) {
@Override
@ -109,5 +114,4 @@ public class HRegionLocator implements RegionLocator {
MetaTableAccessor.scanMetaForTableRegions(connection, visitor, tableName);
return regions;
}
}

91
hbase-server/src/test/java/org/apache/hadoop/hbase/client/AbstractTestRegionLocator.java
View File

@ -47,6 +47,7 @@ public abstract class AbstractTestRegionLocator {
protected static byte[][] SPLIT_KEYS;
protected static void startClusterAndCreateTable() throws Exception {
UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, REGION_REPLICATION);
UTIL.startMiniCluster(3);
TableDescriptor td =
TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(REGION_REPLICATION)
@ -57,12 +58,17 @@ public abstract class AbstractTestRegionLocator {
}
UTIL.getAdmin().createTable(td, SPLIT_KEYS);
UTIL.waitTableAvailable(TABLE_NAME);
try (AsyncRegistry registry = AsyncRegistryFactory.getRegistry(UTIL.getConfiguration())) {
RegionReplicaTestHelper.waitUntilAllMetaReplicasHavingRegionLocation(registry,
REGION_REPLICATION);
}
UTIL.getAdmin().balancerSwitch(false, true);
}
@After
public void tearDownAfterTest() throws IOException {
clearCache();
clearCache(TABLE_NAME);
clearCache(TableName.META_TABLE_NAME);
}
private byte[] getStartKey(int index) {
@ -89,9 +95,9 @@ public abstract class AbstractTestRegionLocator {
@Test
public void testStartEndKeys() throws IOException {
assertStartKeys(getStartKeys());
assertEndKeys(getEndKeys());
Pair<byte[][], byte[][]> startEndKeys = getStartEndKeys();
assertStartKeys(getStartKeys(TABLE_NAME));
assertEndKeys(getEndKeys(TABLE_NAME));
Pair<byte[][], byte[][]> startEndKeys = getStartEndKeys(TABLE_NAME);
assertStartKeys(startEndKeys.getFirst());
assertEndKeys(startEndKeys.getSecond());
}
@ -102,19 +108,24 @@ public abstract class AbstractTestRegionLocator {
assertArrayEquals(startKey, region.getStartKey());
assertArrayEquals(getEndKey(index), region.getEndKey());
assertEquals(replicaId, region.getReplicaId());
ServerName expected =
UTIL.getMiniHBaseCluster().getRegionServerThreads().stream().map(t -> t.getRegionServer())
.filter(rs -> rs.getRegions(TABLE_NAME).stream().map(Region::getRegionInfo)
.anyMatch(r -> r.containsRow(startKey) && r.getReplicaId() == replicaId))
.findFirst().get().getServerName();
ServerName expected = findRegionLocation(TABLE_NAME, region.getStartKey(), replicaId);
assertEquals(expected, loc.getServerName());
}
private ServerName findRegionLocation(TableName tableName, byte[] startKey, int replicaId) {
return UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
.map(t -> t.getRegionServer())
.filter(rs -> rs.getRegions(tableName).stream().map(Region::getRegionInfo)
.anyMatch(r -> r.containsRow(startKey) && r.getReplicaId() == replicaId))
.findFirst().get().getServerName();
}
@Test
public void testGetRegionLocation() throws IOException {
for (int i = 0; i <= SPLIT_KEYS.length; i++) {
for (int replicaId = 0; replicaId < REGION_REPLICATION; replicaId++) {
assertRegionLocation(getRegionLocation(getStartKey(i), replicaId), i, replicaId);
assertRegionLocation(getRegionLocation(TABLE_NAME, getStartKey(i), replicaId), i,
replicaId);
}
}
}
@ -122,7 +133,7 @@ public abstract class AbstractTestRegionLocator {
@Test
public void testGetRegionLocations() throws IOException {
for (int i = 0; i <= SPLIT_KEYS.length; i++) {
List<HRegionLocation> locs = getRegionLocations(getStartKey(i));
List<HRegionLocation> locs = getRegionLocations(TABLE_NAME, getStartKey(i));
assertEquals(REGION_REPLICATION, locs.size());
for (int replicaId = 0; replicaId < REGION_REPLICATION; replicaId++) {
assertRegionLocation(locs.get(replicaId), i, replicaId);
@ -132,7 +143,7 @@ public abstract class AbstractTestRegionLocator {
@Test
public void testGetAllRegionLocations() throws IOException {
List<HRegionLocation> locs = getAllRegionLocations();
List<HRegionLocation> locs = getAllRegionLocations(TABLE_NAME);
assertEquals(REGION_REPLICATION * (SPLIT_KEYS.length + 1), locs.size());
Collections.sort(locs, (l1, l2) -> {
int c = Bytes.compareTo(l1.getRegion().getStartKey(), l2.getRegion().getStartKey());
@ -148,18 +159,60 @@ public abstract class AbstractTestRegionLocator {
}
}
protected abstract byte[][] getStartKeys() throws IOException;
private void assertMetaStartOrEndKeys(byte[][] keys) {
assertEquals(1, keys.length);
assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, keys[0]);
}
protected abstract byte[][] getEndKeys() throws IOException;
private void assertMetaRegionLocation(HRegionLocation loc, int replicaId) {
RegionInfo region = loc.getRegion();
assertArrayEquals(HConstants.EMPTY_START_ROW, region.getStartKey());
assertArrayEquals(HConstants.EMPTY_END_ROW, region.getEndKey());
assertEquals(replicaId, region.getReplicaId());
ServerName expected =
findRegionLocation(TableName.META_TABLE_NAME, region.getStartKey(), replicaId);
assertEquals(expected, loc.getServerName());
}
protected abstract Pair<byte[][], byte[][]> getStartEndKeys() throws IOException;
private void assertMetaRegionLocations(List<HRegionLocation> locs) {
assertEquals(REGION_REPLICATION, locs.size());
for (int replicaId = 0; replicaId < REGION_REPLICATION; replicaId++) {
assertMetaRegionLocation(locs.get(replicaId), replicaId);
}
}
protected abstract HRegionLocation getRegionLocation(byte[] row, int replicaId)
@Test
public void testMeta() throws IOException {
assertMetaStartOrEndKeys(getStartKeys(TableName.META_TABLE_NAME));
assertMetaStartOrEndKeys(getEndKeys(TableName.META_TABLE_NAME));
Pair<byte[][], byte[][]> startEndKeys = getStartEndKeys(TableName.META_TABLE_NAME);
assertMetaStartOrEndKeys(startEndKeys.getFirst());
assertMetaStartOrEndKeys(startEndKeys.getSecond());
for (int replicaId = 0; replicaId < REGION_REPLICATION; replicaId++) {
assertMetaRegionLocation(
getRegionLocation(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, replicaId),
replicaId);
}
assertMetaRegionLocations(
getRegionLocations(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW));
assertMetaRegionLocations(getAllRegionLocations(TableName.META_TABLE_NAME));
}
protected abstract byte[][] getStartKeys(TableName tableName) throws IOException;
protected abstract byte[][] getEndKeys(TableName tableName) throws IOException;
protected abstract Pair<byte[][], byte[][]> getStartEndKeys(TableName tableName)
throws IOException;
protected abstract List<HRegionLocation> getRegionLocations(byte[] row) throws IOException;
protected abstract HRegionLocation getRegionLocation(TableName tableName, byte[] row,
int replicaId) throws IOException;
protected abstract List<HRegionLocation> getAllRegionLocations() throws IOException;
protected abstract List<HRegionLocation> getRegionLocations(TableName tableName, byte[] row)
throws IOException;
protected abstract void clearCache() throws IOException;
protected abstract List<HRegionLocation> getAllRegionLocations(TableName tableName)
throws IOException;
protected abstract void clearCache(TableName tableName) throws IOException;
}

35
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRegionLocator.java
View File

@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Pair;
@ -42,13 +43,10 @@ public class TestAsyncTableRegionLocator extends AbstractTestRegionLocator {
private static AsyncConnection CONN;
private static AsyncTableRegionLocator LOCATOR;
@BeforeClass
public static void setUp() throws Exception {
startClusterAndCreateTable();
CONN = ConnectionFactory.createAsyncConnection(UTIL.getConfiguration()).get();
LOCATOR = CONN.getRegionLocator(TABLE_NAME);
}
@AfterClass
@ -58,18 +56,19 @@ public class TestAsyncTableRegionLocator extends AbstractTestRegionLocator {
}
@Override
protected byte[][] getStartKeys() throws IOException {
return get(LOCATOR.getStartKeys()).toArray(new byte[0][]);
protected byte[][] getStartKeys(TableName tableName) throws IOException {
return get(CONN.getRegionLocator(tableName).getStartKeys()).toArray(new byte[0][]);
}
@Override
protected byte[][] getEndKeys() throws IOException {
return get(LOCATOR.getEndKeys()).toArray(new byte[0][]);
protected byte[][] getEndKeys(TableName tableName) throws IOException {
return get(CONN.getRegionLocator(tableName).getEndKeys()).toArray(new byte[0][]);
}
@Override
protected Pair<byte[][], byte[][]> getStartEndKeys() throws IOException {
List<Pair<byte[], byte[]>> startEndKeys = get(LOCATOR.getStartEndKeys());
protected Pair<byte[][], byte[][]> getStartEndKeys(TableName tableName) throws IOException {
List<Pair<byte[], byte[]>> startEndKeys =
get(CONN.getRegionLocator(tableName).getStartEndKeys());
byte[][] startKeys = new byte[startEndKeys.size()][];
byte[][] endKeys = new byte[startEndKeys.size()][];
for (int i = 0, n = startEndKeys.size(); i < n; i++) {
@ -81,22 +80,24 @@ public class TestAsyncTableRegionLocator extends AbstractTestRegionLocator {
}
@Override
protected HRegionLocation getRegionLocation(byte[] row, int replicaId) throws IOException {
return get(LOCATOR.getRegionLocation(row, replicaId));
protected HRegionLocation getRegionLocation(TableName tableName, byte[] row, int replicaId)
throws IOException {
return get(CONN.getRegionLocator(tableName).getRegionLocation(row, replicaId));
}
@Override
protected List<HRegionLocation> getRegionLocations(byte[] row) throws IOException {
return get(LOCATOR.getRegionLocations(row));
protected List<HRegionLocation> getRegionLocations(TableName tableName, byte[] row)
throws IOException {
return get(CONN.getRegionLocator(tableName).getRegionLocations(row));
}
@Override
protected List<HRegionLocation> getAllRegionLocations() throws IOException {
return get(LOCATOR.getAllRegionLocations());
protected List<HRegionLocation> getAllRegionLocations(TableName tableName) throws IOException {
return get(CONN.getRegionLocator(tableName).getAllRegionLocations());
}
@Override
protected void clearCache() throws IOException {
LOCATOR.clearRegionLocationCache();
protected void clearCache(TableName tableName) throws IOException {
CONN.getRegionLocator(tableName).clearRegionLocationCache();
}
}

51
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRegionLocator.java
View File

@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Pair;
@ -29,8 +30,6 @@ import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.experimental.categories.Category;
import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
@Category({ MediumTests.class, ClientTests.class })
public class TestRegionLocator extends AbstractTestRegionLocator {
@ -38,52 +37,64 @@ public class TestRegionLocator extends AbstractTestRegionLocator {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestRegionLocator.class);
private static RegionLocator LOCATOR;
@BeforeClass
public static void setUp() throws Exception {
startClusterAndCreateTable();
LOCATOR = UTIL.getConnection().getRegionLocator(TABLE_NAME);
}
@AfterClass
public static void tearDown() throws Exception {
Closeables.close(LOCATOR, true);
UTIL.shutdownMiniCluster();
}
@Override
protected byte[][] getStartKeys() throws IOException {
return LOCATOR.getStartKeys();
protected byte[][] getStartKeys(TableName tableName) throws IOException {
try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
return locator.getStartKeys();
}
}
@Override
protected byte[][] getEndKeys() throws IOException {
return LOCATOR.getEndKeys();
protected byte[][] getEndKeys(TableName tableName) throws IOException {
try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
return locator.getEndKeys();
}
}
@Override
protected Pair<byte[][], byte[][]> getStartEndKeys() throws IOException {
return LOCATOR.getStartEndKeys();
protected Pair<byte[][], byte[][]> getStartEndKeys(TableName tableName) throws IOException {
try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
return locator.getStartEndKeys();
}
}
@Override
protected HRegionLocation getRegionLocation(byte[] row, int replicaId) throws IOException {
return LOCATOR.getRegionLocation(row, replicaId);
protected HRegionLocation getRegionLocation(TableName tableName, byte[] row, int replicaId)
throws IOException {
try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
return locator.getRegionLocation(row, replicaId);
}
}
@Override
protected List<HRegionLocation> getRegionLocations(byte[] row) throws IOException {
return LOCATOR.getRegionLocations(row);
protected List<HRegionLocation> getRegionLocations(TableName tableName, byte[] row)
throws IOException {
try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
return locator.getRegionLocations(row);
}
}
@Override
protected List<HRegionLocation> getAllRegionLocations() throws IOException {
return LOCATOR.getAllRegionLocations();
protected List<HRegionLocation> getAllRegionLocations(TableName tableName) throws IOException {
try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
return locator.getAllRegionLocations();
}
}
@Override
protected void clearCache() throws IOException {
LOCATOR.clearRegionLocationCache();
protected void clearCache(TableName tableName) throws IOException {
try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
locator.clearRegionLocationCache();
}
}
}