HBASE-21753 Support getting the locations for all the replicas of a region
This commit is contained in:
Duo Zhang
parent
fa3946fbea
commit
dfad304ddb
@ -20,36 +20,29 @@ package org.apache.hadoop.hbase.client;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HRegionInfo;
|
||||
import org.apache.hadoop.hbase.HRegionLocation;
|
||||
import org.apache.hadoop.hbase.MetaTableAccessor;
|
||||
import org.apache.hadoop.hbase.RegionLocations;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.util.Pair;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.apache.yetus.audience.InterfaceStability;
|
||||
|
||||
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
/**
|
||||
* An implementation of {@link RegionLocator}. Used to view region location information for a single
|
||||
* HBase table. Lightweight. Get as needed and just close when done. Instances of this class SHOULD
|
||||
* NOT be constructed directly. Obtain an instance via {@link Connection}. See
|
||||
* {@link ConnectionFactory} class comment for an example of how.
|
||||
*
|
||||
* <p> This class is thread safe
|
||||
* <p/>
|
||||
* This class is thread safe
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Stable
|
||||
public class HRegionLocator implements RegionLocator {
|
||||
|
||||
private final TableName tableName;
|
||||
private final ClusterConnection connection;
|
||||
private final ConnectionImplementation connection;
|
||||
|
||||
public HRegionLocator(TableName tableName, ClusterConnection connection) {
|
||||
public HRegionLocator(TableName tableName, ConnectionImplementation connection) {
|
||||
this.connection = connection;
|
||||
this.tableName = tableName;
|
||||
}
|
||||
@ -63,22 +56,18 @@ public class HRegionLocator implements RegionLocator {
|
||||
// persistent state, so there is no need to do anything here.
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public HRegionLocation getRegionLocation(final byte [] row)
|
||||
throws IOException {
|
||||
return connection.getRegionLocation(tableName, row, false);
|
||||
public HRegionLocation getRegionLocation(byte[] row, int replicaId, boolean reload)
|
||||
throws IOException {
|
||||
return connection.locateRegion(tableName, row, !reload, true, replicaId)
|
||||
.getRegionLocation(replicaId);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public HRegionLocation getRegionLocation(final byte [] row, boolean reload)
|
||||
throws IOException {
|
||||
return connection.getRegionLocation(tableName, row, reload);
|
||||
public List<HRegionLocation> getRegionLocations(byte[] row, boolean reload) throws IOException {
|
||||
RegionLocations locs =
|
||||
connection.locateRegion(tableName, row, !reload, true, RegionInfo.DEFAULT_REPLICA_ID);
|
||||
return Arrays.asList(locs.getRegionLocations());
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -94,42 +83,9 @@ public class HRegionLocator implements RegionLocator {
|
||||
return regions;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public byte[][] getStartKeys() throws IOException {
|
||||
return getStartEndKeys().getFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public byte[][] getEndKeys() throws IOException {
|
||||
return getStartEndKeys().getSecond();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public Pair<byte[][], byte[][]> getStartEndKeys() throws IOException {
|
||||
return getStartEndKeys(listRegionLocations());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
Pair<byte[][], byte[][]> getStartEndKeys(List<RegionLocations> regions) {
|
||||
final byte[][] startKeyList = new byte[regions.size()][];
|
||||
final byte[][] endKeyList = new byte[regions.size()][];
|
||||
|
||||
for (int i = 0; i < regions.size(); i++) {
|
||||
HRegionInfo region = regions.get(i).getRegionLocation().getRegionInfo();
|
||||
startKeyList[i] = region.getStartKey();
|
||||
endKeyList[i] = region.getEndKey();
|
||||
}
|
||||
|
||||
return new Pair<>(startKeyList, endKeyList);
|
||||
public void clearRegionLocationCache() {
|
||||
connection.clearRegionCache(tableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -137,14 +93,15 @@ public class HRegionLocator implements RegionLocator {
|
||||
return this.tableName;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
List<RegionLocations> listRegionLocations() throws IOException {
|
||||
private List<RegionLocations> listRegionLocations() throws IOException {
|
||||
final List<RegionLocations> regions = new ArrayList<>();
|
||||
MetaTableAccessor.Visitor visitor = new MetaTableAccessor.TableVisitorBase(tableName) {
|
||||
@Override
|
||||
public boolean visitInternal(Result result) throws IOException {
|
||||
RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
|
||||
if (locations == null) return true;
|
||||
if (locations == null) {
|
||||
return true;
|
||||
}
|
||||
regions.add(locations);
|
||||
return true;
|
||||
}
|
||||
@ -153,7 +110,4 @@ public class HRegionLocator implements RegionLocator {
|
||||
return regions;
|
||||
}
|
||||
|
||||
public Configuration getConfiguration() {
|
||||
return connection.getConfiguration();
|
||||
}
|
||||
}
|
||||
|
||||
@ -157,7 +157,7 @@ public class HTable implements Table {
|
||||
* @param pool ExecutorService to be used.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
protected HTable(final ClusterConnection connection,
|
||||
protected HTable(final ConnectionImplementation connection,
|
||||
final TableBuilderBase builder,
|
||||
final RpcRetryingCallerFactory rpcCallerFactory,
|
||||
final RpcControllerFactory rpcControllerFactory,
|
||||
|
||||
@ -21,16 +21,15 @@ package org.apache.hadoop.hbase.client;
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.hadoop.hbase.HRegionLocation;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.apache.hadoop.hbase.util.Pair;
|
||||
|
||||
/**
|
||||
* Used to view region location information for a single HBase table.
|
||||
* Obtain an instance from an {@link Connection}.
|
||||
*
|
||||
* Used to view region location information for a single HBase table. Obtain an instance from an
|
||||
* {@link Connection}.
|
||||
* @see ConnectionFactory
|
||||
* @see Connection
|
||||
* @see Table
|
||||
@ -44,7 +43,9 @@ public interface RegionLocator extends Closeable {
|
||||
* @return Location of the row.
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
*/
|
||||
public HRegionLocation getRegionLocation(final byte [] row) throws IOException;
|
||||
default HRegionLocation getRegionLocation(byte[] row) throws IOException {
|
||||
return getRegionLocation(row, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the region on which the given row is being served.
|
||||
@ -53,16 +54,65 @@ public interface RegionLocator extends Closeable {
|
||||
* @return Location of the row.
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
*/
|
||||
public HRegionLocation getRegionLocation(final byte [] row, boolean reload)
|
||||
throws IOException;
|
||||
default HRegionLocation getRegionLocation(byte[] row, boolean reload) throws IOException {
|
||||
return getRegionLocation(row, RegionInfo.DEFAULT_REPLICA_ID, reload);
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the region with the given replica id on which the given row is being served.
|
||||
* @param row Row to find.
|
||||
* @param replicaId the replica id
|
||||
* @return Location of the row.
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
*/
|
||||
default HRegionLocation getRegionLocation(byte[] row, int replicaId) throws IOException {
|
||||
return getRegionLocation(row, replicaId, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the region with the given replica id on which the given row is being served.
|
||||
* @param row Row to find.
|
||||
* @param replicaId the replica id
|
||||
* @param reload true to reload information or false to use cached information
|
||||
* @return Location of the row.
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
*/
|
||||
HRegionLocation getRegionLocation(byte[] row, int replicaId, boolean reload) throws IOException;
|
||||
|
||||
/**
|
||||
* Find all the replicas for the region on which the given row is being served.
|
||||
* @param row Row to find.
|
||||
* @return Locations for all the replicas of the row.
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
*/
|
||||
default List<HRegionLocation> getRegionLocations(byte[] row) throws IOException {
|
||||
return getRegionLocations(row, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find all the replicas for the region on which the given row is being served.
|
||||
* @param row Row to find.
|
||||
* @param reload true to reload information or false to use cached information
|
||||
* @return Locations for all the replicas of the row.
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
*/
|
||||
List<HRegionLocation> getRegionLocations(byte[] row, boolean reload) throws IOException;
|
||||
|
||||
/**
|
||||
* Clear all the entries in the region location cache.
|
||||
* <p/>
|
||||
* This may cause performance issue so use it with caution.
|
||||
*/
|
||||
void clearRegionLocationCache();
|
||||
|
||||
/**
|
||||
* Retrieves all of the regions associated with this table.
|
||||
* <p/>
|
||||
* Notice that the location for region replicas other than the default replica are also returned.
|
||||
* @return a {@link List} of all regions associated with this table.
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
*/
|
||||
public List<HRegionLocation> getAllRegionLocations()
|
||||
throws IOException;
|
||||
List<HRegionLocation> getAllRegionLocations() throws IOException;
|
||||
|
||||
/**
|
||||
* Gets the starting row key for every region in the currently open table.
|
||||
@ -71,7 +121,9 @@ public interface RegionLocator extends Closeable {
|
||||
* @return Array of region starting row keys
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
*/
|
||||
public byte [][] getStartKeys() throws IOException;
|
||||
default byte[][] getStartKeys() throws IOException {
|
||||
return getStartEndKeys().getFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the ending row key for every region in the currently open table.
|
||||
@ -80,17 +132,30 @@ public interface RegionLocator extends Closeable {
|
||||
* @return Array of region ending row keys
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
*/
|
||||
public byte[][] getEndKeys() throws IOException;
|
||||
default byte[][] getEndKeys() throws IOException {
|
||||
return getStartEndKeys().getSecond();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the starting and ending row keys for every region in the currently
|
||||
* open table.
|
||||
* Gets the starting and ending row keys for every region in the currently open table.
|
||||
* <p>
|
||||
* This is mainly useful for the MapReduce integration.
|
||||
* @return Pair of arrays of region starting and ending row keys
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
*/
|
||||
public Pair<byte[][],byte[][]> getStartEndKeys() throws IOException;
|
||||
default Pair<byte[][], byte[][]> getStartEndKeys() throws IOException {
|
||||
List<HRegionLocation> regions = getAllRegionLocations().stream()
|
||||
.filter(loc -> RegionReplicaUtil.isDefaultReplica(loc.getRegion()))
|
||||
.collect(Collectors.toList());
|
||||
byte[][] startKeys = new byte[regions.size()][];
|
||||
byte[][] endKeys = new byte[regions.size()][];
|
||||
for (int i = 0, n = regions.size(); i < n; i++) {
|
||||
RegionInfo region = regions.get(i).getRegion();
|
||||
startKeys[i] = region.getStartKey();
|
||||
endKeys[i] = region.getEndKey();
|
||||
}
|
||||
return Pair.newPair(startKeys, endKeys);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the fully qualified table name instance of this table.
|
||||
|
||||
@ -6352,6 +6352,19 @@ public class TestFromClientSide {
|
||||
assertEquals(4, count); // 003 004 005 006
|
||||
}
|
||||
|
||||
private static Pair<byte[][], byte[][]> getStartEndKeys(List<RegionLocations> regions) {
|
||||
final byte[][] startKeyList = new byte[regions.size()][];
|
||||
final byte[][] endKeyList = new byte[regions.size()][];
|
||||
|
||||
for (int i = 0; i < regions.size(); i++) {
|
||||
RegionInfo region = regions.get(i).getRegionLocation().getRegion();
|
||||
startKeyList[i] = region.getStartKey();
|
||||
endKeyList[i] = region.getEndKey();
|
||||
}
|
||||
|
||||
return new Pair<>(startKeyList, endKeyList);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetStartEndKeysWithRegionReplicas() throws IOException {
|
||||
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
|
||||
@ -6376,7 +6389,7 @@ public class TestFromClientSide {
|
||||
regionLocations.add(new RegionLocations(arr));
|
||||
}
|
||||
|
||||
Pair<byte[][], byte[][]> startEndKeys = locator.getStartEndKeys(regionLocations);
|
||||
Pair<byte[][], byte[][]> startEndKeys = getStartEndKeys(regionLocations);
|
||||
|
||||
assertEquals(KEYS.length + 1, startEndKeys.getFirst().length);
|
||||
|
||||
|
||||
@ -0,0 +1,166 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.client;
|
||||
|
||||
import static org.junit.Assert.assertArrayEquals;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNull;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.HRegionLocation;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.regionserver.Region;
|
||||
import org.apache.hadoop.hbase.testclassification.ClientTests;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.Pair;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
|
||||
@Category({ MediumTests.class, ClientTests.class })
|
||||
public class TestRegionLocator {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestRegionLocator.class);
|
||||
|
||||
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
|
||||
|
||||
private static TableName TABLE_NAME = TableName.valueOf("Locator");
|
||||
|
||||
private static byte[] FAMILY = Bytes.toBytes("family");
|
||||
|
||||
private static int REGION_REPLICATION = 3;
|
||||
|
||||
private static byte[][] SPLIT_KEYS;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
UTIL.startMiniCluster(3);
|
||||
TableDescriptor td =
|
||||
TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(REGION_REPLICATION)
|
||||
.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build();
|
||||
SPLIT_KEYS = new byte[9][];
|
||||
for (int i = 0; i < 9; i++) {
|
||||
SPLIT_KEYS[i] = Bytes.toBytes(Integer.toString(i + 1));
|
||||
}
|
||||
UTIL.getAdmin().createTable(td, SPLIT_KEYS);
|
||||
UTIL.waitTableAvailable(TABLE_NAME);
|
||||
UTIL.getAdmin().balancerSwitch(false, true);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDown() throws Exception {
|
||||
UTIL.shutdownMiniCluster();
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDownAfterTest() throws IOException {
|
||||
try (RegionLocator locator = UTIL.getConnection().getRegionLocator(TABLE_NAME)) {
|
||||
locator.clearRegionLocationCache();
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] getStartKey(int index) {
|
||||
return index == 0 ? HConstants.EMPTY_START_ROW : SPLIT_KEYS[index - 1];
|
||||
}
|
||||
|
||||
private byte[] getEndKey(int index) {
|
||||
return index == SPLIT_KEYS.length ? HConstants.EMPTY_END_ROW : SPLIT_KEYS[index];
|
||||
}
|
||||
|
||||
private void assertStartKeys(byte[][] startKeys) {
|
||||
assertEquals(SPLIT_KEYS.length + 1, startKeys.length);
|
||||
for (int i = 0; i < startKeys.length; i++) {
|
||||
assertArrayEquals(getStartKey(i), startKeys[i]);
|
||||
}
|
||||
}
|
||||
|
||||
private void assertEndKeys(byte[][] endKeys) {
|
||||
assertEquals(SPLIT_KEYS.length + 1, endKeys.length);
|
||||
for (int i = 0; i < endKeys.length; i++) {
|
||||
assertArrayEquals(getEndKey(i), endKeys[i]);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStartEndKeys() throws IOException {
|
||||
try (RegionLocator locator = UTIL.getConnection().getRegionLocator(TABLE_NAME)) {
|
||||
assertStartKeys(locator.getStartKeys());
|
||||
assertEndKeys(locator.getEndKeys());
|
||||
Pair<byte[][], byte[][]> startEndKeys = locator.getStartEndKeys();
|
||||
assertStartKeys(startEndKeys.getFirst());
|
||||
assertEndKeys(startEndKeys.getSecond());
|
||||
}
|
||||
}
|
||||
|
||||
private void assertRegionLocation(HRegionLocation loc, int index, int replicaId) {
|
||||
RegionInfo region = loc.getRegion();
|
||||
byte[] startKey = getStartKey(index);
|
||||
assertArrayEquals(startKey, region.getStartKey());
|
||||
assertArrayEquals(getEndKey(index), region.getEndKey());
|
||||
assertEquals(replicaId, region.getReplicaId());
|
||||
ServerName expected =
|
||||
UTIL.getMiniHBaseCluster().getRegionServerThreads().stream().map(t -> t.getRegionServer())
|
||||
.filter(rs -> rs.getRegions(TABLE_NAME).stream().map(Region::getRegionInfo)
|
||||
.anyMatch(r -> r.containsRow(startKey) && r.getReplicaId() == replicaId))
|
||||
.findFirst().get().getServerName();
|
||||
assertEquals(expected, loc.getServerName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetRegionLocation() throws IOException {
|
||||
try (RegionLocator locator = UTIL.getConnection().getRegionLocator(TABLE_NAME)) {
|
||||
for (int i = 0; i <= SPLIT_KEYS.length; i++) {
|
||||
for (int replicaId = 0; replicaId < REGION_REPLICATION; replicaId++) {
|
||||
assertRegionLocation(locator.getRegionLocation(getStartKey(i), replicaId), i, replicaId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAllRegionLocations() throws IOException {
|
||||
try (RegionLocator locator = UTIL.getConnection().getRegionLocator(TABLE_NAME)) {
|
||||
List<HRegionLocation> locs = locator.getAllRegionLocations();
|
||||
assertEquals(REGION_REPLICATION * (SPLIT_KEYS.length + 1), locs.size());
|
||||
Collections.sort(locs, (l1, l2) -> {
|
||||
int c = Bytes.compareTo(l1.getRegion().getStartKey(), l2.getRegion().getStartKey());
|
||||
if (c != 0) {
|
||||
return c;
|
||||
}
|
||||
return Integer.compare(l1.getRegion().getReplicaId(), l2.getRegion().getReplicaId());
|
||||
});
|
||||
for (int i = 0; i <= SPLIT_KEYS.length; i++) {
|
||||
for (int replicaId = 0; replicaId < REGION_REPLICATION; replicaId++) {
|
||||
assertRegionLocation(locs.get(i * REGION_REPLICATION + replicaId), i, replicaId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user