Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-16169: Make RegionSizeCalculator scalable 

Signed-off-by: Michael Stack <stack@apache.org>
This commit is contained in:
thiruvel 2016-11-09 17:43:38 -08:00 committed by Michael Stack
parent 48439e5720
commit 80acc2dca5
12 changed files with 1867 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
View File

@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NamespaceNotFoundException;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
@ -1020,6 +1021,25 @@ public interface Admin extends Abortable, Closeable {
*/
ClusterStatus getClusterStatus() throws IOException;
/**
* Get {@link RegionLoad} of all regions hosted on a regionserver.
*
* @param sn region server from which regionload is required.
* @return region load map of all regions hosted on a region server
* @throws IOException if a remote or network exception occurs
*/
Map<byte[], RegionLoad> getRegionLoad(ServerName sn) throws IOException;
/**
* Get {@link RegionLoad} of all regions hosted on a regionserver for a table.
*
* @param sn region server from which regionload is required.
* @param tableName get region load of regions belonging to the table
* @return region load map of all regions of a table hosted on a region server
* @throws IOException if a remote or network exception occurs
*/
Map<byte[], RegionLoad> getRegionLoad(ServerName sn, TableName tableName) throws IOException;
/**
* @return Configuration used by the instance.
*/

20
hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
View File

@ -28,6 +28,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@ -56,6 +57,7 @@ import org.apache.hadoop.hbase.NamespaceNotFoundException;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
@ -1879,6 +1881,24 @@ public class HBaseAdmin implements Admin {
});
}
@Override
public Map<byte[], RegionLoad> getRegionLoad(final ServerName sn) throws IOException {
return getRegionLoad(sn, null);
}
@Override
public Map<byte[], RegionLoad> getRegionLoad(final ServerName sn, final TableName tableName)
throws IOException {
AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
HBaseRpcController controller = rpcControllerFactory.newController();
List<RegionLoad> regionLoads = ProtobufUtil.getRegionLoad(controller, admin, tableName);
Map<byte[], RegionLoad> resultMap = new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
for (RegionLoad regionLoad : regionLoads) {
resultMap.put(regionLoad.getName(), regionLoad);
}
return resultMap;
}
@Override
public Configuration getConfiguration() {
return this.conf;

25
hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
View File

@ -112,6 +112,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRe
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetStoreFileRequest;
@ -1677,6 +1679,29 @@ public final class ProtobufUtil {
}
}
public static List<org.apache.hadoop.hbase.RegionLoad> getRegionLoad(
final RpcController controller, final AdminService.BlockingInterface admin,
final TableName tableName) throws IOException {
GetRegionLoadRequest request = RequestConverter.buildGetRegionLoadRequest(tableName);
GetRegionLoadResponse response;
try {
response = admin.getRegionLoad(controller, request);
} catch (ServiceException se) {
throw getRemoteException(se);
}
return getRegionLoadInfo(response);
}
static List<org.apache.hadoop.hbase.RegionLoad> getRegionLoadInfo(
GetRegionLoadResponse regionLoadResponse) {
List<org.apache.hadoop.hbase.RegionLoad> regionLoadList =
new ArrayList<>(regionLoadResponse.getRegionLoadsCount());
for (RegionLoad regionLoad : regionLoadResponse.getRegionLoadsList()) {
regionLoadList.add(new org.apache.hadoop.hbase.RegionLoad(regionLoad));
}
return regionLoadList;
}
/**
* A helper to close a region given a region name
* using admin protocol.

16
hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
View File

@ -50,6 +50,8 @@ import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest;
@ -750,6 +752,20 @@ public final class RequestConverter {
return builder.build();
}
/**
* Create a protocol buffer GetRegionLoadRequest for all regions/regions of a table.
*
* @param tableName the table for which regionLoad should be obtained from RS
* @return a protocol buffer GetRegionLoadRequest
*/
public static GetRegionLoadRequest buildGetRegionLoadRequest(final TableName tableName) {
GetRegionLoadRequest.Builder builder = GetRegionLoadRequest.newBuilder();
if (tableName != null) {
builder.setTableName(ProtobufUtil.toProtoTableName(tableName));
}
return builder.build();
}
/**
* Create a protocol buffer GetOnlineRegionRequest
*

1678
hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/AdminProtos.java
View File

File diff suppressed because it is too large Load Diff

12
hbase-protocol-shaded/src/main/protobuf/Admin.proto
View File

@ -25,6 +25,7 @@ option java_generic_services = true;
option java_generate_equals_and_hash = true;
option optimize_for = SPEED;
import "ClusterStatus.proto";
import "HBase.proto";
import "WAL.proto";
@ -267,6 +268,14 @@ message UpdateConfigurationRequest {
message UpdateConfigurationResponse {
}
message GetRegionLoadRequest {
optional TableName table_name = 1;
}
message GetRegionLoadResponse {
repeated RegionLoad region_loads = 1;
}
service AdminService {
rpc GetRegionInfo(GetRegionInfoRequest)
returns(GetRegionInfoResponse);
@ -321,4 +330,7 @@ service AdminService {
rpc UpdateConfiguration(UpdateConfigurationRequest)
returns(UpdateConfigurationResponse);
rpc GetRegionLoad(GetRegionLoadRequest)
returns(GetRegionLoadResponse);
}

2
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -1514,7 +1514,7 @@ public class HRegionServer extends HasThread implements
*
* @throws IOException
*/
private RegionLoad createRegionLoad(final Region r, RegionLoad.Builder regionLoadBldr,
RegionLoad createRegionLoad(final Region r, RegionLoad.Builder regionLoadBldr,
RegionSpecifier.Builder regionSpecifier) throws IOException {
byte[] name = r.getRegionInfo().getRegionName();
int stores = 0;

32
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
View File

@ -103,6 +103,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegion
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
@ -160,6 +162,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionActi
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ResultOrException;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionLoad;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameInt64Pair;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier;
@ -1550,6 +1554,34 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
}
}
@Override
@QosPriority(priority=HConstants.ADMIN_QOS)
public GetRegionLoadResponse getRegionLoad(RpcController controller,
GetRegionLoadRequest request) throws ServiceException {
List<Region> regions;
if (request.hasTableName()) {
TableName tableName = ProtobufUtil.toTableName(request.getTableName());
regions = regionServer.getOnlineRegions(tableName);
} else {
regions = regionServer.getOnlineRegions();
}
List<RegionLoad> rLoads = new ArrayList<RegionLoad>(regions.size());
RegionLoad.Builder regionLoadBuilder = ClusterStatusProtos.RegionLoad.newBuilder();
RegionSpecifier.Builder regionSpecifier = RegionSpecifier.newBuilder();
try {
for (Region region : regions) {
rLoads.add(regionServer.createRegionLoad(region, regionLoadBuilder, regionSpecifier));
}
} catch (IOException e) {
throw new ServiceException(e);
}
GetRegionLoadResponse.Builder builder = GetRegionLoadResponse.newBuilder();
builder.addAllRegionLoads(rLoads);
return builder.build();
}
/**
* Get some information of the region server.
*

48
hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSizeCalculator.java
View File

@ -19,21 +19,18 @@ package org.apache.hadoop.hbase.util;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
@ -59,6 +56,7 @@ public class RegionSizeCalculator {
private final Map<byte[], Long> sizeMap = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
static final String ENABLE_REGIONSIZECALCULATOR = "hbase.regionsizecalculator.enable";
private static final long MEGABYTE = 1024L * 1024L;
/**
* Computes size of each region for table and given column families.
@ -95,38 +93,36 @@ public class RegionSizeCalculator {
LOG.info("Calculating region sizes for table \"" + regionLocator.getName() + "\".");
//get regions for table
List<HRegionLocation> tableRegionInfos = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
for (HRegionLocation regionInfo : tableRegionInfos) {
tableRegions.add(regionInfo.getRegionInfo().getRegionName());
}
// Get the servers which host regions of the table
Set<ServerName> tableServers = getRegionServersOfTable(regionLocator);
ClusterStatus clusterStatus = admin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
for (ServerName tableServerName : tableServers) {
Map<byte[], RegionLoad> regionLoads =
admin.getRegionLoad(tableServerName, regionLocator.getName());
for (RegionLoad regionLoad : regionLoads.values()) {
//iterate all cluster regions, filter regions from our table and compute their size
for (ServerName serverName: servers) {
ServerLoad serverLoad = clusterStatus.getLoad(serverName);
for (RegionLoad regionLoad: serverLoad.getRegionsLoad().values()) {
byte[] regionId = regionLoad.getName();
long regionSizeBytes = regionLoad.getStorefileSizeMB() * MEGABYTE;
sizeMap.put(regionId, regionSizeBytes);
if (tableRegions.contains(regionId)) {
long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte;
sizeMap.put(regionId, regionSizeBytes);
if (LOG.isDebugEnabled()) {
LOG.debug("Region " + regionLoad.getNameAsString() + " has size " + regionSizeBytes);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Region " + regionLoad.getNameAsString() + " has size " + regionSizeBytes);
}
}
}
LOG.debug("Region sizes calculated");
}
private Set<ServerName> getRegionServersOfTable(RegionLocator regionLocator)
throws IOException {
Set<ServerName> tableServers = Sets.newHashSet();
for (HRegionLocation regionLocation : regionLocator.getAllRegionLocations()) {
tableServers.add(regionLocation.getServerName());
}
return tableServers;
}
boolean enabled(Configuration configuration) {
return configuration.getBoolean(ENABLE_REGIONSIZECALCULATOR, true);
}

135
hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionLoad.java Normal file
View File

@ -0,0 +1,135 @@
/**
* Copyright The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase;
import static org.junit.Assert.*;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@Category({MiscTests.class, MediumTests.class})
public class TestRegionLoad {
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
private static Admin admin;
private static final TableName TABLE_1 = TableName.valueOf("table_1");
private static final TableName TABLE_2 = TableName.valueOf("table_2");
private static final TableName TABLE_3 = TableName.valueOf("table_3");
private static final TableName[] tables = new TableName[]{TABLE_1, TABLE_2, TABLE_3};
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster(4);
admin = UTIL.getAdmin();
admin.setBalancerRunning(false, true);
createTables();
}
@AfterClass
public static void afterClass() throws Exception {
for (TableName table : tables) {
UTIL.deleteTableIfAny(table);
}
UTIL.shutdownMiniCluster();
}
private static void createTables() throws IOException, InterruptedException {
byte[] FAMILY = Bytes.toBytes("f");
for (TableName tableName : tables) {
Table table = UTIL.createMultiRegionTable(tableName, FAMILY, 16);
UTIL.waitTableAvailable(tableName);
UTIL.loadTable(table, FAMILY);
}
}
@Test
public void testRegionLoad() throws Exception {
// Check if regions match with the regionLoad from the server
for (ServerName serverName : admin.getClusterStatus().getServers()) {
List<HRegionInfo> regions = admin.getOnlineRegions(serverName);
Collection<RegionLoad> regionLoads = admin.getRegionLoad(serverName).values();
checkRegionsAndRegionLoads(regions, regionLoads);
}
// Check if regionLoad matches the table's regions and nothing is missed
for (TableName table : new TableName[]{TABLE_1, TABLE_2, TABLE_3}) {
List<HRegionInfo> tableRegions = admin.getTableRegions(table);
List<RegionLoad> regionLoads = Lists.newArrayList();
for (ServerName serverName : admin.getClusterStatus().getServers()) {
regionLoads.addAll(admin.getRegionLoad(serverName, table).values());
}
checkRegionsAndRegionLoads(tableRegions, regionLoads);
}
// Check RegionLoad matches the regionLoad from ClusterStatus
ClusterStatus clusterStatus = admin.getClusterStatus();
for (ServerName serverName : clusterStatus.getServers()) {
ServerLoad serverLoad = clusterStatus.getLoad(serverName);
Map<byte[], RegionLoad> regionLoads = admin.getRegionLoad(serverName);
compareRegionLoads(serverLoad.getRegionsLoad(), regionLoads);
}
}
private void compareRegionLoads(Map<byte[], RegionLoad> regionLoadCluster,
Map<byte[], RegionLoad> regionLoads) {
assertEquals("No of regionLoads from clusterStatus and regionloads from RS doesn't match",
regionLoadCluster.size(), regionLoads.size());
// The contents of region load from cluster and server should match
for (byte[] regionName : regionLoadCluster.keySet()) {
regionLoads.remove(regionName);
}
assertEquals("regionLoads from SN should be empty", 0, regionLoads.size());
}
private void checkRegionsAndRegionLoads(Collection<HRegionInfo> regions,
Collection<RegionLoad> regionLoads) {
assertEquals("No of regions and regionloads doesn't match",
regions.size(), regionLoads.size());
Map<byte[], RegionLoad> regionLoadMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
for (RegionLoad regionLoad : regionLoads) {
regionLoadMap.put(regionLoad.getName(), regionLoad);
}
for (HRegionInfo info : regions) {
assertTrue("Region not in regionLoadMap region:" + info.getRegionNameAsString() +
" regionMap: " + regionLoadMap, regionLoadMap.containsKey(info.getRegionName()));
}
}
}

9
hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
View File

@ -59,6 +59,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegion
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
@ -451,6 +453,13 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
return builder.build();
}
@Override
public GetRegionLoadResponse getRegionLoad(RpcController controller,
GetRegionLoadRequest request) throws ServiceException {
GetRegionLoadResponse.Builder builder = GetRegionLoadResponse.newBuilder();
return builder.build();
}
@Override
public GetStoreFileResponse getStoreFile(RpcController controller,
GetStoreFileRequest request) throws ServiceException {

66
hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java
View File

@ -18,11 +18,9 @@
package org.apache.hadoop.hbase.util;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
@ -39,8 +37,8 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import static org.apache.hadoop.hbase.HConstants.DEFAULT_REGIONSERVER_PORT;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@Category({MiscTests.class, SmallTests.class})
@ -48,6 +46,8 @@ public class TestRegionSizeCalculator {
private Configuration configuration = new Configuration();
private final long megabyte = 1024L * 1024L;
private final ServerName sn = ServerName.valueOf("local-rs", DEFAULT_REGIONSERVER_PORT,
ServerName.NON_STARTCODE);
@Test
public void testSimpleTestCase() throws Exception {
@ -55,14 +55,9 @@ public class TestRegionSizeCalculator {
RegionLocator regionLocator = mockRegionLocator("region1", "region2", "region3");
Admin admin = mockAdmin(
mockServer(
mockRegion("region1", 123),
mockRegion("region3", 1232)
),
mockServer(
mockRegion("region2", 54321),
mockRegion("otherTableRegion", 110)
)
mockRegion("region3", 1232),
mockRegion("region2", 54321)
);
RegionSizeCalculator calculator = new RegionSizeCalculator(regionLocator, admin);
@ -70,7 +65,7 @@ public class TestRegionSizeCalculator {
assertEquals(123 * megabyte, calculator.getRegionSize("region1".getBytes()));
assertEquals(54321 * megabyte, calculator.getRegionSize("region2".getBytes()));
assertEquals(1232 * megabyte, calculator.getRegionSize("region3".getBytes()));
// if region is not inside our table, it should return 0
// if regionCalculator does not know about a region, it should return 0
assertEquals(0 * megabyte, calculator.getRegionSize("otherTableRegion".getBytes()));
assertEquals(3, calculator.getRegionSizeMap().size());
@ -87,9 +82,7 @@ public class TestRegionSizeCalculator {
RegionLocator regionLocator = mockRegionLocator("largeRegion");
Admin admin = mockAdmin(
mockServer(
mockRegion("largeRegion", Integer.MAX_VALUE)
)
);
RegionSizeCalculator calculator = new RegionSizeCalculator(regionLocator, admin);
@ -104,9 +97,7 @@ public class TestRegionSizeCalculator {
RegionLocator table = mockRegionLocator(regionName);
Admin admin = mockAdmin(
mockServer(
mockRegion(regionName, 999)
)
);
//first request on enabled calculator
@ -133,21 +124,23 @@ public class TestRegionSizeCalculator {
for (String regionName : regionNames) {
HRegionInfo info = Mockito.mock(HRegionInfo.class);
when(info.getRegionName()).thenReturn(regionName.getBytes());
regionLocations.add(new HRegionLocation(info, null));//we are not interested in values
regionLocations.add(new HRegionLocation(info, sn));
}
return mockedTable;
}
/**
* Creates mock returning ClusterStatus info about given servers.
* Creates mock returning RegionLoad info about given servers.
*/
private Admin mockAdmin(ServerLoad... servers) throws Exception {
//get clusterstatus
private Admin mockAdmin(RegionLoad... regionLoadArray) throws Exception {
Admin mockAdmin = Mockito.mock(Admin.class);
ClusterStatus clusterStatus = mockCluster(servers);
Map<byte[], RegionLoad> regionLoads = new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
for (RegionLoad regionLoad : regionLoadArray) {
regionLoads.put(regionLoad.getName(), regionLoad);
}
when(mockAdmin.getConfiguration()).thenReturn(configuration);
when(mockAdmin.getClusterStatus()).thenReturn(clusterStatus);
when(mockAdmin.getRegionLoad(sn, TableName.valueOf("sizeTestTable"))).thenReturn(regionLoads);
return mockAdmin;
}
@ -163,35 +156,4 @@ public class TestRegionSizeCalculator {
when(region.getStorefileSizeMB()).thenReturn(fileSizeMb);
return region;
}
private ClusterStatus mockCluster(ServerLoad[] servers) {
List<ServerName> serverNames = new ArrayList<ServerName>();
ClusterStatus clusterStatus = Mockito.mock(ClusterStatus.class);
when(clusterStatus.getServers()).thenReturn(serverNames);
int serverCounter = 0;
for (ServerLoad server : servers) {
ServerName serverName = mock(ServerName.class);
when(serverName.getServerName()).thenReturn("server" + (serverCounter++));
serverNames.add(serverName);
when(clusterStatus.getLoad(serverName)).thenReturn(server);
}
return clusterStatus;
}
/** Creates mock of region server with given regions*/
private ServerLoad mockServer(RegionLoad... regions) {
ServerLoad serverLoad = Mockito.mock(ServerLoad.class);
Map<byte[], RegionLoad> regionMap = new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
for (RegionLoad regionName : regions) {
regionMap.put(regionName.getName(), regionName);
}
when(serverLoad.getRegionsLoad()).thenReturn(regionMap);
return serverLoad;
}
}