Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-18283 Provide a construct method which accept a thread pool for AsyncAdmin

This commit is contained in:
Guanghao Zhang 2017-07-04 09:51:41 +08:00
parent 9b9fd708b7
commit af36bfb2f7
15 changed files with 3276 additions and 2908 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
View File

@ -46,11 +46,6 @@ import org.apache.hadoop.hbase.util.Pair;
@InterfaceAudience.Public
public interface AsyncAdmin {
/**
* @return Async Connection used by this object.
*/
AsyncConnectionImpl getConnection();
/**
* @param tableName Table to check.
* @return True if table exists already. The return value will be wrapped by a
@ -105,7 +100,9 @@ public interface AsyncAdmin {
* Creates a new table.
* @param desc table descriptor for table
*/
CompletableFuture<Void> createTable(TableDescriptor desc);
default CompletableFuture<Void> createTable(TableDescriptor desc) {
return createTable(desc, Optional.empty());
}
/**
* Creates a new table with the specified number of regions. The start key specified will become
@ -128,7 +125,7 @@ public interface AsyncAdmin {
* @param desc table descriptor for table
* @param splitKeys array of split keys for the initial regions of the table
*/
CompletableFuture<Void> createTable(TableDescriptor desc, byte[][] splitKeys);
CompletableFuture<Void> createTable(TableDescriptor desc, Optional<byte[][]> splitKeys);
/**
* Deletes a table.
@ -186,6 +183,13 @@ public interface AsyncAdmin {
*/
CompletableFuture<List<TableDescriptor>> disableTables(Pattern pattern);
/**
* @param tableName name of table to check
* @return true if table is on-line. The return value will be wrapped by a
* {@link CompletableFuture}.
*/
CompletableFuture<Boolean> isTableEnabled(TableName tableName);
/**
* @param tableName name of table to check
* @return true if table is off-line. The return value will be wrapped by a
@ -198,7 +202,9 @@ public interface AsyncAdmin {
* @return true if all regions of the table are available. The return value will be wrapped by a
* {@link CompletableFuture}.
*/
CompletableFuture<Boolean> isTableAvailable(TableName tableName);
default CompletableFuture<Boolean> isTableAvailable(TableName tableName) {
return isTableAvailable(tableName, null);
}
/**
* Use this api to check if the table has been created with the specified number of splitkeys
@ -274,13 +280,6 @@ public interface AsyncAdmin {
*/
CompletableFuture<List<NamespaceDescriptor>> listNamespaceDescriptors();
/**
* @param tableName name of table to check
* @return true if table is on-line. The return value will be wrapped by a
* {@link CompletableFuture}.
*/
CompletableFuture<Boolean> isTableEnabled(TableName tableName);
/**
* Turn the load balancer on or off.
* @param on
@ -330,7 +329,7 @@ public interface AsyncAdmin {
/**
* Get all the online regions on a region server.
*/
CompletableFuture<List<HRegionInfo>> getOnlineRegions(ServerName sn);
CompletableFuture<List<HRegionInfo>> getOnlineRegions(ServerName serverName);
/**
* Flush a table.
@ -422,15 +421,15 @@ public interface AsyncAdmin {
/**
* Compact all regions on the region server.
* @param sn the region server name
* @param serverName the region server name
*/
CompletableFuture<Void> compactRegionServer(ServerName sn);
CompletableFuture<Void> compactRegionServer(ServerName serverName);
/**
* Compact all regions on the region server.
* @param sn the region server name
* @param serverName the region server name
*/
CompletableFuture<Void> majorCompactRegionServer(ServerName sn);
CompletableFuture<Void> majorCompactRegionServer(ServerName serverName);
/**
* Merge two regions.
@ -563,18 +562,18 @@ public interface AsyncAdmin {
/**
* Append the replicable table-cf config of the specified peer
* @param id a short that identifies the cluster
* @param peerId a short that identifies the cluster
* @param tableCfs A map from tableName to column family names
*/
CompletableFuture<Void> appendReplicationPeerTableCFs(String id,
CompletableFuture<Void> appendReplicationPeerTableCFs(String peerId,
Map<TableName, ? extends Collection<String>> tableCfs);
/**
* Remove some table-cfs from config of the specified peer
* @param id a short name that identifies the cluster
* @param peerId a short name that identifies the cluster
* @param tableCfs A map from tableName to column family names
*/
CompletableFuture<Void> removeReplicationPeerTableCFs(String id,
CompletableFuture<Void> removeReplicationPeerTableCFs(String peerId,
Map<TableName, ? extends Collection<String>> tableCfs);
/**
@ -613,7 +612,9 @@ public interface AsyncAdmin {
* @param snapshotName name of the snapshot to be created
* @param tableName name of the table for which snapshot is created
*/
CompletableFuture<Void> snapshot(String snapshotName, TableName tableName);
default CompletableFuture<Void> snapshot(String snapshotName, TableName tableName) {
return snapshot(snapshotName, tableName, SnapshotType.FLUSH);
}
/**
* Create typed snapshot of the table. Snapshots are considered unique based on <b>the name of the
@ -627,8 +628,10 @@ public interface AsyncAdmin {
* @param tableName name of the table to snapshot
* @param type type of snapshot to take
*/
CompletableFuture<Void> snapshot(String snapshotName, TableName tableName,
SnapshotType type);
default CompletableFuture<Void> snapshot(String snapshotName, TableName tableName,
SnapshotType type) {
return snapshot(new SnapshotDescription(snapshotName, tableName, type));
}
/**
* Take a snapshot and wait for the server to complete that snapshot asynchronously. Only a single
@ -695,14 +698,16 @@ public interface AsyncAdmin {
* @return a list of snapshot descriptors for completed snapshots wrapped by a
* {@link CompletableFuture}
*/
CompletableFuture<List<SnapshotDescription>> listSnapshots();
default CompletableFuture<List<SnapshotDescription>> listSnapshots() {
return listSnapshots(Optional.empty());
}
/**
* List all the completed snapshots matching the given pattern.
* @param pattern The compiled regular expression to match against
* @return - returns a List of SnapshotDescription wrapped by a {@link CompletableFuture}
*/
CompletableFuture<List<SnapshotDescription>> listSnapshots(Pattern pattern);
CompletableFuture<List<SnapshotDescription>> listSnapshots(Optional<Pattern> pattern);
/**
* List all the completed snapshots matching the given table name regular expression and snapshot
@ -725,7 +730,9 @@ public interface AsyncAdmin {
* Delete existing snapshots whose names match the pattern passed.
* @param pattern pattern for names of the snapshot to match
*/
CompletableFuture<Void> deleteSnapshots(Pattern pattern);
default CompletableFuture<Void> deleteSnapshots(Pattern pattern) {
return deleteTableSnapshots(null, pattern);
}
/**
* Delete all existing snapshots matching the given table name regular expression and snapshot

14
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnection.java
View File

@ -96,11 +96,17 @@ public interface AsyncConnection extends Closeable {
AsyncTableBuilder<AsyncTable> getTableBuilder(TableName tableName, ExecutorService pool);
/**
* Retrieve an AsyncAdmin implementation to administer an HBase cluster. The returned AsyncAdmin
* is not guaranteed to be thread-safe. A new instance should be created for each using thread.
* This is a lightweight operation. Pooling or caching of the returned AsyncAdmin is not
* recommended.
* Retrieve an AsyncAdmin implementation to administer an HBase cluster. The returned
* {@code CompletableFuture} will be finished directly in the rpc framework's callback thread, so
* typically you should not do any time consuming work inside these methods.
* @return an AsyncAdmin instance for cluster administration
*/
AsyncAdmin getAdmin();
/**
* Retrieve an AsyncAdmin implementation to administer an HBase cluster.
* @param pool the thread pool to use for executing callback
* @return an AsyncAdmin instance for cluster administration
*/
AsyncAdmin getAdmin(ExecutorService pool);
}

7
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
View File

@ -279,6 +279,11 @@ class AsyncConnectionImpl implements AsyncConnection {
@Override
public AsyncAdmin getAdmin() {
return new AsyncHBaseAdmin(this);
return new RawAsyncHBaseAdmin(this);
}
@Override
public AsyncAdmin getAdmin(ExecutorService pool) {
return new AsyncHBaseAdmin(new RawAsyncHBaseAdmin(this), pool);
}
}

2065
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
View File

File diff suppressed because it is too large Load Diff

2278
hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java Normal file
View File

File diff suppressed because it is too large Load Diff

21
hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
View File

@ -19,11 +19,13 @@ package org.apache.hadoop.hbase.shaded.protobuf;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Pattern;
import org.apache.hadoop.hbase.CellScannable;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
@ -1271,19 +1273,26 @@ public final class RequestConverter {
final byte [][] splitKeys,
final long nonceGroup,
final long nonce) {
return buildCreateTableRequest(hTableDesc, Optional.ofNullable(splitKeys), nonceGroup, nonce);
}
/**
* Creates a protocol buffer CreateTableRequest
* @param hTableDesc
* @param splitKeys
* @return a CreateTableRequest
*/
public static CreateTableRequest buildCreateTableRequest(TableDescriptor hTableDesc,
Optional<byte[][]> splitKeys, long nonceGroup, long nonce) {
CreateTableRequest.Builder builder = CreateTableRequest.newBuilder();
builder.setTableSchema(ProtobufUtil.convertToTableSchema(hTableDesc));
if (splitKeys != null) {
for (byte [] splitKey : splitKeys) {
builder.addSplitKeys(UnsafeByteOperations.unsafeWrap(splitKey));
}
}
splitKeys.ifPresent(keys -> Arrays.stream(keys).forEach(
key -> builder.addSplitKeys(UnsafeByteOperations.unsafeWrap(key))));
builder.setNonceGroup(nonceGroup);
builder.setNonce(nonce);
return builder.build();
}
/**
* Creates a protocol buffer ModifyTableRequest
*

80
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminBase.java
View File

@ -19,15 +19,32 @@ package org.apache.hadoop.hbase.client;
import static org.apache.hadoop.hbase.client.AsyncProcess.START_LOG_ERRORS_AFTER_COUNT_KEY;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;
/**
* Class to test AsyncAdmin.
@ -36,13 +53,34 @@ public abstract class TestAsyncAdminBase {
protected static final Log LOG = LogFactory.getLog(TestAsyncAdminBase.class);
protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
protected static byte[] FAMILY = Bytes.toBytes("testFamily");
protected static final byte[] FAMILY = Bytes.toBytes("testFamily");
protected static final byte[] FAMILY_0 = Bytes.toBytes("cf0");
protected static final byte[] FAMILY_1 = Bytes.toBytes("cf1");
protected static AsyncConnection ASYNC_CONN;
protected AsyncAdmin admin;
@Parameter
public Supplier<AsyncAdmin> getAdmin;
private static AsyncAdmin getRawAsyncAdmin() {
return ASYNC_CONN.getAdmin();
}
private static AsyncAdmin getAsyncAdmin() {
return ASYNC_CONN.getAdmin(ForkJoinPool.commonPool());
}
@Parameters
public static List<Object[]> params() {
return Arrays.asList(new Supplier<?>[] { TestAsyncAdminBase::getRawAsyncAdmin },
new Supplier<?>[] { TestAsyncAdminBase::getAsyncAdmin });
}
@Rule
public TestName testName = new TestName();
protected TableName tableName;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 60000);
@ -60,7 +98,43 @@ public abstract class TestAsyncAdminBase {
}
@Before
public void setUp() throws Exception {
this.admin = ASYNC_CONN.getAdmin();
public void setUp() {
admin = ASYNC_CONN.getAdmin();
String methodName = testName.getMethodName();
tableName = TableName.valueOf(methodName.substring(0, methodName.length() - 3));
}
@After
public void tearDown() {
admin.listTableNames(Optional.of(Pattern.compile(tableName.getNameAsString() + ".*")), false)
.whenCompleteAsync((tables, err) -> {
if (tables != null) {
tables.forEach(table -> {
try {
admin.disableTable(table).join();
} catch (Exception e) {
LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
}
admin.deleteTable(table).join();
});
}
}, ForkJoinPool.commonPool()).join();
}
protected void createTableWithDefaultConf(TableName tableName) {
createTableWithDefaultConf(tableName, Optional.empty());
}
protected void createTableWithDefaultConf(TableName tableName, Optional<byte[][]> splitKeys) {
createTableWithDefaultConf(tableName, splitKeys, FAMILY);
}
protected void createTableWithDefaultConf(TableName tableName, Optional<byte[][]> splitKeys,
byte[]... families) {
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
for (byte[] family : families) {
builder.addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).build());
}
admin.createTable(builder.build(), splitKeys).join();
}
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncBalancerAdminApi.java
View File

@ -23,7 +23,10 @@ import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@RunWith(Parameterized.class)
@Category({ MediumTests.class, ClientTests.class })
public class TestAsyncBalancerAdminApi extends TestAsyncAdminBase {

3
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNamespaceAdminApi.java
View File

@ -42,10 +42,13 @@ import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
/**
* Class to test asynchronous namespace admin operations.
*/
@RunWith(Parameterized.class)
@Category({ LargeTests.class, ClientTests.class })
public class TestAsyncNamespaceAdminApi extends TestAsyncAdminBase {

4
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcedureAdminApi.java
View File

@ -66,7 +66,7 @@ public class TestAsyncProcedureAdminApi extends TestAsyncAdminBase {
@Test
public void testExecProcedure() throws Exception {
TableName tableName = TableName.valueOf("testExecProcedure");
String snapshotString = "offlineTableSnapshot";
try {
Table table = TEST_UTIL.createTable(tableName, Bytes.toBytes("cf"));
for (int i = 0; i < 100; i++) {
@ -74,13 +74,13 @@ public class TestAsyncProcedureAdminApi extends TestAsyncAdminBase {
table.put(put);
}
// take a snapshot of the enabled table
String snapshotString = "offlineTableSnapshot";
Map<String, String> props = new HashMap<>();
props.put("table", tableName.getNameAsString());
admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, snapshotString,
props).get();
LOG.debug("Snapshot completed.");
} finally {
admin.deleteSnapshot(snapshotString).join();
TEST_UTIL.deleteTable(tableName);
}
}

37
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncQuotaAdminApi.java
View File

@ -18,10 +18,6 @@
package org.apache.hadoop.hbase.client;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.quotas.QuotaCache;
@ -34,50 +30,35 @@ import org.apache.hadoop.hbase.quotas.ThrottleType;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hbase.client.AsyncProcess.START_LOG_ERRORS_AFTER_COUNT_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@RunWith(Parameterized.class)
@Category({ ClientTests.class, MediumTests.class })
public class TestAsyncQuotaAdminApi {
private static final Log LOG = LogFactory.getLog(TestAsyncQuotaAdminApi.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
protected static AsyncConnection ASYNC_CONN;
protected AsyncAdmin admin;
public class TestAsyncQuotaAdminApi extends TestAsyncAdminBase {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, 2000);
TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 60000);
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 120000);
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
TEST_UTIL.getConfiguration().setInt(START_LOG_ERRORS_AFTER_COUNT_KEY, 0);
TEST_UTIL.startMiniCluster(1);
TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
ASYNC_CONN = ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
IOUtils.closeQuietly(ASYNC_CONN);
TEST_UTIL.shutdownMiniCluster();
}
@Before
public void setUp() throws Exception {
this.admin = ASYNC_CONN.getAdmin();
}
@Test
public void testThrottleType() throws Exception {
String userName = User.getCurrent().getShortName();

737
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.client;
import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@ -31,11 +32,10 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.AsyncMetaTableAccessor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.master.HMaster;
@ -51,36 +51,29 @@ import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
/**
* Class to test asynchronous region admin operations.
*/
@RunWith(Parameterized.class)
@Category({ LargeTests.class, ClientTests.class })
public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
public static Random RANDOM = new Random(System.currentTimeMillis());
private void createTableWithDefaultConf(TableName TABLENAME) throws Exception {
HTableDescriptor htd = new HTableDescriptor(TABLENAME);
HColumnDescriptor hcd = new HColumnDescriptor("value");
htd.addFamily(hcd);
admin.createTable(htd, null).get();
}
@Test
public void testCloseRegion() throws Exception {
TableName TABLENAME = TableName.valueOf("TestHBACloseRegion");
createTableWithDefaultConf(TABLENAME);
createTableWithDefaultConf(tableName);
HRegionInfo info = null;
HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME);
HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName);
List<HRegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
for (HRegionInfo regionInfo : onlineRegions) {
if (!regionInfo.getTable().isSystemTable()) {
@ -102,16 +95,14 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
@Test
public void testCloseRegionIfInvalidRegionNameIsPassed() throws Exception {
final String name = "TestHBACloseRegion1";
byte[] TABLENAME = Bytes.toBytes(name);
createTableWithDefaultConf(TableName.valueOf(TABLENAME));
createTableWithDefaultConf(tableName);
HRegionInfo info = null;
HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME));
HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName);
List<HRegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
for (HRegionInfo regionInfo : onlineRegions) {
if (!regionInfo.isMetaTable()) {
if (regionInfo.getRegionNameAsString().contains(name)) {
if (regionInfo.getRegionNameAsString().contains(tableName.getNameAsString())) {
info = regionInfo;
boolean catchNotServingException = false;
try {
@ -132,10 +123,9 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
@Test
public void testCloseRegionWhenServerNameIsEmpty() throws Exception {
byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegionWhenServerNameIsEmpty");
createTableWithDefaultConf(TableName.valueOf(TABLENAME));
createTableWithDefaultConf(tableName);
HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME));
HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName);
List<HRegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
for (HRegionInfo regionInfo : onlineRegions) {
if (!regionInfo.isMetaTable()) {
@ -147,166 +137,61 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
}
@Test
public void testGetRegion() throws Exception {
AsyncHBaseAdmin rawAdmin = (AsyncHBaseAdmin) admin;
final TableName tableName = TableName.valueOf("testGetRegion");
LOG.info("Started " + tableName);
public void testGetRegionLocation() throws Exception {
RawAsyncHBaseAdmin rawAdmin = (RawAsyncHBaseAdmin) ASYNC_CONN.getAdmin();
TEST_UTIL.createMultiRegionTable(tableName, HConstants.CATALOG_FAMILY);
try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
HRegionLocation regionLocation = locator.getRegionLocation(Bytes.toBytes("mmm"));
HRegionInfo region = regionLocation.getRegionInfo();
byte[] regionName = region.getRegionName();
HRegionLocation location = rawAdmin.getRegionLocation(regionName).get();
assertTrue(Bytes.equals(regionName, location.getRegionInfo().getRegionName()));
location = rawAdmin.getRegionLocation(region.getEncodedNameAsBytes()).get();
assertTrue(Bytes.equals(regionName, location.getRegionInfo().getRegionName()));
}
}
@Test
public void testMergeRegions() throws Exception {
final TableName tableName = TableName.valueOf("testMergeRegions");
HColumnDescriptor cd = new HColumnDescriptor("d");
HTableDescriptor td = new HTableDescriptor(tableName);
td.addFamily(cd);
byte[][] splitRows = new byte[][] { Bytes.toBytes("3"), Bytes.toBytes("6") };
Admin syncAdmin = TEST_UTIL.getAdmin();
try {
TEST_UTIL.createTable(td, splitRows);
TEST_UTIL.waitTableAvailable(tableName);
List<HRegionInfo> tableRegions;
HRegionInfo regionA;
HRegionInfo regionB;
// merge with full name
tableRegions = syncAdmin.getTableRegions(tableName);
assertEquals(3, syncAdmin.getTableRegions(tableName).size());
regionA = tableRegions.get(0);
regionB = tableRegions.get(1);
admin.mergeRegions(regionA.getRegionName(), regionB.getRegionName(), false).get();
assertEquals(2, syncAdmin.getTableRegions(tableName).size());
// merge with encoded name
tableRegions = syncAdmin.getTableRegions(tableName);
regionA = tableRegions.get(0);
regionB = tableRegions.get(1);
admin.mergeRegions(regionA.getRegionName(), regionB.getRegionName(), false).get();
assertEquals(1, syncAdmin.getTableRegions(tableName).size());
} finally {
syncAdmin.disableTable(tableName);
syncAdmin.deleteTable(tableName);
}
}
@Test
public void testSplitTable() throws Exception {
splitTests(TableName.valueOf("testSplitTable"), 3000, false, null);
splitTests(TableName.valueOf("testSplitTableWithSplitPoint"), 3000, false, Bytes.toBytes("3"));
splitTests(TableName.valueOf("testSplitRegion"), 3000, true, null);
splitTests(TableName.valueOf("testSplitRegionWithSplitPoint"), 3000, true, Bytes.toBytes("3"));
}
private void splitTests(TableName tableName, int rowCount, boolean isSplitRegion,
byte[] splitPoint) throws Exception {
int count = 0;
// create table
HColumnDescriptor cd = new HColumnDescriptor("d");
HTableDescriptor td = new HTableDescriptor(tableName);
td.addFamily(cd);
Table table = TEST_UTIL.createTable(td, null);
TEST_UTIL.waitTableAvailable(tableName);
List<HRegionInfo> regions = TEST_UTIL.getAdmin().getTableRegions(tableName);
assertEquals(regions.size(), 1);
List<Put> puts = new ArrayList<>();
for (int i = 0; i < rowCount; i++) {
Put put = new Put(Bytes.toBytes(i));
put.addColumn(Bytes.toBytes("d"), null, Bytes.toBytes("value" + i));
puts.add(put);
}
table.put(puts);
if (isSplitRegion) {
admin.splitRegion(regions.get(0).getRegionName(), Optional.ofNullable(splitPoint)).get();
} else {
if (splitPoint == null) {
admin.split(tableName).get();
} else {
admin.split(tableName, splitPoint).get();
}
}
for (int i = 0; i < 45; i++) {
try {
List<HRegionInfo> hRegionInfos = TEST_UTIL.getAdmin().getTableRegions(tableName);
count = hRegionInfos.size();
if (count >= 2) {
break;
}
Thread.sleep(1000L);
} catch (Exception e) {
LOG.error(e);
}
}
assertEquals(count, 2);
AsyncTableRegionLocator locator = ASYNC_CONN.getRegionLocator(tableName);
HRegionLocation regionLocation = locator.getRegionLocation(Bytes.toBytes("mmm")).get();
HRegionInfo region = regionLocation.getRegionInfo();
byte[] regionName = regionLocation.getRegionInfo().getRegionName();
HRegionLocation location = rawAdmin.getRegionLocation(regionName).get();
assertTrue(Bytes.equals(regionName, location.getRegionInfo().getRegionName()));
location = rawAdmin.getRegionLocation(region.getEncodedNameAsBytes()).get();
assertTrue(Bytes.equals(regionName, location.getRegionInfo().getRegionName()));
}
@Test
public void testAssignRegionAndUnassignRegion() throws Exception {
final TableName tableName = TableName.valueOf("testAssignRegionAndUnassignRegion");
createTableWithDefaultConf(tableName);
// assign region.
HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
AssignmentManager am = master.getAssignmentManager();
HRegionInfo hri = am.getRegionStates().getRegionsOfTable(tableName).get(0);
// assert region on server
RegionStates regionStates = am.getRegionStates();
ServerName serverName = regionStates.getRegionServerOfRegion(hri);
TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
assertTrue(regionStates.getRegionState(hri).isOpened());
// Region is assigned now. Let's assign it again.
// Master should not abort, and region should stay assigned.
admin.assign(hri.getRegionName()).get();
try {
// create test table
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(new HColumnDescriptor(FAMILY));
admin.createTable(desc).get();
// assign region.
HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
AssignmentManager am = master.getAssignmentManager();
HRegionInfo hri = am.getRegionStates().getRegionsOfTable(tableName).get(0);
// assert region on server
RegionStates regionStates = am.getRegionStates();
ServerName serverName = regionStates.getRegionServerOfRegion(hri);
TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
assertTrue(regionStates.getRegionState(hri).isOpened());
// Region is assigned now. Let's assign it again.
// Master should not abort, and region should stay assigned.
admin.assign(hri.getRegionName()).get();
try {
am.waitForAssignment(hri);
fail("Expected NoSuchProcedureException");
} catch (NoSuchProcedureException e) {
// Expected
}
assertTrue(regionStates.getRegionState(hri).isOpened());
// unassign region
admin.unassign(hri.getRegionName(), true).get();
try {
am.waitForAssignment(hri);
fail("Expected NoSuchProcedureException");
} catch (NoSuchProcedureException e) {
// Expected
}
assertTrue(regionStates.getRegionState(hri).isClosed());
} finally {
TEST_UTIL.deleteTable(tableName);
am.waitForAssignment(hri);
fail("Expected NoSuchProcedureException");
} catch (NoSuchProcedureException e) {
// Expected
}
assertTrue(regionStates.getRegionState(hri).isOpened());
// unassign region
admin.unassign(hri.getRegionName(), true).get();
try {
am.waitForAssignment(hri);
fail("Expected NoSuchProcedureException");
} catch (NoSuchProcedureException e) {
// Expected
}
assertTrue(regionStates.getRegionState(hri).isClosed());
}
HRegionInfo createTableAndGetOneRegion(final TableName tableName)
throws IOException, InterruptedException, ExecutionException {
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(new HColumnDescriptor(FAMILY));
TableDescriptor desc =
TableDescriptorBuilder.newBuilder(tableName)
.addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build()).build();
admin.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), 5).get();
// wait till the table is assigned
@ -333,280 +218,341 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
// Will cause the Master to tell the regionserver to shut itself down because
// regionserver is reporting the state as OPEN.
public void testOfflineRegion() throws Exception {
final TableName tableName = TableName.valueOf("testOfflineRegion");
try {
HRegionInfo hri = createTableAndGetOneRegion(tableName);
HRegionInfo hri = createTableAndGetOneRegion(tableName);
RegionStates regionStates =
TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
admin.offline(hri.getRegionName()).get();
RegionStates regionStates =
TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
admin.offline(hri.getRegionName()).get();
long timeoutTime = System.currentTimeMillis() + 3000;
while (true) {
if (regionStates.getRegionByStateOfTable(tableName).get(RegionState.State.OFFLINE)
.contains(hri))
break;
long now = System.currentTimeMillis();
if (now > timeoutTime) {
fail("Failed to offline the region in time");
break;
}
Thread.sleep(10);
long timeoutTime = System.currentTimeMillis() + 3000;
while (true) {
if (regionStates.getRegionByStateOfTable(tableName).get(RegionState.State.OFFLINE)
.contains(hri)) break;
long now = System.currentTimeMillis();
if (now > timeoutTime) {
fail("Failed to offline the region in time");
break;
}
RegionState regionState = regionStates.getRegionState(hri);
assertTrue(regionState.isOffline());
} finally {
TEST_UTIL.deleteTable(tableName);
Thread.sleep(10);
}
RegionState regionState = regionStates.getRegionState(hri);
assertTrue(regionState.isOffline());
}
@Test
public void testGetRegionByStateOfTable() throws Exception {
final TableName tableName = TableName.valueOf("testGetRegionByStateOfTable");
try {
HRegionInfo hri = createTableAndGetOneRegion(tableName);
HRegionInfo hri = createTableAndGetOneRegion(tableName);
RegionStates regionStates =
TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
assertTrue(regionStates.getRegionByStateOfTable(tableName)
.get(RegionState.State.OPEN)
.contains(hri));
assertFalse(regionStates.getRegionByStateOfTable(TableName.valueOf("I_am_the_phantom"))
.get(RegionState.State.OPEN)
.contains(hri));
} finally {
TEST_UTIL.deleteTable(tableName);
}
RegionStates regionStates =
TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
assertTrue(regionStates.getRegionByStateOfTable(tableName).get(RegionState.State.OPEN)
.contains(hri));
assertFalse(regionStates.getRegionByStateOfTable(TableName.valueOf("I_am_the_phantom"))
.get(RegionState.State.OPEN).contains(hri));
}
@Test
public void testMoveRegion() throws Exception {
final TableName tableName = TableName.valueOf("testMoveRegion");
try {
HRegionInfo hri = createTableAndGetOneRegion(tableName);
admin.setBalancerOn(false).join();
HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
RegionStates regionStates = master.getAssignmentManager().getRegionStates();
ServerName serverName = regionStates.getRegionServerOfRegion(hri);
ServerManager serverManager = master.getServerManager();
ServerName destServerName = null;
List<JVMClusterUtil.RegionServerThread> regionServers =
TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads();
for (JVMClusterUtil.RegionServerThread regionServer : regionServers) {
HRegionServer destServer = regionServer.getRegionServer();
destServerName = destServer.getServerName();
if (!destServerName.equals(serverName) && serverManager.isServerOnline(destServerName)) {
break;
}
}
assertTrue(destServerName != null && !destServerName.equals(serverName));
admin.move(hri.getRegionName(), Optional.of(destServerName)).get();
HRegionInfo hri = createTableAndGetOneRegion(tableName);
RawAsyncHBaseAdmin rawAdmin = (RawAsyncHBaseAdmin) ASYNC_CONN.getAdmin();
ServerName serverName = rawAdmin.getRegionLocation(hri.getRegionName()).get().getServerName();
long timeoutTime = System.currentTimeMillis() + 30000;
while (true) {
ServerName sn = regionStates.getRegionServerOfRegion(hri);
if (sn != null && sn.equals(destServerName)) {
TEST_UTIL.assertRegionOnServer(hri, sn, 200);
break;
}
long now = System.currentTimeMillis();
if (now > timeoutTime) {
fail("Failed to move the region in time: " + regionStates.getRegionState(hri));
}
regionStates.wait(50);
HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
ServerManager serverManager = master.getServerManager();
ServerName destServerName = null;
List<JVMClusterUtil.RegionServerThread> regionServers =
TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads();
for (JVMClusterUtil.RegionServerThread regionServer : regionServers) {
HRegionServer destServer = regionServer.getRegionServer();
destServerName = destServer.getServerName();
if (!destServerName.equals(serverName) && serverManager.isServerOnline(destServerName)) {
break;
}
} finally {
TEST_UTIL.deleteTable(tableName);
}
assertTrue(destServerName != null && !destServerName.equals(serverName));
admin.move(hri.getRegionName(), Optional.of(destServerName)).get();
long timeoutTime = System.currentTimeMillis() + 30000;
while (true) {
ServerName sn = rawAdmin.getRegionLocation(hri.getRegionName()).get().getServerName();
if (sn != null && sn.equals(destServerName)) {
break;
}
long now = System.currentTimeMillis();
if (now > timeoutTime) {
fail("Failed to move the region in time: " + hri);
}
Thread.sleep(100);
}
admin.setBalancerOn(true).join();
}
@Test
public void testGetOnlineRegions() throws Exception {
final TableName tableName = TableName.valueOf("testGetOnlineRegions");
try {
createTableAndGetOneRegion(tableName);
AtomicInteger regionServerCount = new AtomicInteger(0);
TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().stream()
.map(rsThread -> rsThread.getRegionServer().getServerName()).forEach(serverName -> {
createTableAndGetOneRegion(tableName);
AtomicInteger regionServerCount = new AtomicInteger(0);
TEST_UTIL
.getHBaseCluster()
.getLiveRegionServerThreads()
.stream()
.map(rsThread -> rsThread.getRegionServer())
.forEach(
rs -> {
ServerName serverName = rs.getServerName();
try {
Assert.assertEquals(admin.getOnlineRegions(serverName).get().size(),
TEST_UTIL.getAdmin().getOnlineRegions(serverName).size());
Assert.assertEquals(admin.getOnlineRegions(serverName).get().size(), rs
.getOnlineRegions().size());
} catch (Exception e) {
fail("admin.getOnlineRegions() method throws a exception: " + e.getMessage());
}
regionServerCount.incrementAndGet();
});
Assert.assertEquals(regionServerCount.get(), 2);
} catch (Exception e) {
LOG.info("Exception", e);
throw e;
} finally {
TEST_UTIL.deleteTable(tableName);
}
Assert.assertEquals(regionServerCount.get(), 2);
}
@Test
public void testFlushTableAndRegion() throws Exception {
final TableName tableName = TableName.valueOf("testFlushRegion");
try {
HRegionInfo hri = createTableAndGetOneRegion(tableName);
ServerName serverName = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
.getRegionStates().getRegionServerOfRegion(hri);
HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().stream()
.map(rsThread -> rsThread.getRegionServer())
.filter(rs -> rs.getServerName().equals(serverName)).findFirst().get();
// write a put into the specific region
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
table.put(new Put(hri.getStartKey()).addColumn(FAMILY, FAMILY_0, Bytes.toBytes("value-1")));
}
Assert.assertTrue(regionServer.getOnlineRegion(hri.getRegionName()).getMemstoreSize() > 0);
// flush region and wait flush operation finished.
LOG.info("flushing region: " + Bytes.toStringBinary(hri.getRegionName()));
admin.flushRegion(hri.getRegionName()).get();
LOG.info("blocking until flush is complete: " + Bytes.toStringBinary(hri.getRegionName()));
Threads.sleepWithoutInterrupt(500);
while (regionServer.getOnlineRegion(hri.getRegionName()).getMemstoreSize() > 0) {
Threads.sleep(50);
}
// check the memstore.
Assert.assertEquals(regionServer.getOnlineRegion(hri.getRegionName()).getMemstoreSize(), 0);
HRegionInfo hri = createTableAndGetOneRegion(tableName);
ServerName serverName =
TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
.getRegionServerOfRegion(hri);
HRegionServer regionServer =
TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().stream()
.map(rsThread -> rsThread.getRegionServer())
.filter(rs -> rs.getServerName().equals(serverName)).findFirst().get();
// write another put into the specific region
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
table.put(new Put(hri.getStartKey()).addColumn(FAMILY, FAMILY_0, Bytes.toBytes("value-2")));
}
Assert.assertTrue(regionServer.getOnlineRegion(hri.getRegionName()).getMemstoreSize() > 0);
admin.flush(tableName).get();
Threads.sleepWithoutInterrupt(500);
while (regionServer.getOnlineRegion(hri.getRegionName()).getMemstoreSize() > 0) {
Threads.sleep(50);
}
// check the memstore.
Assert.assertEquals(regionServer.getOnlineRegion(hri.getRegionName()).getMemstoreSize(), 0);
} finally {
TEST_UTIL.deleteTable(tableName);
// write a put into the specific region
ASYNC_CONN.getRawTable(tableName)
.put(new Put(hri.getStartKey()).addColumn(FAMILY, FAMILY_0, Bytes.toBytes("value-1")))
.join();
Assert.assertTrue(regionServer.getOnlineRegion(hri.getRegionName()).getMemstoreSize() > 0);
// flush region and wait flush operation finished.
LOG.info("flushing region: " + Bytes.toStringBinary(hri.getRegionName()));
admin.flushRegion(hri.getRegionName()).get();
LOG.info("blocking until flush is complete: " + Bytes.toStringBinary(hri.getRegionName()));
Threads.sleepWithoutInterrupt(500);
while (regionServer.getOnlineRegion(hri.getRegionName()).getMemstoreSize() > 0) {
Threads.sleep(50);
}
// check the memstore.
Assert.assertEquals(regionServer.getOnlineRegion(hri.getRegionName()).getMemstoreSize(), 0);
// write another put into the specific region
ASYNC_CONN.getRawTable(tableName)
.put(new Put(hri.getStartKey()).addColumn(FAMILY, FAMILY_0, Bytes.toBytes("value-2")))
.join();
Assert.assertTrue(regionServer.getOnlineRegion(hri.getRegionName()).getMemstoreSize() > 0);
admin.flush(tableName).get();
Threads.sleepWithoutInterrupt(500);
while (regionServer.getOnlineRegion(hri.getRegionName()).getMemstoreSize() > 0) {
Threads.sleep(50);
}
// check the memstore.
Assert.assertEquals(regionServer.getOnlineRegion(hri.getRegionName()).getMemstoreSize(), 0);
}
@Test(timeout = 600000)
public void testCompactRpcAPI() throws Exception {
String tableName = "testCompactRpcAPI";
compactionTest(tableName, 8, CompactionState.MAJOR, false);
compactionTest(tableName, 15, CompactionState.MINOR, false);
compactionTest(tableName, 8, CompactionState.MAJOR, true);
compactionTest(tableName, 15, CompactionState.MINOR, true);
@Test
public void testMergeRegions() throws Exception {
byte[][] splitRows = new byte[][] { Bytes.toBytes("3"), Bytes.toBytes("6") };
createTableWithDefaultConf(tableName, Optional.of(splitRows));
RawAsyncTable metaTable = ASYNC_CONN.getRawTable(META_TABLE_NAME);
List<HRegionLocation> regionLocations =
AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName)).get();
HRegionInfo regionA;
HRegionInfo regionB;
// merge with full name
assertEquals(3, regionLocations.size());
regionA = regionLocations.get(0).getRegionInfo();
regionB = regionLocations.get(1).getRegionInfo();
admin.mergeRegions(regionA.getRegionName(), regionB.getRegionName(), false).get();
regionLocations =
AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName)).get();
assertEquals(2, regionLocations.size());
// merge with encoded name
regionA = regionLocations.get(0).getRegionInfo();
regionB = regionLocations.get(1).getRegionInfo();
admin.mergeRegions(regionA.getRegionName(), regionB.getRegionName(), false).get();
regionLocations =
AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName)).get();
assertEquals(1, regionLocations.size());
}
@Test(timeout = 600000)
@Test
public void testSplitTable() throws Exception {
splitTest(TableName.valueOf("testSplitTable"), 3000, false, null);
splitTest(TableName.valueOf("testSplitTableWithSplitPoint"), 3000, false, Bytes.toBytes("3"));
splitTest(TableName.valueOf("testSplitTableRegion"), 3000, true, null);
splitTest(TableName.valueOf("testSplitTableRegionWithSplitPoint2"), 3000, true, Bytes.toBytes("3"));
}
private void
splitTest(TableName tableName, int rowCount, boolean isSplitRegion, byte[] splitPoint)
throws Exception {
// create table
createTableWithDefaultConf(tableName);
RawAsyncTable metaTable = ASYNC_CONN.getRawTable(META_TABLE_NAME);
List<HRegionLocation> regionLocations =
AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName)).get();
assertEquals(1, regionLocations.size());
RawAsyncTable table = ASYNC_CONN.getRawTable(tableName);
List<Put> puts = new ArrayList<>();
for (int i = 0; i < rowCount; i++) {
Put put = new Put(Bytes.toBytes(i));
put.addColumn(FAMILY, null, Bytes.toBytes("value" + i));
puts.add(put);
}
table.putAll(puts).join();
if (isSplitRegion) {
admin.splitRegion(regionLocations.get(0).getRegionInfo().getRegionName(),
Optional.ofNullable(splitPoint)).get();
} else {
if (splitPoint == null) {
admin.split(tableName).get();
} else {
admin.split(tableName, splitPoint).get();
}
}
int count = 0;
for (int i = 0; i < 45; i++) {
try {
regionLocations =
AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName))
.get();
count = regionLocations.size();
if (count >= 2) {
break;
}
Thread.sleep(1000L);
} catch (Exception e) {
LOG.error(e);
}
}
assertEquals(count, 2);
}
@Test
public void testCompactRegionServer() throws Exception {
TableName table = TableName.valueOf("testCompactRegionServer");
byte[][] families = { Bytes.toBytes("f1"), Bytes.toBytes("f2"), Bytes.toBytes("f3") };
Table ht = null;
try {
ht = TEST_UTIL.createTable(table, families);
loadData(ht, families, 3000, 8);
List<HRegionServer> rsList = TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().stream()
.map(rsThread -> rsThread.getRegionServer()).collect(Collectors.toList());
List<Region> regions = new ArrayList<>();
rsList.forEach(rs -> regions.addAll(rs.getOnlineRegions(table)));
Assert.assertEquals(regions.size(), 1);
int countBefore = countStoreFilesInFamilies(regions, families);
Assert.assertTrue(countBefore > 0);
// Minor compaction for all region servers.
for (HRegionServer rs : rsList)
admin.compactRegionServer(rs.getServerName()).get();
Thread.sleep(5000);
int countAfterMinorCompaction = countStoreFilesInFamilies(regions, families);
Assert.assertTrue(countAfterMinorCompaction < countBefore);
// Major compaction for all region servers.
for (HRegionServer rs : rsList)
admin.majorCompactRegionServer(rs.getServerName()).get();
Thread.sleep(5000);
int countAfterMajorCompaction = countStoreFilesInFamilies(regions, families);
Assert.assertEquals(countAfterMajorCompaction, 3);
} finally {
if (ht != null) {
TEST_UTIL.deleteTable(table);
}
}
createTableWithDefaultConf(tableName, Optional.empty(), families);
loadData(tableName, families, 3000, 8);
List<HRegionServer> rsList =
TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().stream()
.map(rsThread -> rsThread.getRegionServer()).collect(Collectors.toList());
List<Region> regions = new ArrayList<>();
rsList.forEach(rs -> regions.addAll(rs.getOnlineRegions(tableName)));
Assert.assertEquals(regions.size(), 1);
int countBefore = countStoreFilesInFamilies(regions, families);
Assert.assertTrue(countBefore > 0);
// Minor compaction for all region servers.
for (HRegionServer rs : rsList)
admin.compactRegionServer(rs.getServerName()).get();
Thread.sleep(5000);
int countAfterMinorCompaction = countStoreFilesInFamilies(regions, families);
Assert.assertTrue(countAfterMinorCompaction < countBefore);
// Major compaction for all region servers.
for (HRegionServer rs : rsList)
admin.majorCompactRegionServer(rs.getServerName()).get();
Thread.sleep(5000);
int countAfterMajorCompaction = countStoreFilesInFamilies(regions, families);
Assert.assertEquals(countAfterMajorCompaction, 3);
}
private void compactionTest(final String tableName, final int flushes,
@Test
public void testCompact() throws Exception {
compactionTest(TableName.valueOf("testCompact1"), 8, CompactionState.MAJOR, false);
compactionTest(TableName.valueOf("testCompact2"), 15, CompactionState.MINOR, false);
compactionTest(TableName.valueOf("testCompact3"), 8, CompactionState.MAJOR, true);
compactionTest(TableName.valueOf("testCompact4"), 15, CompactionState.MINOR, true);
}
private void compactionTest(final TableName tableName, final int flushes,
final CompactionState expectedState, boolean singleFamily) throws Exception {
// Create a table with regions
final TableName table = TableName.valueOf(tableName);
byte[] family = Bytes.toBytes("family");
byte[][] families =
{ family, Bytes.add(family, Bytes.toBytes("2")), Bytes.add(family, Bytes.toBytes("3")) };
Table ht = null;
try {
ht = TEST_UTIL.createTable(table, families);
loadData(ht, families, 3000, flushes);
List<Region> regions = new ArrayList<>();
TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads()
.forEach(rsThread -> regions.addAll(rsThread.getRegionServer().getOnlineRegions(table)));
Assert.assertEquals(regions.size(), 1);
int countBefore = countStoreFilesInFamilies(regions, families);
int countBeforeSingleFamily = countStoreFilesInFamily(regions, family);
assertTrue(countBefore > 0); // there should be some data files
if (expectedState == CompactionState.MINOR) {
if (singleFamily) {
admin.compact(table, Optional.of(family)).get();
} else {
admin.compact(table, Optional.empty()).get();
}
createTableWithDefaultConf(tableName, Optional.empty(), families);
loadData(tableName, families, 3000, flushes);
List<Region> regions = new ArrayList<>();
TEST_UTIL
.getHBaseCluster()
.getLiveRegionServerThreads()
.forEach(rsThread -> regions.addAll(rsThread.getRegionServer().getOnlineRegions(tableName)));
Assert.assertEquals(regions.size(), 1);
int countBefore = countStoreFilesInFamilies(regions, families);
int countBeforeSingleFamily = countStoreFilesInFamily(regions, family);
assertTrue(countBefore > 0); // there should be some data files
if (expectedState == CompactionState.MINOR) {
if (singleFamily) {
admin.compact(tableName, Optional.of(family)).get();
} else {
if (singleFamily) {
admin.majorCompact(table, Optional.of(family)).get();
} else {
admin.majorCompact(table, Optional.empty()).get();
}
admin.compact(tableName, Optional.empty()).get();
}
long curt = System.currentTimeMillis();
long waitTime = 5000;
long endt = curt + waitTime;
CompactionState state = TEST_UTIL.getAdmin().getCompactionState(table);
while (state == CompactionState.NONE && curt < endt) {
Thread.sleep(10);
state = TEST_UTIL.getAdmin().getCompactionState(table);
curt = System.currentTimeMillis();
}
// Now, should have the right compaction state,
// otherwise, the compaction should have already been done
if (expectedState != state) {
for (Region region : regions) {
state = CompactionState.valueOf(region.getCompactionState().toString());
assertEquals(CompactionState.NONE, state);
}
} else {
if (singleFamily) {
admin.majorCompact(tableName, Optional.of(family)).get();
} else {
// Wait until the compaction is done
state = TEST_UTIL.getAdmin().getCompactionState(table);
while (state != CompactionState.NONE && curt < endt) {
Thread.sleep(10);
state = TEST_UTIL.getAdmin().getCompactionState(table);
}
// Now, compaction should be done.
admin.majorCompact(tableName, Optional.empty()).get();
}
}
long curt = System.currentTimeMillis();
long waitTime = 5000;
long endt = curt + waitTime;
CompactionState state = TEST_UTIL.getAdmin().getCompactionState(tableName);
while (state == CompactionState.NONE && curt < endt) {
Thread.sleep(10);
state = TEST_UTIL.getAdmin().getCompactionState(tableName);
curt = System.currentTimeMillis();
}
// Now, should have the right compaction state,
// otherwise, the compaction should have already been done
if (expectedState != state) {
for (Region region : regions) {
state = CompactionState.valueOf(region.getCompactionState().toString());
assertEquals(CompactionState.NONE, state);
}
int countAfter = countStoreFilesInFamilies(regions, families);
int countAfterSingleFamily = countStoreFilesInFamily(regions, family);
assertTrue(countAfter < countBefore);
if (!singleFamily) {
if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter);
else assertTrue(families.length < countAfter);
} else {
int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily;
// assert only change was to single column family
assertTrue(singleFamDiff == (countBefore - countAfter));
if (expectedState == CompactionState.MAJOR) {
assertTrue(1 == countAfterSingleFamily);
} else {
assertTrue(1 < countAfterSingleFamily);
}
} else {
// Wait until the compaction is done
state = TEST_UTIL.getAdmin().getCompactionState(tableName);
while (state != CompactionState.NONE && curt < endt) {
Thread.sleep(10);
state = TEST_UTIL.getAdmin().getCompactionState(tableName);
}
} finally {
if (ht != null) {
TEST_UTIL.deleteTable(table);
// Now, compaction should be done.
assertEquals(CompactionState.NONE, state);
}
int countAfter = countStoreFilesInFamilies(regions, families);
int countAfterSingleFamily = countStoreFilesInFamily(regions, family);
assertTrue(countAfter < countBefore);
if (!singleFamily) {
if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter);
else assertTrue(families.length < countAfter);
} else {
int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily;
// assert only change was to single column family
assertTrue(singleFamDiff == (countBefore - countAfter));
if (expectedState == CompactionState.MAJOR) {
assertTrue(1 == countAfterSingleFamily);
} else {
assertTrue(1 < countAfterSingleFamily);
}
}
}
@ -623,8 +569,9 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
return count;
}
private static void loadData(final Table ht, final byte[][] families, final int rows,
private static void loadData(final TableName tableName, final byte[][] families, final int rows,
final int flushes) throws IOException {
RawAsyncTable table = ASYNC_CONN.getRawTable(tableName);
List<Put> puts = new ArrayList<>(rows);
byte[] qualifier = Bytes.toBytes("val");
for (int i = 0; i < flushes; i++) {
@ -636,7 +583,7 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
}
puts.add(p);
}
ht.put(puts);
table.putAll(puts).join();
TEST_UTIL.flush();
puts.clear();
}

30
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
View File

@ -45,10 +45,13 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
/**
* Class to test asynchronous replication admin operations.
*/
@RunWith(Parameterized.class)
@Category({LargeTests.class, ClientTests.class})
public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
@ -57,9 +60,6 @@ public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
private final String ID_SECOND = "2";
private final String KEY_SECOND = "127.0.0.1:2181:/hbase2";
@Rule
public TestName name = new TestName();
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 60000);
@ -142,12 +142,12 @@ public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
public void testAppendPeerTableCFs() throws Exception {
ReplicationPeerConfig rpc1 = new ReplicationPeerConfig();
rpc1.setClusterKey(KEY_ONE);
final TableName tableName1 = TableName.valueOf(name.getMethodName() + "t1");
final TableName tableName2 = TableName.valueOf(name.getMethodName() + "t2");
final TableName tableName3 = TableName.valueOf(name.getMethodName() + "t3");
final TableName tableName4 = TableName.valueOf(name.getMethodName() + "t4");
final TableName tableName5 = TableName.valueOf(name.getMethodName() + "t5");
final TableName tableName6 = TableName.valueOf(name.getMethodName() + "t6");
final TableName tableName1 = TableName.valueOf(tableName.getNameAsString() + "t1");
final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "t2");
final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "t3");
final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "t4");
final TableName tableName5 = TableName.valueOf(tableName.getNameAsString() + "t5");
final TableName tableName6 = TableName.valueOf(tableName.getNameAsString() + "t6");
// Add a valid peer
admin.addReplicationPeer(ID_ONE, rpc1).join();
@ -244,10 +244,10 @@ public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
public void testRemovePeerTableCFs() throws Exception {
ReplicationPeerConfig rpc1 = new ReplicationPeerConfig();
rpc1.setClusterKey(KEY_ONE);
final TableName tableName1 = TableName.valueOf(name.getMethodName() + "t1");
final TableName tableName2 = TableName.valueOf(name.getMethodName() + "t2");
final TableName tableName3 = TableName.valueOf(name.getMethodName() + "t3");
final TableName tableName4 = TableName.valueOf(name.getMethodName() + "t4");
final TableName tableName1 = TableName.valueOf(tableName.getNameAsString() + "t1");
final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "t2");
final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "t3");
final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "t4");
// Add a valid peer
admin.addReplicationPeer(ID_ONE, rpc1).join();
Map<TableName, List<String>> tableCFs = new HashMap<>();
@ -360,8 +360,8 @@ public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
public void testNamespacesAndTableCfsConfigConflict() throws Exception {
String ns1 = "ns1";
String ns2 = "ns2";
final TableName tableName1 = TableName.valueOf(ns1 + ":" + name.getMethodName());
final TableName tableName2 = TableName.valueOf(ns2 + ":" + name.getMethodName() + "2");
final TableName tableName1 = TableName.valueOf(ns1 + ":" + tableName.getNameAsString() + "1");
final TableName tableName2 = TableName.valueOf(ns2 + ":" + tableName.getNameAsString() + "2");
ReplicationPeerConfig rpc = new ReplicationPeerConfig();
rpc.setClusterKey(KEY_ONE);

24
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSnapshotAdminApi.java
View File

@ -29,12 +29,16 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.regex.Pattern;
@RunWith(Parameterized.class)
@Category({ LargeTests.class, ClientTests.class })
public class TestAsyncSnapshotAdminApi extends TestAsyncAdminBase {
@ -42,15 +46,6 @@ public class TestAsyncSnapshotAdminApi extends TestAsyncAdminBase {
String snapshotName2 = "snapshotName2";
String snapshotName3 = "snapshotName3";
@Rule
public TestName testName = new TestName();
TableName tableName;
@Before
public void setup() {
tableName = TableName.valueOf(testName.getMethodName());
}
@After
public void cleanup() throws Exception {
admin.deleteSnapshots(Pattern.compile(".*")).get();
@ -175,10 +170,13 @@ public class TestAsyncSnapshotAdminApi extends TestAsyncAdminBase {
admin.snapshot(snapshotName3, tableName).get();
Assert.assertEquals(admin.listSnapshots().get().size(), 3);
Assert.assertEquals(admin.listSnapshots(Pattern.compile("(.*)")).get().size(), 3);
Assert.assertEquals(admin.listSnapshots(Pattern.compile("snapshotName(\\d+)")).get().size(), 3);
Assert.assertEquals(admin.listSnapshots(Pattern.compile("snapshotName[1|3]")).get().size(), 2);
Assert.assertEquals(admin.listSnapshots(Pattern.compile("snapshot(.*)")).get().size(), 3);
Assert.assertEquals(admin.listSnapshots(Optional.of(Pattern.compile("(.*)"))).get().size(), 3);
Assert.assertEquals(admin.listSnapshots(Optional.of(Pattern.compile("snapshotName(\\d+)")))
.get().size(), 3);
Assert.assertEquals(admin.listSnapshots(Optional.of(Pattern.compile("snapshotName[1|3]")))
.get().size(), 2);
Assert.assertEquals(admin.listSnapshots(Optional.of(Pattern.compile("snapshot(.*)"))).get()
.size(), 3);
Assert.assertEquals(
admin.listTableSnapshots(Pattern.compile("testListSnapshots"), Pattern.compile("s(.*)")).get()
.size(),

814
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java
View File

File diff suppressed because it is too large Load Diff