diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index b8153a97ae4..20b1fc08868 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -1292,10 +1292,24 @@ public interface Admin extends Abortable, Closeable { * the request was submitted successfully. We need to check logs for the details of which regions * were split/merged. * - * @return true if region normalizer ran, false otherwise. + * @return {@code true} if region normalizer ran, {@code false} otherwise. * @throws IOException if a remote or network exception occurs */ - boolean normalize() throws IOException; + default boolean normalize() throws IOException { + return normalize(new NormalizeTableFilterParams.Builder().build()); + } + + /** + * Invoke region normalizer. Can NOT run for various reasons. Check logs. + * This is a non-blocking invocation to region normalizer. If return value is true, it means + * the request was submitted successfully. We need to check logs for the details of which regions + * were split/merged. + * + * @param ntfp limit to tables matching the specified filter. + * @return {@code true} if region normalizer ran, {@code false} otherwise. + * @throws IOException if a remote or network exception occurs + */ + boolean normalize(NormalizeTableFilterParams ntfp) throws IOException; /** * Query the current state of the region normalizer. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 87e1df9c6aa..b37785d2eb6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -1253,7 +1253,17 @@ public interface AsyncAdmin { * @return true if region normalizer ran, false otherwise. The return value will be wrapped by a * {@link CompletableFuture} */ - CompletableFuture normalize(); + default CompletableFuture normalize() { + return normalize(new NormalizeTableFilterParams.Builder().build()); + } + + /** + * Invoke region normalizer. Can NOT run for various reasons. Check logs. + * @param ntfp limit to tables matching the specified filter. + * @return true if region normalizer ran, false otherwise. The return value will be wrapped by a + * {@link CompletableFuture} + */ + CompletableFuture normalize(NormalizeTableFilterParams ntfp); /** * Turn the cleaner chore on/off. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 376c5dcba90..29bf6ea043f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -704,8 +704,8 @@ class AsyncHBaseAdmin implements AsyncAdmin { } @Override - public CompletableFuture normalize() { - return wrap(rawAdmin.normalize()); + public CompletableFuture normalize(NormalizeTableFilterParams ntfp) { + return wrap(rawAdmin.normalize(ntfp)); } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index ac7857885a4..137572e8a2f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -1557,18 +1557,13 @@ public class HBaseAdmin implements Admin { } } - /** - * Invoke region normalizer. Can NOT run for various reasons. Check logs. - * - * @return True if region normalizer ran, false otherwise. - */ @Override - public boolean normalize() throws IOException { + public boolean normalize(NormalizeTableFilterParams ntfp) throws IOException { return executeCallable(new MasterCallable(getConnection(), getRpcControllerFactory()) { @Override protected Boolean rpcCall() throws Exception { return master.normalize(getRpcController(), - RequestConverter.buildNormalizeRequest()).getNormalizerRan(); + RequestConverter.buildNormalizeRequest(ntfp)).getNormalizerRan(); } }); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NormalizeTableFilterParams.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NormalizeTableFilterParams.java new file mode 100644 index 00000000000..982ec5b0065 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NormalizeTableFilterParams.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.util.List; +import org.apache.hadoop.hbase.TableName; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * A collection of criteria used for table selection. The logic of table selection is as follows: + *
    + *
  • + * When no parameter values are provided, an unfiltered list of all user tables is returned. + *
    • + *
  • + * When a list of {@link TableName TableNames} are provided, the filter starts with any of + * these tables that exist. + *
    • + *
  • + * When a {@code namespace} name is provided, the filter starts with all the tables present in + * that namespace. + *
    • + *
  • + * If both a list of {@link TableName TableNames} and a {@code namespace} name are provided, + * the {@link TableName} list is honored and the {@code namespace} name is ignored. + *
    • + *
  • + * If a {@code regex} is provided, this subset of {@link TableName TableNames} is further + * reduced to those that match the provided regular expression. + *
    • + *
+ */ +@InterfaceAudience.Public +public final class NormalizeTableFilterParams { + private final List tableNames; + private final String regex; + private final String namespace; + + private NormalizeTableFilterParams(final List tableNames, final String regex, + final String namespace) { + this.tableNames = tableNames; + this.regex = regex; + this.namespace = namespace; + } + + public List getTableNames() { + return tableNames; + } + + public String getRegex() { + return regex; + } + + public String getNamespace() { + return namespace; + } + + /** + * Used to instantiate an instance of {@link NormalizeTableFilterParams}. + */ + public static class Builder { + private List tableNames; + private String regex; + private String namespace; + + public Builder tableFilterParams(final NormalizeTableFilterParams ntfp) { + this.tableNames = ntfp.getTableNames(); + this.regex = ntfp.getRegex(); + this.namespace = ntfp.getNamespace(); + return this; + } + + public Builder tableNames(final List tableNames) { + this.tableNames = tableNames; + return this; + } + + public Builder regex(final String regex) { + this.regex = regex; + return this; + } + + public Builder namespace(final String namespace) { + this.namespace = namespace; + return this; + } + + public NormalizeTableFilterParams build() { + return new NormalizeTableFilterParams(tableNames, regex, namespace); + } + } +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 02cbcefaaff..392447a4063 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -3257,14 +3257,18 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { } @Override - public CompletableFuture normalize() { + public CompletableFuture normalize(NormalizeTableFilterParams ntfp) { + return normalize(RequestConverter.buildNormalizeRequest(ntfp)); + } + + private CompletableFuture normalize(NormalizeRequest request) { return this - . newMasterCaller() - .action( - (controller, stub) -> this. call( - controller, stub, RequestConverter.buildNormalizeRequest(), - (s, c, req, done) -> s.normalize(c, req, done), (resp) -> resp.getNormalizerRan())) - .call(); + . newMasterCaller() + .action( + (controller, stub) -> this.call( + controller, stub, request, MasterService.Interface::normalize, + NormalizeResponse::getNormalizerRan)) + .call(); } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index c82243a9837..094065cefc8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -2251,6 +2251,13 @@ public final class ProtobufUtil { .setQualifier(UnsafeByteOperations.unsafeWrap(tableName.getQualifier())).build(); } + public static List toProtoTableNameList(List tableNameList) { + if (tableNameList == null) { + return new ArrayList<>(); + } + return tableNameList.stream().map(ProtobufUtil::toProtoTableName).collect(Collectors.toList()); + } + public static List toTableNameList(List tableNamesList) { if (tableNamesList == null) { return new ArrayList<>(); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index f377d6c8daa..c877050b8b7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.LogQueryFilter; import org.apache.hadoop.hbase.client.MasterSwitchType; import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.NormalizeTableFilterParams; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionCoprocessorServiceExec; import org.apache.hadoop.hbase.client.RegionInfo; @@ -1724,8 +1725,18 @@ public final class RequestConverter { * * @return a NormalizeRequest */ - public static NormalizeRequest buildNormalizeRequest() { - return NormalizeRequest.newBuilder().build(); + public static NormalizeRequest buildNormalizeRequest(NormalizeTableFilterParams ntfp) { + final NormalizeRequest.Builder builder = NormalizeRequest.newBuilder(); + if (ntfp.getTableNames() != null) { + builder.addAllTableNames(ProtobufUtil.toProtoTableNameList(ntfp.getTableNames())); + } + if (ntfp.getRegex() != null) { + builder.setRegex(ntfp.getRegex()); + } + if (ntfp.getNamespace() != null) { + builder.setNamespace(ntfp.getNamespace()); + } + return builder.build(); } /** diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index 505eb81aa2f..78674ab5f15 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -353,6 +353,9 @@ message IsSplitOrMergeEnabledResponse { } message NormalizeRequest { + repeated TableName table_names = 1; + optional string regex = 2; + optional string namespace = 3; } message NormalizeResponse { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 6218536d90a..4d2042284b7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -21,7 +21,6 @@ import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_SPLIT_COORDINATED import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS; import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK; import static org.apache.hadoop.hbase.util.DNS.MASTER_HOSTNAME_KEY; - import com.google.protobuf.Descriptors; import com.google.protobuf.Service; import java.io.IOException; @@ -40,6 +39,7 @@ import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -82,9 +82,9 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.UnknownRegionException; -import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.MasterSwitchType; +import org.apache.hadoop.hbase.client.NormalizeTableFilterParams; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.RegionStatesCount; @@ -214,6 +214,7 @@ import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.hadoop.hbase.zookeeper.ZNodePaths; +import org.apache.hbase.thirdparty.com.google.common.collect.Sets; import org.apache.yetus.audience.InterfaceAudience; import org.apache.zookeeper.KeeperException; import org.eclipse.jetty.server.Server; @@ -222,12 +223,10 @@ import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.webapp.WebAppContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.common.collect.Maps; import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; - import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; @@ -1882,14 +1881,18 @@ public class HMaster extends HRegionServer implements MasterServices { return this.normalizer; } + public boolean normalizeRegions() throws IOException { + return normalizeRegions(new NormalizeTableFilterParams.Builder().build()); + } + /** - * Perform normalization of cluster (invoked by {@link RegionNormalizerChore}). + * Perform normalization of cluster. * * @return true if an existing normalization was already in progress, or if a new normalization * was performed successfully; false otherwise (specifically, if HMaster finished initializing * or normalization is globally disabled). */ - public boolean normalizeRegions() throws IOException { + public boolean normalizeRegions(final NormalizeTableFilterParams ntfp) throws IOException { final long startTime = EnvironmentEdgeManager.currentTime(); if (regionNormalizerTracker == null || !regionNormalizerTracker.isNormalizerOn()) { LOG.debug("Region normalization is disabled, don't run region normalizer."); @@ -1910,12 +1913,19 @@ public class HMaster extends HRegionServer implements MasterServices { int affectedTables = 0; try { - final List allEnabledTables = - new ArrayList<>(tableStateManager.getTablesInStates(TableState.State.ENABLED)); - Collections.shuffle(allEnabledTables); + final Set matchingTables = getTableDescriptors(new LinkedList<>(), + ntfp.getNamespace(), ntfp.getRegex(), ntfp.getTableNames(), false) + .stream() + .map(TableDescriptor::getTableName) + .collect(Collectors.toSet()); + final Set allEnabledTables = + tableStateManager.getTablesInStates(TableState.State.ENABLED); + final List targetTables = + new ArrayList<>(Sets.intersection(matchingTables, allEnabledTables)); + Collections.shuffle(targetTables); final List submittedPlanProcIds = new ArrayList<>(); - for (TableName table : allEnabledTables) { + for (TableName table : targetTables) { if (table.isSystemTable()) { continue; } @@ -3370,9 +3380,9 @@ public class HMaster extends HRegionServer implements MasterServices { } /** - * @return list of table table descriptors after filtering by regex and whether to include system - * tables, etc. - * @throws IOException + * Return a list of table table descriptors after applying any provided filter parameters. Note + * that the user-facing description of this filter logic is presented on the class-level javadoc + * of {@link NormalizeTableFilterParams}. */ private List getTableDescriptors(final List htds, final String namespace, final String regex, final List tableNameList, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index b46abc8549e..3e7a2ac30a5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.MasterSwitchType; +import org.apache.hadoop.hbase.client.NormalizeTableFilterParams; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; @@ -1932,7 +1933,14 @@ public class MasterRpcServices extends RSRpcServices implements NormalizeRequest request) throws ServiceException { rpcPreCheck("normalize"); try { - return NormalizeResponse.newBuilder().setNormalizerRan(master.normalizeRegions()).build(); + final NormalizeTableFilterParams ntfp = new NormalizeTableFilterParams.Builder() + .tableNames(ProtobufUtil.toTableNameList(request.getTableNamesList())) + .regex(request.hasRegex() ? request.getRegex() : null) + .namespace(request.hasNamespace() ? request.getNamespace() : null) + .build(); + return NormalizeResponse.newBuilder() + .setNormalizerRan(master.normalizeRegions(ntfp)) + .build(); } catch (IOException ex) { throw new ServiceException(ex); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java index da4c52ea8bd..7d6ea20b1ba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.concurrent.TimeUnit; @@ -36,6 +37,7 @@ import org.apache.hadoop.hbase.Size; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter.ExplainingPredicate; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.NormalizeTableFilterParams; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Table; @@ -94,6 +96,9 @@ public class TestSimpleRegionNormalizerOnCluster { // no way for the test to set the regionId on a created region, so disable this feature. TEST_UTIL.getConfiguration().setInt("hbase.normalizer.merge.min_region_age.days", 0); + // disable the normalizer coming along and running via Chore + TEST_UTIL.getConfiguration().setInt("hbase.normalizer.period", Integer.MAX_VALUE); + TEST_UTIL.startMiniCluster(1); TestNamespaceAuditor.waitForQuotaInitialize(TEST_UTIL); admin = TEST_UTIL.getAdmin(); @@ -107,13 +112,13 @@ public class TestSimpleRegionNormalizerOnCluster { } @Before - public void before() throws IOException { + public void before() throws Exception { // disable the normalizer ahead of time, let the test enable it when its ready. admin.normalizerSwitch(false); } @Test - public void testHonorsNormalizerSwitch() throws IOException { + public void testHonorsNormalizerSwitch() throws Exception { assertFalse(admin.isNormalizerEnabled()); assertFalse(admin.normalize()); assertFalse(admin.normalizerSwitch(true)); @@ -220,14 +225,103 @@ public class TestSimpleRegionNormalizerOnCluster { } } - private static TableName buildTableNameForQuotaTest(final String methodName) throws IOException { + @Test + public void testHonorsNamespaceFilter() throws Exception { + final NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build(); + final TableName tn1 = TableName.valueOf("ns", name.getMethodName()); + final TableName tn2 = TableName.valueOf(name.getMethodName()); + + try { + admin.createNamespace(namespaceDescriptor); + final int tn1RegionCount = createTableBegsSplit(tn1, true, false); + final int tn2RegionCount = createTableBegsSplit(tn2, true, false); + final NormalizeTableFilterParams ntfp = new NormalizeTableFilterParams.Builder() + .namespace("ns") + .build(); + + assertFalse(admin.normalizerSwitch(true)); + assertTrue(admin.normalize(ntfp)); + waitForTableSplit(tn1, tn1RegionCount + 1); + + // confirm that tn1 has (tn1RegionCount + 1) number of regions. + // tn2 has tn2RegionCount number of regions because it's not a member of the target namespace. + assertEquals( + tn1 + " should have split.", + tn1RegionCount + 1, + MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tn1)); + waitForTableRegionCount(tn2, tn2RegionCount); + } finally { + dropIfExists(tn1); + dropIfExists(tn2); + } + } + + @Test + public void testHonorsPatternFilter() throws Exception { + final TableName tn1 = TableName.valueOf(name.getMethodName() + "1"); + final TableName tn2 = TableName.valueOf(name.getMethodName() + "2"); + + try { + final int tn1RegionCount = createTableBegsSplit(tn1, true, false); + final int tn2RegionCount = createTableBegsSplit(tn2, true, false); + final NormalizeTableFilterParams ntfp = new NormalizeTableFilterParams.Builder() + .regex(".*[1]") + .build(); + + assertFalse(admin.normalizerSwitch(true)); + assertTrue(admin.normalize(ntfp)); + waitForTableSplit(tn1, tn1RegionCount + 1); + + // confirm that tn1 has (tn1RegionCount + 1) number of regions. + // tn2 has tn2RegionCount number of regions because it fails filter. + assertEquals( + tn1 + " should have split.", + tn1RegionCount + 1, + MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tn1)); + waitForTableRegionCount(tn2, tn2RegionCount); + } finally { + dropIfExists(tn1); + dropIfExists(tn2); + } + } + + @Test + public void testHonorsNameFilter() throws Exception { + final TableName tn1 = TableName.valueOf(name.getMethodName() + "1"); + final TableName tn2 = TableName.valueOf(name.getMethodName() + "2"); + + try { + final int tn1RegionCount = createTableBegsSplit(tn1, true, false); + final int tn2RegionCount = createTableBegsSplit(tn2, true, false); + final NormalizeTableFilterParams ntfp = new NormalizeTableFilterParams.Builder() + .tableNames(Collections.singletonList(tn1)) + .build(); + + assertFalse(admin.normalizerSwitch(true)); + assertTrue(admin.normalize(ntfp)); + waitForTableSplit(tn1, tn1RegionCount + 1); + + // confirm that tn1 has (tn1RegionCount + 1) number of regions. + // tn2 has tn3RegionCount number of regions because it fails filter: + assertEquals( + tn1 + " should have split.", + tn1RegionCount + 1, + MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tn1)); + waitForTableRegionCount(tn2, tn2RegionCount); + } finally { + dropIfExists(tn1); + dropIfExists(tn2); + } + } + + private static TableName buildTableNameForQuotaTest(final String methodName) throws Exception { String nsp = "np2"; NamespaceDescriptor nspDesc = NamespaceDescriptor.create(nsp) .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5") .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build(); admin.createNamespace(nspDesc); - return TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + methodName); + return TableName.valueOf(nsp, methodName); } private static void waitForSkippedSplits(final HMaster master, @@ -347,12 +441,13 @@ public class TestSimpleRegionNormalizerOnCluster { */ private static int createTableBegsSplit(final TableName tableName, final boolean normalizerEnabled, final boolean isMergeEnabled) - throws IOException { + throws Exception { final List generatedRegions = generateTestData(tableName, 1, 1, 2, 3, 5); assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName)); admin.flush(tableName); - final TableDescriptor td = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName)) + final TableDescriptor td = TableDescriptorBuilder + .newBuilder(admin.getDescriptor(tableName)) .setNormalizationEnabled(normalizerEnabled) .setMergeEnabled(isMergeEnabled) .build(); @@ -383,13 +478,14 @@ public class TestSimpleRegionNormalizerOnCluster { *
  • sum of sizes of first two regions < average
    • * */ - private static int createTableBegsMerge(final TableName tableName) throws IOException { + private static int createTableBegsMerge(final TableName tableName) throws Exception { // create 5 regions with sizes to trigger merge of small regions final List generatedRegions = generateTestData(tableName, 1, 1, 3, 3, 5); assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName)); admin.flush(tableName); - final TableDescriptor td = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName)) + final TableDescriptor td = TableDescriptorBuilder + .newBuilder(admin.getDescriptor(tableName)) .setNormalizationEnabled(true) .build(); admin.modifyTable(td); @@ -411,7 +507,7 @@ public class TestSimpleRegionNormalizerOnCluster { return 5; } - private static void dropIfExists(final TableName tableName) throws IOException { + private static void dropIfExists(final TableName tableName) throws Exception { if (tableName != null && admin.tableExists(tableName)) { if (admin.isTableEnabled(tableName)) { admin.disableTable(tableName); diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 1849aad3d75..6c29e548761 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -234,9 +234,54 @@ module Hbase #---------------------------------------------------------------------------------------------- # Requests region normalization for all configured tables in the cluster - # Returns true if normalizer ran successfully - def normalize - @admin.normalize + # Returns true if normalize request was successfully submitted + def normalize(*args) + builder = org.apache.hadoop.hbase.client.NormalizeTableFilterParams::Builder.new + args.each do |arg| + unless arg.is_a?(String) || arg.is_a?(Hash) + raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") + end + + if arg.key?(TABLE_NAME) + table_name = arg.delete(TABLE_NAME) + unless table_name.is_a?(String) + raise(ArgumentError, "#{TABLE_NAME} must be of type String") + end + + builder.tableNames(java.util.Collections.singletonList(TableName.valueOf(table_name))) + elsif arg.key?(TABLE_NAMES) + table_names = arg.delete(TABLE_NAMES) + unless table_names.is_a?(Array) + raise(ArgumentError, "#{TABLE_NAMES} must be of type Array") + end + + table_name_list = java.util.LinkedList.new + table_names.each do |tn| + unless tn.is_a?(String) + raise(ArgumentError, "#{TABLE_NAMES} value #{tn} must be of type String") + end + + table_name_list.add(TableName.valueOf(tn)) + end + builder.tableNames(table_name_list) + elsif arg.key?(REGEX) + regex = arg.delete(REGEX) + raise(ArgumentError, "#{REGEX} must be of type String") unless regex.is_a?(String) + + builder.regex(regex) + elsif arg.key?(NAMESPACE) + namespace = arg.delete(NAMESPACE) + unless namespace.is_a?(String) + raise(ArgumentError, "#{NAMESPACE} must be of type String") + end + + builder.namespace(namespace) + else + raise(ArgumentError, "Unrecognized argument #{arg}") + end + end + ntfp = builder.build + @admin.normalize(ntfp) end #---------------------------------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/hbase_constants.rb b/hbase-shell/src/main/ruby/hbase_constants.rb index badd7d0caaa..712973b18ff 100644 --- a/hbase-shell/src/main/ruby/hbase_constants.rb +++ b/hbase-shell/src/main/ruby/hbase_constants.rb @@ -69,6 +69,7 @@ module HBaseConstants POLICY = 'POLICY'.freeze RAW = 'RAW'.freeze READ_TYPE = 'READ_TYPE'.freeze + REGEX = 'REGEX'.freeze REGIONSERVER = 'REGIONSERVER'.freeze REGION_REPLICATION = 'REGION_REPLICATION'.freeze REGION_REPLICA_ID = 'REGION_REPLICA_ID'.freeze @@ -87,6 +88,8 @@ module HBaseConstants STOPROW = 'STOPROW'.freeze TABLE = 'TABLE'.freeze TABLE_CFS = 'TABLE_CFS'.freeze + TABLE_NAME = 'TABLE_NAME'.freeze + TABLE_NAMES = 'TABLE_NAMES'.freeze TIMERANGE = 'TIMERANGE'.freeze TIMESTAMP = 'TIMESTAMP'.freeze TYPE = 'TYPE'.freeze diff --git a/hbase-shell/src/main/ruby/shell/commands/normalize.rb b/hbase-shell/src/main/ruby/shell/commands/normalize.rb index 2840e845bd6..70e524ae11c 100644 --- a/hbase-shell/src/main/ruby/shell/commands/normalize.rb +++ b/hbase-shell/src/main/ruby/shell/commands/normalize.rb @@ -22,18 +22,24 @@ module Shell class Normalize < Command def help <<-EOF -Trigger region normalizer for all tables which have NORMALIZATION_ENABLED flag set. Returns true - if normalizer ran successfully, false otherwise. Note that this command has no effect - if region normalizer is disabled (make sure it's turned on using 'normalizer_switch' command). +Trigger the region normalizer. Without arguments, invokes the normalizer without a table filter. +Any arguments are used to limit table selection. Returns true if the normalize request was +submitted successfully, false otherwise. Note that this command has no effect if region normalizer +is disabled (make sure it's turned on using 'normalizer_switch' command). - Examples: +Examples: - hbase> normalize + hbase> normalize + hbase> normalize TABLE_NAME => 'my_table' + hbase> normalize TABLE_NAMES => ['foo', 'bar', 'baz'] + hbase> normalize REGEX => 'my_.*' + hbase> normalize NAMESPACE => 'ns1' + hbase> normalize NAMESPACE => 'ns', REGEX => '*._BIG_.*' EOF end - def command - did_normalize_run = !!admin.normalize + def command(*args) + did_normalize_run = !!admin.normalize(*args) formatter.row([did_normalize_run.to_s]) did_normalize_run end diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java index 1d0980b4611..9359d4360ee 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java @@ -1,5 +1,4 @@ -/** - * +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -46,6 +45,7 @@ import org.apache.hadoop.hbase.client.CompactType; import org.apache.hadoop.hbase.client.CompactionState; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.LogQueryFilter; +import org.apache.hadoop.hbase.client.NormalizeTableFilterParams; import org.apache.hadoop.hbase.client.OnlineLogRecord; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.SnapshotDescription; @@ -796,7 +796,7 @@ public class ThriftAdmin implements Admin { } @Override - public boolean normalize() { + public boolean normalize(NormalizeTableFilterParams ntfp) { throw new NotImplementedException("normalize not supported in ThriftAdmin"); }