From c1b6d47e7974a5d9d75933bab9a28572e9d95c14 Mon Sep 17 00:00:00 2001 From: tedyu Date: Wed, 30 Dec 2015 08:04:23 -0800 Subject: [PATCH] HBASE-15044 Region normalization should be allowed when underlying namespace has quota --- .../hbase/master/AssignmentManager.java | 19 +++++-- .../apache/hadoop/hbase/master/HMaster.java | 14 +++--- .../hadoop/hbase/master/MasterServices.java | 6 +++ .../normalizer/EmptyNormalizationPlan.java | 6 +++ .../normalizer/MergeNormalizationPlan.java | 6 +++ .../master/normalizer/NormalizationPlan.java | 10 ++++ .../master/normalizer/RegionNormalizer.java | 16 ++++++ .../normalizer/SimpleRegionNormalizer.java | 12 +++++ .../normalizer/SplitNormalizationPlan.java | 6 +++ .../hbase/master/TestCatalogJanitor.java | 6 +++ .../TestSimpleRegionNormalizerOnCluster.java | 50 +++++++++++++++---- .../hbase/namespace/TestNamespaceAuditor.java | 10 ++-- 12 files changed, 135 insertions(+), 26 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index f7f98fe9287..4feb2e745db 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -72,8 +72,10 @@ import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper; import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer; import org.apache.hadoop.hbase.master.handler.DisableTableHandler; import org.apache.hadoop.hbase.master.handler.EnableTableHandler; +import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition; import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; +import org.apache.hadoop.hbase.quotas.QuotaExceededException; import org.apache.hadoop.hbase.quotas.RegionStateListener; import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.regionserver.RegionServerAbortedException; @@ -1654,9 +1656,13 @@ public class AssignmentManager { } } RegionLocations rl = MetaTableAccessor.getRegionLocations(result); - if (rl == null) continue; + if (rl == null) { + continue; + } HRegionLocation[] locations = rl.getRegionLocations(); - if (locations == null) continue; + if (locations == null) { + continue; + } for (HRegionLocation hrl : locations) { if (hrl == null) continue; HRegionInfo regionInfo = hrl.getRegionInfo(); @@ -1996,7 +2002,9 @@ public class AssignmentManager { */ public boolean waitOnRegionToClearRegionsInTransition(final HRegionInfo hri, long timeOut) throws InterruptedException { - if (!regionStates.isRegionInTransition(hri)) return true; + if (!regionStates.isRegionInTransition(hri)) { + return true; + } long end = (timeOut <= 0) ? Long.MAX_VALUE : EnvironmentEdgeManager.currentTime() + timeOut; // There is already a timeout monitor on regions in transition so I @@ -2836,7 +2844,10 @@ public class AssignmentManager { regionStateListener.onRegionSplit(hri); errorMsg = onRegionReadyToSplit(current, hri, serverName, transition); } catch (IOException exp) { - errorMsg = StringUtils.stringifyException(exp); + if (exp instanceof QuotaExceededException) { + server.getRegionNormalizer().planSkipped(hri, PlanType.SPLIT); + } + errorMsg = StringUtils.stringifyException(exp); } break; case SPLIT_PONR: diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index bdb19f436ef..8e51f25a6f6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -92,6 +92,7 @@ import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; import org.apache.hadoop.hbase.master.cleaner.LogCleaner; import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler; +import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer; import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore; import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory; @@ -1326,6 +1327,11 @@ public class HMaster extends HRegionServer implements MasterServices, Server { return true; } + @VisibleForTesting + public RegionNormalizer getRegionNormalizer() { + return this.normalizer; + } + /** * Perform normalization of cluster (invoked by {@link RegionNormalizerChore}). * @@ -1353,11 +1359,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server { Collections.shuffle(allEnabledTables); for (TableName table : allEnabledTables) { - if (quotaManager.getNamespaceQuotaManager() != null && - quotaManager.getNamespaceQuotaManager().getState(table.getNamespaceAsString()) != null){ - LOG.debug("Skipping normalizing " + table + " since its namespace has quota"); - continue; - } TableDescriptor tblDesc = getTableDescriptors().getDescriptor(table); if (table.isSystemTable() || (tblDesc != null && tblDesc.getHTableDescriptor() != null && @@ -1366,7 +1367,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server { + " table or doesn't have auto normalization turned on"); continue; } - this.normalizer.computePlanForTable(table).execute(clusterConnection.getAdmin()); + NormalizationPlan plan = this.normalizer.computePlanForTable(table); + plan.execute(clusterConnection.getAdmin()); } } // If Region did not generate any plans, it means the cluster is already balanced. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java index 935644f922e..af0e4907ec0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.executor.ExecutorService; @@ -88,6 +89,11 @@ public interface MasterServices extends Server { * @return Master's instance of {@link MasterQuotaManager} */ MasterQuotaManager getMasterQuotaManager(); + + /** + * @return Master's instance of {@link RegionNormalizer} + */ + RegionNormalizer getRegionNormalizer(); /** * @return Master's instance of {@link ProcedureExecutor} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java index a36dd07eb28..5aecc48726c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master.normalizer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; /** * Plan which signifies that no normalization is required, @@ -45,4 +46,9 @@ public final class EmptyNormalizationPlan implements NormalizationPlan { @Override public void execute(Admin admin) { } + + @Override + public PlanType getType() { + return PlanType.NONE; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java index 08a58a50594..e2035bbef90 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; import java.io.IOException; @@ -41,6 +42,11 @@ public class MergeNormalizationPlan implements NormalizationPlan { this.secondRegion = secondRegion; } + @Override + public PlanType getType() { + return PlanType.MERGE; + } + HRegionInfo getFirstRegion() { return firstRegion; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/NormalizationPlan.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/NormalizationPlan.java index 96eed8ca71d..9f866d3a250 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/NormalizationPlan.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/NormalizationPlan.java @@ -26,10 +26,20 @@ import org.apache.hadoop.hbase.client.Admin; */ @InterfaceAudience.Private public interface NormalizationPlan { + enum PlanType { + SPLIT, + MERGE, + NONE + } /** * Executes normalization plan on cluster (does actual splitting/merging work). * @param admin instance of Admin */ void execute(Admin admin); + + /** + * @return the type of this plan + */ + PlanType getType(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java index 19abcf22476..d60474d8392 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java @@ -19,9 +19,11 @@ package org.apache.hadoop.hbase.master.normalizer; import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; /** * Performs "normalization" of regions on the cluster, making sure that suboptimal @@ -48,4 +50,18 @@ public interface RegionNormalizer { * @return Next (perhaps most urgent) normalization action to perform */ NormalizationPlan computePlanForTable(TableName table) throws HBaseIOException; + + /** + * Notification for the case where plan couldn't be executed due to constraint violation, such as + * namespace quota + * @param hri the region which is involved in the plan + * @param type type of plan + */ + void planSkipped(HRegionInfo hri, PlanType type); + + /** + * @param type type of plan for which skipped count is to be returned + * @return the count of plans of specified type which were skipped + */ + long getSkippedCount(NormalizationPlan.PlanType type); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index fe6034b4deb..fe10bd12d39 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; import org.apache.hadoop.hbase.util.Triple; import java.util.ArrayList; @@ -59,6 +60,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer { private static final Log LOG = LogFactory.getLog(SimpleRegionNormalizer.class); private static final int MIN_REGION_COUNT = 3; private MasterServices masterServices; + private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length]; /** * Set the master service. @@ -85,6 +87,16 @@ public class SimpleRegionNormalizer implements RegionNormalizer { } }; + @Override + public void planSkipped(HRegionInfo hri, PlanType type) { + skippedCount[type.ordinal()]++; + } + + @Override + public long getSkippedCount(NormalizationPlan.PlanType type) { + return skippedCount[type.ordinal()]; + } + /** * Computes next most "urgent" normalization action on the table. * Action may be either a split, or a merge, or no action. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java index c96988af01d..b95bfb7380a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; import java.io.IOException; import java.util.Arrays; @@ -42,6 +43,11 @@ public class SplitNormalizationPlan implements NormalizationPlan { this.splitPoint = splitPoint; } + @Override + public PlanType getType() { + return PlanType.SPLIT; + } + public HRegionInfo getRegionInfo() { return regionInfo; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java index b5f20f5bd6a..8e35bbf5dd8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java @@ -63,6 +63,7 @@ import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLog import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator; +import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; @@ -251,6 +252,11 @@ public class TestCatalogJanitor { return null; } + @Override + public RegionNormalizer getRegionNormalizer() { + return null; + } + @Override public MasterFileSystem getMasterFileSystem() { return this.mfs; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java index d24023d75ed..4fe42edbea8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java @@ -25,11 +25,16 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.TableNamespaceManager; +import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; +import org.apache.hadoop.hbase.namespace.TestNamespaceAuditor; +import org.apache.hadoop.hbase.quotas.QuotaUtil; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -63,9 +68,11 @@ public class TestSimpleRegionNormalizerOnCluster { // we will retry operations when PleaseHoldException is thrown TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3); TEST_UTIL.getConfiguration().setBoolean(HConstants.HBASE_NORMALIZER_ENABLED, true); + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); // Start a cluster of two regionservers. TEST_UTIL.startMiniCluster(1); + TestNamespaceAuditor.waitForQuotaEnabled(TEST_UTIL); admin = TEST_UTIL.getHBaseAdmin(); } @@ -74,11 +81,27 @@ public class TestSimpleRegionNormalizerOnCluster { TEST_UTIL.shutdownMiniCluster(); } - @Test(timeout = 60000) + @Test(timeout = 90000) @SuppressWarnings("deprecation") public void testRegionNormalizationSplitOnCluster() throws Exception { - final TableName TABLENAME = - TableName.valueOf("testRegionNormalizationSplitOnCluster"); + testRegionNormalizationSplitOnCluster(false); + testRegionNormalizationSplitOnCluster(true); + } + + void testRegionNormalizationSplitOnCluster(boolean limitedByQuota) throws Exception { + TableName TABLENAME; + if (limitedByQuota) { + String nsp = "np2"; + NamespaceDescriptor nspDesc = + NamespaceDescriptor.create(nsp) + .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5") + .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build(); + admin.createNamespace(nspDesc); + TABLENAME = TableName.valueOf(nsp + + TableName.NAMESPACE_DELIM + "testRegionNormalizationSplitOnCluster"); + } else { + TABLENAME = TableName.valueOf("testRegionNormalizationSplitOnCluster"); + } MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HMaster m = cluster.getMaster(); @@ -119,21 +142,26 @@ public class TestSimpleRegionNormalizerOnCluster { admin.flush(TABLENAME); - System.out.println(admin.getTableDescriptor(TABLENAME)); - assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME)); // Now trigger a split and stop when the split is in progress Thread.sleep(5000); // to let region load to update m.normalizeRegions(); - - while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME) < 6) { - LOG.info("Waiting for normalization split to complete"); - Thread.sleep(100); + if (limitedByQuota) { + long skippedSplitcnt = 0; + do { + skippedSplitcnt = m.getRegionNormalizer().getSkippedCount(PlanType.SPLIT); + Thread.sleep(100); + } while (skippedSplitcnt == 0L); + assert(skippedSplitcnt > 0); + } else { + while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME) < 6) { + LOG.info("Waiting for normalization split to complete"); + Thread.sleep(100); + } + assertEquals(6, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME)); } - assertEquals(6, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME)); - admin.disableTable(TABLENAME); admin.deleteTable(TABLENAME); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java index 7ded3d3f7fb..e3b841bd3b5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java @@ -115,7 +115,7 @@ public class TestNamespaceAuditor { conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class, RegionServerObserver.class); UTIL.startMiniCluster(1, 1); - waitForQuotaEnabled(); + waitForQuotaEnabled(UTIL); ADMIN = UTIL.getHBaseAdmin(); } @@ -620,11 +620,11 @@ public class TestNamespaceAuditor { .getTables().size(), after.getTables().size()); } - private static void waitForQuotaEnabled() throws Exception { - UTIL.waitFor(60000, new Waiter.Predicate() { + public static void waitForQuotaEnabled(final HBaseTestingUtility util) throws Exception { + util.waitFor(60000, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { - HMaster master = UTIL.getHBaseCluster().getMaster(); + HMaster master = util.getHBaseCluster().getMaster(); if (master == null) { return false; } @@ -638,7 +638,7 @@ public class TestNamespaceAuditor { UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again"); UTIL.getHBaseCluster().waitOnMaster(0); UTIL.getHBaseCluster().startMaster(); - waitForQuotaEnabled(); + waitForQuotaEnabled(UTIL); } private NamespaceAuditor getQuotaManager() {