HBASE-15044 Region normalization should be allowed when underlying namespace has quota

hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java

@@ -72,8 +72,10 @@ import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
 import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
 import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
+import org.apache.hadoop.hbase.quotas.QuotaExceededException;
 import org.apache.hadoop.hbase.quotas.RegionStateListener;
 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
 import org.apache.hadoop.hbase.regionserver.RegionServerAbortedException;
@@ -1654,9 +1656,13 @@ public class AssignmentManager {
         }
       }
       RegionLocations rl =  MetaTableAccessor.getRegionLocations(result);
-      if (rl == null) continue;
+      if (rl == null) {
+        continue;
+      }
       HRegionLocation[] locations = rl.getRegionLocations();
-      if (locations == null) continue;
+      if (locations == null) {
+        continue;
+      }
       for (HRegionLocation hrl : locations) {
         if (hrl == null) continue;
         HRegionInfo regionInfo = hrl.getRegionInfo();
@@ -1996,7 +2002,9 @@ public class AssignmentManager {
    */
   public boolean waitOnRegionToClearRegionsInTransition(final HRegionInfo hri, long timeOut)
       throws InterruptedException {
-    if (!regionStates.isRegionInTransition(hri)) return true;
+    if (!regionStates.isRegionInTransition(hri)) {
+      return true;
+    }
     long end = (timeOut <= 0) ? Long.MAX_VALUE : EnvironmentEdgeManager.currentTime()
         + timeOut;
     // There is already a timeout monitor on regions in transition so I
@@ -2836,6 +2844,9 @@ public class AssignmentManager {
           regionStateListener.onRegionSplit(hri);
           errorMsg = onRegionReadyToSplit(current, hri, serverName, transition);
         } catch (IOException exp) {
+            if (exp instanceof QuotaExceededException) {
+              server.getRegionNormalizer().planSkipped(hri, PlanType.SPLIT);
+            }
             errorMsg = StringUtils.stringifyException(exp);
         }
         break;

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -92,6 +92,7 @@ import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
 import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
 import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore;
 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory;
@@ -1326,6 +1327,11 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
     return true;
   }
 
+  @VisibleForTesting
+  public RegionNormalizer getRegionNormalizer() {
+    return this.normalizer;
+  }
+
   /**
    * Perform normalization of cluster (invoked by {@link RegionNormalizerChore}).
    *
@@ -1353,11 +1359,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
       Collections.shuffle(allEnabledTables);
 
       for (TableName table : allEnabledTables) {
-        if (quotaManager.getNamespaceQuotaManager() != null &&
-            quotaManager.getNamespaceQuotaManager().getState(table.getNamespaceAsString()) != null){
-          LOG.debug("Skipping normalizing " + table + " since its namespace has quota");
-          continue;
-        }
         TableDescriptor tblDesc = getTableDescriptors().getDescriptor(table);
         if (table.isSystemTable() || (tblDesc != null &&
             tblDesc.getHTableDescriptor() != null &&
@@ -1366,7 +1367,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
             + " table or doesn't have auto normalization turned on");
           continue;
         }
-        this.normalizer.computePlanForTable(table).execute(clusterConnection.getAdmin());
+        NormalizationPlan plan = this.normalizer.computePlanForTable(table);
+        plan.execute(clusterConnection.getAdmin());
       }
     }
     // If Region did not generate any plans, it means the cluster is already balanced.

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java

@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotDisabledException;
 import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.executor.ExecutorService;
@@ -89,6 +90,11 @@ public interface MasterServices extends Server {
    */
   MasterQuotaManager getMasterQuotaManager();
   
+  /**
+   * @return Master's instance of {@link RegionNormalizer}
+   */
+  RegionNormalizer getRegionNormalizer();
+
   /**
    * @return Master's instance of {@link ProcedureExecutor}
    */

hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master.normalizer;
 
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
 
 /**
  * Plan which signifies that no normalization is required,
@@ -45,4 +46,9 @@ public final class EmptyNormalizationPlan implements NormalizationPlan {
   @Override
   public void execute(Admin admin) {
   }
+
+  @Override
+  public PlanType getType() {
+    return PlanType.NONE;
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java

@@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
 
 import java.io.IOException;
 
@@ -41,6 +42,11 @@ public class MergeNormalizationPlan implements NormalizationPlan {
     this.secondRegion = secondRegion;
   }
 
+  @Override
+  public PlanType getType() {
+    return PlanType.MERGE;
+  }
+
   HRegionInfo getFirstRegion() {
     return firstRegion;
   }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/NormalizationPlan.java

@@ -26,10 +26,20 @@ import org.apache.hadoop.hbase.client.Admin;
  */
 @InterfaceAudience.Private
 public interface NormalizationPlan {
+  enum PlanType {
+    SPLIT,
+    MERGE,
+    NONE
+  }
 
   /**
    * Executes normalization plan on cluster (does actual splitting/merging work).
    * @param admin instance of Admin
    */
   void execute(Admin admin);
+
+  /**
+   * @return the type of this plan
+   */
+  PlanType getType();
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java

@@ -19,9 +19,11 @@
 package org.apache.hadoop.hbase.master.normalizer;
 
 import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
 
 /**
  * Performs "normalization" of regions on the cluster, making sure that suboptimal
@@ -48,4 +50,18 @@ public interface RegionNormalizer {
    * @return Next (perhaps most urgent) normalization action to perform
    */
   NormalizationPlan computePlanForTable(TableName table) throws HBaseIOException;
+
+  /**
+   * Notification for the case where plan couldn't be executed due to constraint violation, such as
+   * namespace quota
+   * @param hri the region which is involved in the plan
+   * @param type type of plan
+   */
+  void planSkipped(HRegionInfo hri, PlanType type);
+  
+  /**
+   * @param type type of plan for which skipped count is to be returned
+   * @return the count of plans of specified type which were skipped
+   */
+  long getSkippedCount(NormalizationPlan.PlanType type);
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java

@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
 import org.apache.hadoop.hbase.util.Triple;
 
 import java.util.ArrayList;
@@ -59,6 +60,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
   private static final Log LOG = LogFactory.getLog(SimpleRegionNormalizer.class);
   private static final int MIN_REGION_COUNT = 3;
   private MasterServices masterServices;
+  private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length];
 
   /**
    * Set the master service.
@@ -85,6 +87,16 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
     }
   };
 
+  @Override
+  public void planSkipped(HRegionInfo hri, PlanType type) {
+    skippedCount[type.ordinal()]++;
+  }
+
+  @Override
+  public long getSkippedCount(NormalizationPlan.PlanType type) {
+    return skippedCount[type.ordinal()];
+  }
+
   /**
    * Computes next most "urgent" normalization action on the table.
    * Action may be either a split, or a merge, or no action.

hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java

@@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -42,6 +43,11 @@ public class SplitNormalizationPlan implements NormalizationPlan {
     this.splitPoint = splitPoint;
   }
 
+  @Override
+  public PlanType getType() {
+    return PlanType.SPLIT;
+  }
+
   public HRegionInfo getRegionInfo() {
     return regionInfo;
   }

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java

@@ -63,6 +63,7 @@ import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLog
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
+import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
@@ -251,6 +252,11 @@ public class TestCatalogJanitor {
       return null;
     }
 
+    @Override
+    public RegionNormalizer getRegionNormalizer() {
+      return null;
+    }
+
     @Override
     public MasterFileSystem getMasterFileSystem() {
       return this.mfs;

hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java

@@ -25,11 +25,16 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.MetaTableAccessor;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.TableNamespaceManager;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
+import org.apache.hadoop.hbase.namespace.TestNamespaceAuditor;
+import org.apache.hadoop.hbase.quotas.QuotaUtil;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -63,9 +68,11 @@ public class TestSimpleRegionNormalizerOnCluster {
     // we will retry operations when PleaseHoldException is thrown
     TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
     TEST_UTIL.getConfiguration().setBoolean(HConstants.HBASE_NORMALIZER_ENABLED, true);
+    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
 
     // Start a cluster of two regionservers.
     TEST_UTIL.startMiniCluster(1);
+    TestNamespaceAuditor.waitForQuotaEnabled(TEST_UTIL);
     admin = TEST_UTIL.getHBaseAdmin();
   }
 
@@ -74,11 +81,27 @@ public class TestSimpleRegionNormalizerOnCluster {
     TEST_UTIL.shutdownMiniCluster();
   }
 
-  @Test(timeout = 60000)
+  @Test(timeout = 90000)
   @SuppressWarnings("deprecation")
   public void testRegionNormalizationSplitOnCluster() throws Exception {
-    final TableName TABLENAME =
-      TableName.valueOf("testRegionNormalizationSplitOnCluster");
+    testRegionNormalizationSplitOnCluster(false);
+    testRegionNormalizationSplitOnCluster(true);
+  }
+
+  void testRegionNormalizationSplitOnCluster(boolean limitedByQuota) throws Exception {
+    TableName TABLENAME;
+    if (limitedByQuota) {
+      String nsp = "np2";
+      NamespaceDescriptor nspDesc =
+          NamespaceDescriptor.create(nsp)
+          .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
+          .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
+      admin.createNamespace(nspDesc);
+      TABLENAME = TableName.valueOf(nsp +
+        TableName.NAMESPACE_DELIM + "testRegionNormalizationSplitOnCluster");
+    } else {
+      TABLENAME = TableName.valueOf("testRegionNormalizationSplitOnCluster");
+    }
     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
     HMaster m = cluster.getMaster();
 
@@ -119,20 +142,25 @@ public class TestSimpleRegionNormalizerOnCluster {
 
     admin.flush(TABLENAME);
 
-    System.out.println(admin.getTableDescriptor(TABLENAME));
-
     assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
 
     // Now trigger a split and stop when the split is in progress
     Thread.sleep(5000); // to let region load to update
     m.normalizeRegions();
-
+    if (limitedByQuota) {
+      long skippedSplitcnt = 0;
+      do {
+        skippedSplitcnt = m.getRegionNormalizer().getSkippedCount(PlanType.SPLIT);
+        Thread.sleep(100);
+      } while (skippedSplitcnt == 0L);
+      assert(skippedSplitcnt > 0);
+    } else {
       while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME) < 6) {
         LOG.info("Waiting for normalization split to complete");
         Thread.sleep(100);
       }
-
       assertEquals(6, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
+    }
 
     admin.disableTable(TABLENAME);
     admin.deleteTable(TABLENAME);

hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java

@@ -115,7 +115,7 @@ public class TestNamespaceAuditor {
     conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
       RegionServerObserver.class);
     UTIL.startMiniCluster(1, 1);
-    waitForQuotaEnabled();
+    waitForQuotaEnabled(UTIL);
     ADMIN = UTIL.getHBaseAdmin();
   }
 
@@ -620,11 +620,11 @@ public class TestNamespaceAuditor {
         .getTables().size(), after.getTables().size());
   }
 
-  private static void waitForQuotaEnabled() throws Exception {
-    UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
+  public static void waitForQuotaEnabled(final HBaseTestingUtility util) throws Exception {
+    util.waitFor(60000, new Waiter.Predicate<Exception>() {
       @Override
       public boolean evaluate() throws Exception {
-        HMaster master = UTIL.getHBaseCluster().getMaster();
+        HMaster master = util.getHBaseCluster().getMaster();
         if (master == null) {
           return false;
         }
@@ -638,7 +638,7 @@ public class TestNamespaceAuditor {
     UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
     UTIL.getHBaseCluster().waitOnMaster(0);
     UTIL.getHBaseCluster().startMaster();
-    waitForQuotaEnabled();
+    waitForQuotaEnabled(UTIL);
   }
 
   private NamespaceAuditor getQuotaManager() {