HBASE-22796 [HBCK2] Add fix of overlaps to fixMeta hbck Service

Signed-off-by: Sakthi <sakthi@apache.org>

hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java

@@ -794,6 +794,9 @@ public interface RegionInfo {
    * @see #isDegenerate()
    */
   default boolean isOverlap(RegionInfo other) {
+    if (other == null) {
+      return false;
+    }
     if (!getTable().equals(other.getTable())) {
       return false;
     }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java

@@ -457,6 +457,12 @@ public class CatalogJanitor extends ScheduledChore {
       return this.holes;
     }
 
+    /**
+     * @return Overlap pairs found as we scanned hbase:meta; ordered by hbase:meta
+     *   table sort. Pairs of overlaps may have overlap with subsequent pairs.
+     * @see MetaFixer#calculateMerges(int, List) where we aggregate overlaps
+     *   for a single 'merge' call.
+     */
     public List<Pair<RegionInfo, RegionInfo>> getOverlaps() {
       return this.overlaps;
     }
@@ -479,20 +485,20 @@ public class CatalogJanitor extends ScheduledChore {
 
     @Override
     public String toString() {
-      StringBuffer sb = new StringBuffer();
+      StringBuilder sb = new StringBuilder();
       for (Pair<RegionInfo, RegionInfo> p: this.holes) {
         if (sb.length() > 0) {
           sb.append(", ");
         }
-        sb.append("hole=" + p.getFirst().getRegionNameAsString() + "/" +
+        sb.append("hole=").append(p.getFirst().getRegionNameAsString()).append("/").
-            p.getSecond().getRegionNameAsString());
+            append(p.getSecond().getRegionNameAsString());
       }
       for (Pair<RegionInfo, RegionInfo> p: this.overlaps) {
         if (sb.length() > 0) {
           sb.append(", ");
         }
-        sb.append("overlap=" + p.getFirst().getRegionNameAsString() + "/" +
+        sb.append("overlap=").append(p.getFirst().getRegionNameAsString()).append("/").
-            p.getSecond().getRegionNameAsString());
+            append(p.getSecond().getRegionNameAsString());
       }
       for (byte [] r: this.emptyRegionInfo) {
         if (sb.length() > 0) {
@@ -534,6 +540,16 @@ public class CatalogJanitor extends ScheduledChore {
      */
     private RegionInfo previous = null;
 
+    /**
+     * Keep account of the highest end key seen as we move through hbase:meta.
+     * Usually, the current RegionInfo has the highest end key but if an overlap,
+     * this may no longer hold. An overlap may be a region with startkey 'd' and
+     * endkey 'g'. The next region in meta may be 'e' to 'f' and then 'f' to 'g'.
+     * Looking at previous and current meta row, we won't know about the 'd' to 'g'
+     * overlap unless we keep a running 'highest-endpoint-seen'.
+     */
+    private RegionInfo highestEndKeyRegionInfo = null;
+
     ReportMakingVisitor(MasterServices services) {
       this.services = services;
     }
@@ -587,7 +603,7 @@ public class CatalogJanitor extends ScheduledChore {
             MetaTableAccessor.getRegionInfoColumn());
       } else {
         ri = locations.getDefaultRegionLocation().getRegion();
-        checkServer(metaTableRow.getRow(), locations);
+        checkServer(locations);
       }
 
       if (ri == null) {
@@ -615,13 +631,22 @@ public class CatalogJanitor extends ScheduledChore {
           if (!this.previous.isNext(ri)) {
             if (this.previous.isOverlap(ri)) {
               addOverlap(this.previous, ri);
+            } else if (ri.isOverlap(this.highestEndKeyRegionInfo)) {
+              // We may have seen a region a few rows back that overlaps this one.
+              addOverlap(this.highestEndKeyRegionInfo, ri);
             } else {
               addHole(this.previous, ri);
             }
+          } else if (ri.isOverlap(this.highestEndKeyRegionInfo)) {
+            // We may have seen a region a few rows back that overlaps this one
+            // even though it properly 'follows' the region just before.
+            addOverlap(this.highestEndKeyRegionInfo, ri);
           }
         }
       }
       this.previous = ri;
+      this.highestEndKeyRegionInfo =
+          MetaFixer.getRegionInfoWithLargestEndKey(ri, this.highestEndKeyRegionInfo);
       return ri;
     }
 
@@ -658,7 +683,7 @@ public class CatalogJanitor extends ScheduledChore {
     /**
      * Run through referenced servers and save off unknown and the dead.
      */
-    private void checkServer(byte [] metaTableRow, RegionLocations locations) {
+    private void checkServer(RegionLocations locations) {
       if (this.services == null) {
         // Can't do this test if no services.
         return;
@@ -672,6 +697,11 @@ public class CatalogJanitor extends ScheduledChore {
         if (sn == null) {
           continue;
         }
+        if (location.getRegion() == null) {
+          LOG.warn("Empty RegionInfo in {}", location);
+          // This should never happen but if it does, will mess up below.
+          continue;
+        }
         // skip the offline regions which belong to disabled table.
         if (isTableDisabled(location.getRegion())) {
           continue;
@@ -680,7 +710,7 @@ public class CatalogJanitor extends ScheduledChore {
             isServerKnownAndOnline(sn);
         switch (state) {
           case UNKNOWN:
-            this.report.unknownServers.add(new Pair(location.getRegion(), sn));
+            this.report.unknownServers.add(new Pair<>(location.getRegion(), sn));
             break;
 
           default:

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetaFixer.java

@@ -18,8 +18,12 @@
 package org.apache.hadoop.hbase.master;
 
 import java.io.IOException;
-import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
@@ -28,6 +32,7 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.exceptions.MergeRegionException;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -36,20 +41,32 @@ import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
+
 /**
  * Server-side fixing of bad or inconsistent state in hbase:meta.
  * Distinct from MetaTableAccessor because {@link MetaTableAccessor} is about low-level
  * manipulations driven by the Master. This class MetaFixer is
- * employed by the Master and it 'knows' about holes and orphan
+ * employed by the Master and it 'knows' about holes and orphans
  * and encapsulates their fixing on behalf of the Master.
  */
 @InterfaceAudience.Private
 class MetaFixer {
   private static final Logger LOG = LoggerFactory.getLogger(MetaFixer.class);
+  private static final String MAX_MERGE_COUNT_KEY = "hbase.master.metafixer.max.merge.count";
+  private static final int MAX_MERGE_COUNT_DEFAULT = 10;
   private final MasterServices masterServices;
+  /**
+   * Maximum for many regions to merge at a time.
+   */
+  private final int maxMergeCount;
 
   MetaFixer(MasterServices masterServices) {
     this.masterServices = masterServices;
+    this.maxMergeCount = this.masterServices.getConfiguration().
+        getInt(MAX_MERGE_COUNT_KEY, MAX_MERGE_COUNT_DEFAULT);
   }
 
   void fix() throws IOException {
@@ -66,14 +83,12 @@ class MetaFixer {
   /**
    * If hole, it papers it over by adding a region in the filesystem and to hbase:meta.
    * Does not assign.
-   * @return True if we fixed any 'holes'.
    */
-  boolean fixHoles(CatalogJanitor.Report report) throws IOException {
+  void fixHoles(CatalogJanitor.Report report) throws IOException {
-    boolean result = false;
     List<Pair<RegionInfo, RegionInfo>> holes = report.getHoles();
     if (holes.isEmpty()) {
       LOG.debug("No holes.");
-      return result;
+      return;
     }
     for (Pair<RegionInfo, RegionInfo> p: holes) {
       RegionInfo ri = getHoleCover(p);
@@ -86,11 +101,10 @@ class MetaFixer {
       // in hbase:meta (if the below fails). Should be able to rerun the fix.
       // The second call to createRegionDir will just go through. Idempotent.
       Put put = MetaTableAccessor.makePutFromRegionInfo(ri, HConstants.LATEST_TIMESTAMP);
-      MetaTableAccessor.putsToMetaTable(this.masterServices.getConnection(), Arrays.asList(put));
+      MetaTableAccessor.putsToMetaTable(this.masterServices.getConnection(),
+          Collections.singletonList(put));
       LOG.info("Fixed hole by adding {}; region is NOT assigned (assign to online).", ri);
-      result = true;
     }
-    return result;
   }
 
   /**
@@ -136,28 +150,94 @@ class MetaFixer {
     return RegionInfoBuilder.newBuilder(tn).setStartKey(start).setEndKey(end).build();
   }
 
-  boolean fixOverlaps(CatalogJanitor.Report report) throws IOException {
+  /**
-    boolean result = false;
+   * Fix overlaps noted in CJ consistency report.
-    List<Pair<RegionInfo, RegionInfo>> overlaps = report.getOverlaps();
+   */
+  void fixOverlaps(CatalogJanitor.Report report) throws IOException {
+    for (Set<RegionInfo> regions: calculateMerges(maxMergeCount, report.getOverlaps())) {
+      RegionInfo [] regionsArray = regions.toArray(new RegionInfo [] {});
+      try {
+        this.masterServices.mergeRegions(regionsArray,
+            false, HConstants.NO_NONCE, HConstants.NO_NONCE);
+      } catch (MergeRegionException mre) {
+        LOG.warn("Failed overlap fix of {}", regionsArray, mre);
+      }
+    }
+  }
+
+  /**
+   * Run through <code>overlaps</code> and return a list of merges to run.
+   * Presumes overlaps are ordered (which they are coming out of the CatalogJanitor
+   * consistency report).
+   * @param maxMergeCount Maximum regions to merge at a time (avoid merging
+   *   100k regions in one go!)
+   */
+  @VisibleForTesting
+  static List<SortedSet<RegionInfo>> calculateMerges(int maxMergeCount,
+      List<Pair<RegionInfo, RegionInfo>> overlaps) {
     if (overlaps.isEmpty()) {
       LOG.debug("No overlaps.");
-      return result;
+      return Collections.emptyList();
     }
-    for (Pair<RegionInfo, RegionInfo> p: overlaps) {
+    List<SortedSet<RegionInfo>> merges = new ArrayList<>();
-      RegionInfo ri = getHoleCover(p);
+    SortedSet<RegionInfo> currentMergeSet = new TreeSet<>();
-      if (ri == null) {
+    RegionInfo regionInfoWithlargestEndKey =  null;
-        continue;
+    for (Pair<RegionInfo, RegionInfo> pair: overlaps) {
+      if (regionInfoWithlargestEndKey != null) {
+        if (!isOverlap(regionInfoWithlargestEndKey, pair) ||
+            currentMergeSet.size() >= maxMergeCount) {
+          merges.add(currentMergeSet);
+          currentMergeSet = new TreeSet<>();
+        }
       }
-      Configuration configuration = this.masterServices.getConfiguration();
+      currentMergeSet.add(pair.getFirst());
-      HRegion.createRegionDir(configuration, ri, FSUtils.getRootDir(configuration));
+      currentMergeSet.add(pair.getSecond());
-      // If an error here, then we'll have a region in the filesystem but not
+      regionInfoWithlargestEndKey = getRegionInfoWithLargestEndKey(
-      // in hbase:meta (if the below fails). Should be able to rerun the fix.
+        getRegionInfoWithLargestEndKey(pair.getFirst(), pair.getSecond()),
-      // The second call to createRegionDir will just go through. Idempotent.
+          regionInfoWithlargestEndKey);
-      Put put = MetaTableAccessor.makePutFromRegionInfo(ri, HConstants.LATEST_TIMESTAMP);
-      MetaTableAccessor.putsToMetaTable(this.masterServices.getConnection(), Arrays.asList(put));
-      LOG.info("Fixed hole by adding {}; region is NOT assigned (assign to online).", ri);
-      result = true;
     }
-    return result;
+    merges.add(currentMergeSet);
+    return merges;
+  }
+
+  /**
+   * @return Either <code>a</code> or <code>b</code>, whichever has the
+   *   endkey that is furthest along in the Table.
+   */
+  @VisibleForTesting
+  static RegionInfo getRegionInfoWithLargestEndKey(RegionInfo a, RegionInfo b) {
+    if (a == null) {
+      // b may be null.
+      return b;
+    }
+    if (b == null) {
+      // Both are null. The return is not-defined.
+      return a;
+    }
+    if (!a.getTable().equals(b.getTable())) {
+      // This is an odd one. This should be the right answer.
+      return b;
+    }
+    if (a.isLast()) {
+      return a;
+    }
+    if (b.isLast()) {
+      return b;
+    }
+    int compare = Bytes.compareTo(a.getEndKey(), b.getEndKey());
+    return compare == 0 || compare > 0? a: b;
+  }
+
+  /**
+   * @return True if an overlap found between passed in <code>ri</code> and
+   *   the <code>pair</code>. Does NOT check the pairs themselves overlap.
+   */
+  @VisibleForTesting
+  static boolean isOverlap(RegionInfo ri, Pair<RegionInfo, RegionInfo> pair) {
+    if (ri == null || pair == null) {
+      // Can't be an overlap in either of these cases.
+      return false;
+    }
+    return ri.isOverlap(pair.getFirst()) || ri.isOverlap(pair.getSecond());
   }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateNode.java

@@ -294,7 +294,7 @@ public class RegionStateNode implements Comparable<RegionStateNode> {
     RegionInfo ri = getRegionInfo();
     State s = state;
     if (s != State.OPEN) {
-      throw new DoNotRetryRegionException(ri.getEncodedName() + " is no OPEN; state=" + s);
+      throw new DoNotRetryRegionException(ri.getEncodedName() + " is not OPEN; state=" + s);
     }
     if (ri.isSplitParent()) {
       throw new DoNotRetryRegionException(

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaFixer.java

@@ -17,7 +17,10 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -26,8 +29,11 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MetaTableAccessor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.util.Threads;
+
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -36,9 +42,6 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 
 @Category({MasterTests.class, LargeTests.class})
 public class TestMetaFixer {
@@ -47,7 +50,6 @@ public class TestMetaFixer {
       HBaseClassTestRule.forClass(TestMetaFixer.class);
   @Rule
   public TestName name = new TestName();
-  private static final Logger LOG = LoggerFactory.getLogger(TestMetaFixer.class);
 
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
 
@@ -75,21 +77,20 @@ public class TestMetaFixer {
     MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
     services.getCatalogJanitor().scan();
     CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
-    Assert.assertTrue(report.isEmpty());
+    assertTrue(report.isEmpty());
     int originalCount = ris.size();
     // Remove first, last and middle region. See if hole gets plugged. Table has 26 regions.
     deleteRegion(services, ris.get(ris.size() -1));
     deleteRegion(services, ris.get(3));
     deleteRegion(services, ris.get(0));
-    ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
     services.getCatalogJanitor().scan();
     report = services.getCatalogJanitor().getLastReport();
     Assert.assertEquals(report.toString(), 3, report.getHoles().size());
     MetaFixer fixer = new MetaFixer(services);
-    Assert.assertTrue(fixer.fixHoles(report));
+    fixer.fixHoles(report);
     services.getCatalogJanitor().scan();
     report = services.getCatalogJanitor().getLastReport();
-    Assert.assertTrue(report.toString(), report.isEmpty());
+    assertTrue(report.toString(), report.isEmpty());
     // Disable and reenable so the added regions get reassigned.
     TEST_UTIL.getAdmin().disableTable(tn);
     TEST_UTIL.getAdmin().enableTable(tn);
@@ -110,18 +111,61 @@ public class TestMetaFixer {
     List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
     MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
     services.getCatalogJanitor().scan();
-    CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
-    int originalCount = ris.size();
     deleteRegion(services, ris.get(0));
     services.getCatalogJanitor().scan();
-    report = services.getCatalogJanitor().getLastReport();
+    CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
     ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
-    Assert.assertTrue(ris.isEmpty());
+    assertTrue(ris.isEmpty());
     MetaFixer fixer = new MetaFixer(services);
-    Assert.assertFalse(fixer.fixHoles(report));
+    fixer.fixHoles(report);
     report = services.getCatalogJanitor().getLastReport();
-    Assert.assertTrue(report.isEmpty());
+    assertTrue(report.isEmpty());
     ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
     Assert.assertEquals(0, ris.size());
   }
+
+  private static void makeOverlap(MasterServices services, RegionInfo a, RegionInfo b)
+      throws IOException {
+    RegionInfo overlapRegion = RegionInfoBuilder.newBuilder(a.getTable()).
+        setStartKey(a.getStartKey()).
+        setEndKey(b.getEndKey()).
+        build();
+    MetaTableAccessor.putsToMetaTable(services.getConnection(),
+        Collections.singletonList(MetaTableAccessor.makePutFromRegionInfo(overlapRegion,
+            System.currentTimeMillis())));
+    // TODO: Add checks at assign time to PREVENT being able to assign over existing assign.
+    services.getAssignmentManager().assign(overlapRegion);
+  }
+
+  @Test
+  public void testOverlap() throws IOException {
+    TableName tn = TableName.valueOf(this.name.getMethodName());
+    TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
+    List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
+    assertTrue(ris.size() > 5);
+    MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
+    services.getCatalogJanitor().scan();
+    CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
+    assertTrue(report.isEmpty());
+    // Make a simple overlap spanning second and third region.
+    makeOverlap(services, ris.get(1), ris.get(3));
+    makeOverlap(services, ris.get(2), ris.get(3));
+    makeOverlap(services, ris.get(2), ris.get(4));
+    Threads.sleep(10000);
+    services.getCatalogJanitor().scan();
+    report = services.getCatalogJanitor().getLastReport();
+    Assert.assertEquals(6, report.getOverlaps().size());
+    Assert.assertEquals(1, MetaFixer.calculateMerges(10, report.getOverlaps()).size());
+    MetaFixer fixer = new MetaFixer(services);
+    fixer.fixOverlaps(report);
+    while (true) {
+      services.getCatalogJanitor().scan();
+      report = services.getCatalogJanitor().getLastReport();
+      if (report.isEmpty()) {
+        break;
+      }
+      Threads.sleep(10);
+    }
+    assertTrue(report.toString(), report.isEmpty());
+  }
 }

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaFixerNoCluster.java

@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedSet;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+
+
+/**
+ * Test small utility methods inside {@link MetaFixer}.
+ * For cluster tests see {@link TestMetaFixer}
+ */
+@Category({MasterTests.class, SmallTests.class})
+public class TestMetaFixerNoCluster {
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+      HBaseClassTestRule.forClass(TestMetaFixerNoCluster.class);
+  private static byte [] A = Bytes.toBytes("a");
+  private static byte [] B = Bytes.toBytes("b");
+  private static byte [] C = Bytes.toBytes("c");
+  private static byte [] D = Bytes.toBytes("d");
+  private static RegionInfo ALL = RegionInfoBuilder.FIRST_META_REGIONINFO;
+  private static RegionInfo _ARI = RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).
+      setEndKey(A).build();
+  private static RegionInfo _BRI = RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).
+          setEndKey(B).build();
+  private static RegionInfo ABRI = RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).
+          setStartKey(A).setEndKey(B).build();
+  private static RegionInfo ACRI =
+      org.apache.hadoop.hbase.client.RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).
+          setStartKey(A).setEndKey(C).build();
+  private static RegionInfo CDRI =
+      org.apache.hadoop.hbase.client.RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).
+          setStartKey(C).setEndKey(D).build();
+  private static RegionInfo ADRI =
+      org.apache.hadoop.hbase.client.RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).
+          setStartKey(A).setEndKey(D).build();
+  private static RegionInfo D_RI =
+      org.apache.hadoop.hbase.client.RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).
+          setStartKey(D).build();
+  private static RegionInfo C_RI =
+      org.apache.hadoop.hbase.client.RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).
+          setStartKey(C).build();
+
+  @Test
+  public void testGetRegionInfoWithLargestEndKey() {
+    assertTrue(MetaFixer.getRegionInfoWithLargestEndKey(_ARI, _BRI).equals(_BRI));
+    assertTrue(MetaFixer.getRegionInfoWithLargestEndKey(C_RI, D_RI).equals(C_RI));
+    assertTrue(MetaFixer.getRegionInfoWithLargestEndKey(ABRI, CDRI).equals(CDRI));
+    assertTrue(MetaFixer.getRegionInfoWithLargestEndKey(null, CDRI).equals(CDRI));
+    assertTrue(MetaFixer.getRegionInfoWithLargestEndKey(null, null) == null);
+  }
+
+  @Test
+  public void testIsOverlap() {
+    assertTrue(MetaFixer.isOverlap(_BRI, new Pair<RegionInfo, RegionInfo>(ABRI, ACRI)));
+    assertFalse(MetaFixer.isOverlap(_ARI, new Pair<RegionInfo, RegionInfo>(C_RI, D_RI)));
+    assertTrue(MetaFixer.isOverlap(ADRI, new Pair<RegionInfo, RegionInfo>(CDRI, C_RI)));
+    assertFalse(MetaFixer.isOverlap(_BRI, new Pair<RegionInfo, RegionInfo>(CDRI, C_RI)));
+  }
+
+  @Test
+  public void testCalculateMergesNoAggregation() {
+    List<Pair<RegionInfo, RegionInfo>> overlaps = new ArrayList<>();
+    overlaps.add(new Pair<RegionInfo, RegionInfo>(_ARI, _BRI));
+    overlaps.add(new Pair<RegionInfo, RegionInfo>(C_RI, D_RI));
+    List<SortedSet<RegionInfo>> merges = MetaFixer.calculateMerges(10, overlaps);
+    assertEquals(2, merges.size());
+    assertEquals(2, merges.get(0).size());
+    assertEquals(2, merges.get(1).size());
+  }
+
+  @Test
+  public void testCalculateMergesAggregation() {
+    List<Pair<RegionInfo, RegionInfo>> overlaps = new ArrayList<>();
+    overlaps.add(new Pair<RegionInfo, RegionInfo>(ALL, D_RI));
+    overlaps.add(new Pair<RegionInfo, RegionInfo>(_ARI, _BRI));
+    overlaps.add(new Pair<RegionInfo, RegionInfo>(C_RI, D_RI));
+    List<SortedSet<RegionInfo>> merges = MetaFixer.calculateMerges(10, overlaps);
+    assertEquals(1, merges.size());
+    assertEquals(5, merges.get(0).size());
+  }
+
+  @Test
+  public void testCalculateMergesNoRepeatOfRegionNames() {
+    List<Pair<RegionInfo, RegionInfo>> overlaps = new ArrayList<>();
+    overlaps.add(new Pair<RegionInfo, RegionInfo>(_BRI, ABRI));
+    overlaps.add(new Pair<RegionInfo, RegionInfo>(ABRI, ADRI));
+    List<SortedSet<RegionInfo>> merges = MetaFixer.calculateMerges(10, overlaps);
+    assertEquals(1, merges.size());
+    // There should be three regions to merge, not four.
+    assertEquals(3, merges.get(0).size());
+  }
+
+  @Test
+  public void testCalculateMergesRespectsMax() {
+    List<Pair<RegionInfo, RegionInfo>> overlaps = new ArrayList<>();
+    overlaps.add(new Pair<RegionInfo, RegionInfo>(_BRI, ABRI));
+    overlaps.add(new Pair<RegionInfo, RegionInfo>(ABRI, ADRI));
+    overlaps.add(new Pair<RegionInfo, RegionInfo>(C_RI, D_RI));
+    List<SortedSet<RegionInfo>> merges = MetaFixer.calculateMerges(3, overlaps);
+    assertEquals(2, merges.size());
+    // There should be three regions to merge, not four.
+    assertEquals(3, merges.get(0).size());
+    assertEquals(2, merges.get(1).size());
+  }
+}