HBASE-26193 Do not store meta region location as permanent state on zookeeper (#3583)

Signed-off-by: stack <stack@apache.org>
Signed-off-by: Zach York <zyork@apache.org>

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -22,6 +22,7 @@ import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS
 import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK;
 import static org.apache.hadoop.hbase.util.DNS.MASTER_HOSTNAME_KEY;
 
+import com.google.errorprone.annotations.RestrictedApi;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.lang.reflect.Constructor;
@@ -55,6 +56,10 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.CatalogFamilyFormat;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
 import org.apache.hadoop.hbase.ClusterId;
 import org.apache.hadoop.hbase.ClusterMetrics;
 import org.apache.hadoop.hbase.ClusterMetrics.Option;
@@ -81,9 +86,12 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
 import org.apache.hadoop.hbase.client.CompactionState;
 import org.apache.hadoop.hbase.client.MasterSwitchType;
 import org.apache.hadoop.hbase.client.NormalizeTableFilterParams;
+import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
 import org.apache.hadoop.hbase.client.RegionStatesCount;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.client.TableState;
@@ -100,6 +108,7 @@ import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
 import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
 import org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure;
 import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
+import org.apache.hadoop.hbase.master.assignment.RegionStateStore;
 import org.apache.hadoop.hbase.master.assignment.RegionStates;
 import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
 import org.apache.hadoop.hbase.master.balancer.BalancerChore;
@@ -213,6 +222,7 @@ import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
 import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker;
 import org.apache.hadoop.hbase.zookeeper.SnapshotCleanupTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
@@ -389,7 +399,7 @@ public class HMaster extends HRegionServer implements MasterServices {
   private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
   private ProcedureStore procedureStore;
 
-  // the master local storage to store procedure data, etc.
+  // the master local storage to store procedure data, meta region locations, etc.
   private MasterRegion masterRegion;
 
   // handle table states
@@ -753,8 +763,49 @@ public class HMaster extends HRegionServer implements MasterServices {
 
   // Will be overriden in test to inject customized AssignmentManager
   @InterfaceAudience.Private
-  protected AssignmentManager createAssignmentManager(MasterServices master) {
+  protected AssignmentManager createAssignmentManager(MasterServices master,
-    return new AssignmentManager(master);
+    MasterRegion masterRegion) {
+    return new AssignmentManager(master, masterRegion);
+  }
+
+  private void tryMigrateMetaLocationsFromZooKeeper() throws IOException, KeeperException {
+    // try migrate data from zookeeper
+    try (ResultScanner scanner =
+      masterRegion.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
+      if (scanner.next() != null) {
+        // notice that all replicas for a region are in the same row, so the migration can be
+        // done with in a one row put, which means if we have data in catalog family then we can
+        // make sure that the migration is done.
+        LOG.info("The {} family in master local region already has data in it, skip migrating...",
+          HConstants.CATALOG_FAMILY);
+        return;
+      }
+    }
+    // start migrating
+    byte[] row = CatalogFamilyFormat.getMetaKeyForRegion(RegionInfoBuilder.FIRST_META_REGIONINFO);
+    Put put = new Put(row);
+    List<String> metaReplicaNodes = zooKeeper.getMetaReplicaNodes();
+    StringBuilder info = new StringBuilder("Migrating meta locations:");
+    for (String metaReplicaNode : metaReplicaNodes) {
+      int replicaId = zooKeeper.getZNodePaths().getMetaReplicaIdFromZNode(metaReplicaNode);
+      RegionState state = MetaTableLocator.getMetaRegionState(zooKeeper, replicaId);
+      info.append(" ").append(state);
+      put.setTimestamp(state.getStamp());
+      MetaTableAccessor.addRegionInfo(put, state.getRegion());
+      if (state.getServerName() != null) {
+        MetaTableAccessor.addLocation(put, state.getServerName(), HConstants.NO_SEQNUM, replicaId);
+      }
+      put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(put.getRow())
+        .setFamily(HConstants.CATALOG_FAMILY)
+        .setQualifier(RegionStateStore.getStateColumn(replicaId)).setTimestamp(put.getTimestamp())
+        .setType(Cell.Type.Put).setValue(Bytes.toBytes(state.getState().name())).build());
+    }
+    if (!put.isEmpty()) {
+      LOG.info(info.toString());
+      masterRegion.update(r -> r.put(put));
+    } else {
+      LOG.info("No meta location available on zookeeper, skip migrating...");
+    }
   }
 
   /**
@@ -770,6 +821,7 @@ public class HMaster extends HRegionServer implements MasterServices {
    * region server tracker
    * <ol type='i'>
    * <li>Create server manager</li>
+   * <li>Create master local region</li>
    * <li>Create procedure executor, load the procedures, but do not start workers. We will start it
    * later after we finish scheduling SCPs to avoid scheduling duplicated SCPs for the same
    * server</li>
@@ -851,13 +903,16 @@ public class HMaster extends HRegionServer implements MasterServices {
 
     // initialize master local region
     masterRegion = MasterRegionFactory.create(this);
+
+    tryMigrateMetaLocationsFromZooKeeper();
+
     createProcedureExecutor();
     Map<Class<?>, List<Procedure<MasterProcedureEnv>>> procsByType =
       procedureExecutor.getActiveProceduresNoCopy().stream()
         .collect(Collectors.groupingBy(p -> p.getClass()));
 
     // Create Assignment Manager
-    this.assignmentManager = createAssignmentManager(this);
+    this.assignmentManager = createAssignmentManager(this, masterRegion);
     this.assignmentManager.start();
     // TODO: TRSP can perform as the sub procedure for other procedures, so even if it is marked as
     // completed, it could still be in the procedure list. This is a bit strange but is another
@@ -3857,4 +3912,10 @@ public class HMaster extends HRegionServer implements MasterServices {
   public MetaLocationSyncer getMetaLocationSyncer() {
     return metaLocationSyncer;
   }
+
+  @RestrictedApi(explanation = "Should only be called in tests", link = "",
+    allowedOnPath = ".*/src/test/.*")
+  MasterRegion getMasterRegion() {
+    return masterRegion;
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java

@@ -48,6 +48,8 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder;
 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
 import org.apache.hadoop.hbase.client.RegionStatesCount;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableState;
 import org.apache.hadoop.hbase.exceptions.UnexpectedStateException;
@@ -67,6 +69,7 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler;
 import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
 import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
@@ -196,18 +199,21 @@ public class AssignmentManager {
   private final int assignMaxAttempts;
   private final int assignRetryImmediatelyMaxAttempts;
 
+  private final MasterRegion masterRegion;
+
   private final Object checkIfShouldMoveSystemRegionLock = new Object();
 
   private Thread assignThread;
 
-  public AssignmentManager(final MasterServices master) {
+  public AssignmentManager(MasterServices master, MasterRegion masterRegion) {
-    this(master, new RegionStateStore(master));
+    this(master, masterRegion, new RegionStateStore(master, masterRegion));
   }
 
-  AssignmentManager(final MasterServices master, final RegionStateStore stateStore) {
+  AssignmentManager(MasterServices master, MasterRegion masterRegion, RegionStateStore stateStore) {
     this.master = master;
     this.regionStateStore = stateStore;
     this.metrics = new MetricsAssignmentManager();
+    this.masterRegion = masterRegion;
 
     final Configuration conf = master.getConfiguration();
 
@@ -240,6 +246,30 @@ public class AssignmentManager {
       DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG);
   }
 
+  private void mirrorMetaLocations() throws IOException, KeeperException {
+    // For compatibility, mirror the meta region state to zookeeper
+    // And we still need to use zookeeper to publish the meta region locations to region
+    // server, so they can serve as ClientMetaService
+    ZKWatcher zk = master.getZooKeeper();
+    if (zk == null || !zk.getRecoverableZooKeeper().getState().isAlive()) {
+      // this is possible in tests, we do not provide a zk watcher or the zk watcher has been closed
+      return;
+    }
+    Collection<RegionStateNode> metaStates = regionStates.getRegionStateNodes();
+    for (RegionStateNode metaState : metaStates) {
+      MetaTableLocator.setMetaLocation(zk, metaState.getRegionLocation(),
+        metaState.getRegionInfo().getReplicaId(), metaState.getState());
+    }
+    int replicaCount = metaStates.size();
+    // remove extra mirror locations
+    for (String znode : zk.getMetaReplicaNodes()) {
+      int replicaId = zk.getZNodePaths().getMetaReplicaIdFromZNode(znode);
+      if (replicaId >= replicaCount) {
+        MetaTableLocator.deleteMetaLocation(zk, replicaId);
+      }
+    }
+  }
+
   public void start() throws IOException, KeeperException {
     if (!running.compareAndSet(false, true)) {
       return;
@@ -249,35 +279,38 @@ public class AssignmentManager {
 
     // Start the Assignment Thread
     startAssignmentThread();
-
+    // load meta region states.
-    // load meta region state
+    // here we are still in the early steps of active master startup. There is only one thread(us)
-    ZKWatcher zkw = master.getZooKeeper();
+    // can access AssignmentManager and create region node, so here we do not need to lock the
-    // it could be null in some tests
+    // region node.
-    if (zkw == null) {
+    try (ResultScanner scanner =
-      return;
+      masterRegion.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
-    }
+      for (;;) {
-    List<String> metaZNodes = zkw.getMetaReplicaNodes();
+        Result result = scanner.next();
-    LOG.debug("hbase:meta replica znodes: {}", metaZNodes);
+        if (result == null) {
-    for (String metaZNode : metaZNodes) {
+          break;
-      int replicaId = zkw.getZNodePaths().getMetaReplicaIdFromZNode(metaZNode);
+        }
-      // here we are still in the early steps of active master startup. There is only one thread(us)
+        RegionStateStore
-      // can access AssignmentManager and create region node, so here we do not need to lock the
+          .visitMetaEntry((r, regionInfo, state, regionLocation, lastHost, openSeqNum) -> {
-      // region node.
+            RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo);
-      RegionState regionState = MetaTableLocator.getMetaRegionState(zkw, replicaId);
+            regionNode.setState(state);
-      RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionState.getRegion());
+            regionNode.setLastHost(lastHost);
-      regionNode.setRegionLocation(regionState.getServerName());
+            regionNode.setRegionLocation(regionLocation);
-      regionNode.setState(regionState.getState());
+            regionNode.setOpenSeqNum(openSeqNum);
-      if (regionNode.getProcedure() != null) {
+            if (regionNode.getProcedure() != null) {
-        regionNode.getProcedure().stateLoaded(this, regionNode);
+              regionNode.getProcedure().stateLoaded(this, regionNode);
-      }
+            }
-      if (regionState.getServerName() != null) {
+            if (regionLocation != null) {
-        regionStates.addRegionToServer(regionNode);
+              regionStates.addRegionToServer(regionNode);
-      }
+            }
-      if (RegionReplicaUtil.isDefaultReplica(replicaId)) {
+            if (RegionReplicaUtil.isDefaultReplica(regionInfo.getReplicaId())) {
-        setMetaAssigned(regionState.getRegion(), regionState.getState() == State.OPEN);
+              setMetaAssigned(regionInfo, state == State.OPEN);
-      }
+            }
-      LOG.debug("Loaded hbase:meta {}", regionNode);
+            LOG.debug("Loaded hbase:meta {}", regionNode);
+          }, result);
+      }
     }
+    mirrorMetaLocations();
   }
 
   /**

hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java

@@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RegionState;
 import org.apache.hadoop.hbase.master.RegionState.State;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.procedure2.util.StringUtils;
 import org.apache.hadoop.hbase.replication.ReplicationBarrierFamilyFormat;
@@ -64,8 +65,6 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FutureUtils;
 import org.apache.hadoop.hbase.wal.WALSplitUtil;
 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
@@ -92,10 +91,14 @@ public class RegionStateStore {
 
   private final MasterServices master;
 
-  public RegionStateStore(final MasterServices master) {
+  private final MasterRegion masterRegion;
+
+  public RegionStateStore(MasterServices master, MasterRegion masterRegion) {
     this.master = master;
+    this.masterRegion = masterRegion;
   }
 
+  @FunctionalInterface
   public interface RegionStateVisitor {
     void visitRegionState(Result result, RegionInfo regionInfo, State state,
       ServerName regionLocation, ServerName lastHost, long openSeqNum);
@@ -142,8 +145,8 @@ public class RegionStateStore {
     }
   }
 
-  private void visitMetaEntry(final RegionStateVisitor visitor, final Result result)
+  public static void visitMetaEntry(final RegionStateVisitor visitor, final Result result)
-    throws IOException {
+      throws IOException {
     final RegionLocations rl = CatalogFamilyFormat.getRegionLocations(result);
     if (rl == null) return;
 
@@ -173,33 +176,14 @@ public class RegionStateStore {
   }
 
   void updateRegionLocation(RegionStateNode regionStateNode) throws IOException {
-    if (regionStateNode.getRegionInfo().isMetaRegion()) {
-      updateMetaLocation(regionStateNode.getRegionInfo(), regionStateNode.getRegionLocation(),
-        regionStateNode.getState());
-    } else {
-      long openSeqNum = regionStateNode.getState() == State.OPEN ? regionStateNode.getOpenSeqNum() :
-        HConstants.NO_SEQNUM;
-      updateUserRegionLocation(regionStateNode.getRegionInfo(), regionStateNode.getState(),
-        regionStateNode.getRegionLocation(), openSeqNum,
-        // The regionStateNode may have no procedure in a test scenario; allow for this.
-        regionStateNode.getProcedure() != null ? regionStateNode.getProcedure().getProcId() :
-          Procedure.NO_PROC_ID);
-    }
-  }
-
-  private void updateMetaLocation(RegionInfo regionInfo, ServerName serverName, State state)
-    throws IOException {
-    try {
-      MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName, regionInfo.getReplicaId(),
-        state);
-    } catch (KeeperException e) {
-      throw new IOException(e);
-    }
-  }
-
-  private void updateUserRegionLocation(RegionInfo regionInfo, State state,
-    ServerName regionLocation, long openSeqNum, long pid) throws IOException {
     long time = EnvironmentEdgeManager.currentTime();
+    long openSeqNum = regionStateNode.getState() == State.OPEN ? regionStateNode.getOpenSeqNum() :
+      HConstants.NO_SEQNUM;
+    RegionInfo regionInfo = regionStateNode.getRegionInfo();
+    State state = regionStateNode.getState();
+    ServerName regionLocation = regionStateNode.getRegionLocation();
+    TransitRegionStateProcedure rit = regionStateNode.getProcedure();
+    long pid = rit != null ? rit.getProcId() : Procedure.NO_PROC_ID;
     final int replicaId = regionInfo.getReplicaId();
     final Put put = new Put(CatalogFamilyFormat.getMetaKeyForRegion(regionInfo), time);
     MetaTableAccessor.addRegionInfo(put, regionInfo);
@@ -234,12 +218,43 @@ public class RegionStateStore {
       .build());
     LOG.info(info.toString());
     updateRegionLocation(regionInfo, state, put);
+    if (regionInfo.isMetaRegion() && regionInfo.isFirst()) {
+      // mirror the meta location to zookeeper
+      mirrorMetaLocation(regionInfo, regionLocation, state);
+    }
+  }
+
+  private void mirrorMetaLocation(RegionInfo regionInfo, ServerName serverName, State state)
+    throws IOException {
+    try {
+      MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName, regionInfo.getReplicaId(),
+        state);
+    } catch (KeeperException e) {
+      throw new IOException(e);
+    }
+  }
+
+  private void removeMirrorMetaLocation(int oldReplicaCount, int newReplicaCount)
+    throws IOException {
+    try {
+      for (int i = newReplicaCount; i < oldReplicaCount; i++) {
+        MetaTableLocator.deleteMetaLocation(master.getZooKeeper(), i);
+      }
+    } catch (KeeperException e) {
+      throw new IOException(e);
+    }
   }
 
   private void updateRegionLocation(RegionInfo regionInfo, State state, Put put)
     throws IOException {
-    try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) {
+    try {
-      table.put(put);
+      if (regionInfo.isMetaRegion()) {
+        masterRegion.update(r -> r.put(put));
+      } else {
+        try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) {
+          table.put(put);
+        }
+      }
     } catch (IOException e) {
       // TODO: Revist!!!! Means that if a server is loaded, then we will abort our host!
       // In tests we abort the Master!
@@ -537,50 +552,68 @@ public class RegionStateStore {
   }
 
   private Scan getScanForUpdateRegionReplicas(TableName tableName) {
-    return MetaTableAccessor.getScanForTableName(master.getConfiguration(), tableName)
+    Scan scan;
-      .addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+    if (TableName.isMetaTableName(tableName)) {
+      // Notice that, we do not use MetaCellComparator for master local region, so we can not use
+      // the same logic to set start key and end key for scanning meta table when locating entries
+      // in master local region. And since there is only one table in master local region(the record
+      // for meta table), so we do not need set start key and end key.
+      scan = new Scan();
+    } else {
+      scan = MetaTableAccessor.getScanForTableName(master.getConfiguration(), tableName);
+    }
+    return scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
   }
 
+  private List<Delete> deleteRegionReplicas(ResultScanner scanner, int oldReplicaCount,
+    int newReplicaCount, long now) throws IOException {
+    List<Delete> deletes = new ArrayList<>();
+    for (;;) {
+      Result result = scanner.next();
+      if (result == null) {
+        break;
+      }
+      RegionInfo primaryRegionInfo = CatalogFamilyFormat.getRegionInfo(result);
+      if (primaryRegionInfo == null || primaryRegionInfo.isSplit()) {
+        continue;
+      }
+      Delete delete = new Delete(result.getRow());
+      for (int i = newReplicaCount; i < oldReplicaCount; i++) {
+        delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(i), now);
+        delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getSeqNumColumn(i), now);
+        delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getStartCodeColumn(i),
+          now);
+        delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerNameColumn(i),
+          now);
+        delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getRegionStateColumn(i),
+          now);
+      }
+      deletes.add(delete);
+    }
+    return deletes;
+  }
+
+
   public void removeRegionReplicas(TableName tableName, int oldReplicaCount, int newReplicaCount)
     throws IOException {
+    Scan scan = getScanForUpdateRegionReplicas(tableName);
+    long now = EnvironmentEdgeManager.currentTime();
     if (TableName.isMetaTableName(tableName)) {
-      ZKWatcher zk = master.getZooKeeper();
+      List<Delete> deletes;
-      try {
+      try (ResultScanner scanner = masterRegion.getScanner(scan)) {
-        for (int i = newReplicaCount; i < oldReplicaCount; i++) {
+        deletes = deleteRegionReplicas(scanner, oldReplicaCount, newReplicaCount, now);
-          ZKUtil.deleteNode(zk, zk.getZNodePaths().getZNodeForReplica(i));
-        }
-      } catch (KeeperException e) {
-        throw new IOException(e);
       }
-    } else {
+      debugLogMutations(deletes);
-      Scan scan = getScanForUpdateRegionReplicas(tableName);
+      masterRegion.update(r -> {
-      List<Delete> deletes = new ArrayList<>();
+        for (Delete d : deletes) {
-      long now = EnvironmentEdgeManager.currentTime();
+          r.delete(d);
-      try (Table metaTable = getMetaTable(); ResultScanner scanner = metaTable.getScanner(scan)) {
-        for (;;) {
-          Result result = scanner.next();
-          if (result == null) {
-            break;
-          }
-          RegionInfo primaryRegionInfo = CatalogFamilyFormat.getRegionInfo(result);
-          if (primaryRegionInfo == null || primaryRegionInfo.isSplitParent()) {
-            continue;
-          }
-          Delete delete = new Delete(result.getRow());
-          for (int i = newReplicaCount; i < oldReplicaCount; i++) {
-            delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(i),
-              now);
-            delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getSeqNumColumn(i),
-              now);
-            delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getStartCodeColumn(i),
-              now);
-            delete.addColumns(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerNameColumn(i),
-              now);
-            delete.addColumns(HConstants.CATALOG_FAMILY,
-              CatalogFamilyFormat.getRegionStateColumn(i), now);
-          }
-          deletes.add(delete);
         }
+      });
+      // also delete the mirrored location on zk
+      removeMirrorMetaLocation(oldReplicaCount, newReplicaCount);
+    } else {
+      try (Table metaTable = getMetaTable(); ResultScanner scanner = metaTable.getScanner(scan)) {
+        List<Delete> deletes = deleteRegionReplicas(scanner, oldReplicaCount, newReplicaCount, now);
         debugLogMutations(deletes);
         metaTable.delete(deletes);
       }
@@ -634,7 +667,7 @@ public class RegionStateStore {
     }
   }
 
-  private static byte[] getStateColumn(int replicaId) {
+  public static byte[] getStateColumn(int replicaId) {
     return replicaId == 0 ? HConstants.STATE_QUALIFIER :
       Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER +
         String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId));

hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java

@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -134,7 +135,11 @@ public final class MasterRegion {
     return region.get(get);
   }
 
-  public RegionScanner getScanner(Scan scan) throws IOException {
+  public ResultScanner getScanner(Scan scan) throws IOException {
+    return new RegionScannerAsResultScanner(region.getScanner(scan));
+  }
+
+  public RegionScanner getRegionScanner(Scan scan) throws IOException {
     return region.getScanner(scan);
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java

@@ -20,11 +20,14 @@ package org.apache.hadoop.hbase.master.region;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.yetus.audience.InterfaceAudience;
 
@@ -80,6 +83,10 @@ public final class MasterRegionFactory {
   public static final byte[] PROC_FAMILY = Bytes.toBytes("proc");
 
   private static final TableDescriptor TABLE_DESC = TableDescriptorBuilder.newBuilder(TABLE_NAME)
+    .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY)
+      .setMaxVersions(HConstants.DEFAULT_HBASE_META_VERSIONS).setInMemory(true)
+      .setBlocksize(HConstants.DEFAULT_HBASE_META_BLOCK_SIZE).setBloomFilterType(BloomType.ROWCOL)
+      .setDataBlockEncoding(DataBlockEncoding.ROW_INDEX_V1).build())
     .setColumnFamily(ColumnFamilyDescriptorBuilder.of(PROC_FAMILY)).build();
 
   public static MasterRegion create(Server server) throws IOException {

hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/RegionScannerAsResultScanner.java

@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.region;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Wrap a {@link RegionScanner} as a {@link ResultScanner}.
+ */
+@InterfaceAudience.Private
+class RegionScannerAsResultScanner implements ResultScanner {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RegionScannerAsResultScanner.class);
+
+  private final RegionScanner scanner;
+
+  private boolean moreRows = true;
+
+  private final List<Cell> cells = new ArrayList<>();
+
+  RegionScannerAsResultScanner(RegionScanner scanner) {
+    this.scanner = scanner;
+  }
+
+  @Override
+  public boolean renewLease() {
+    return true;
+  }
+
+  @Override
+  public Result next() throws IOException {
+    if (!moreRows) {
+      return null;
+    }
+    for (;;) {
+      moreRows = scanner.next(cells);
+      if (cells.isEmpty()) {
+        if (!moreRows) {
+          return null;
+        } else {
+          continue;
+        }
+      }
+      Result result = Result.create(cells);
+      cells.clear();
+      return result;
+    }
+  }
+
+  @Override
+  public ScanMetrics getScanMetrics() {
+    return null;
+  }
+
+  @Override
+  public void close() {
+    try {
+      scanner.close();
+    } catch (IOException e) {
+      LOG.warn("Failed to close scanner", e);
+    }
+  }
+}

hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java

@@ -261,7 +261,7 @@ public class RegionProcedureStore extends ProcedureStoreBase {
     long maxProcId = 0;
 
     try (RegionScanner scanner =
-      region.getScanner(new Scan().addColumn(PROC_FAMILY, PROC_QUALIFIER))) {
+      region.getRegionScanner(new Scan().addColumn(PROC_FAMILY, PROC_QUALIFIER))) {
       List<Cell> cells = new ArrayList<>();
       boolean moreRows;
       do {
@@ -426,8 +426,8 @@ public class RegionProcedureStore extends ProcedureStoreBase {
   public void cleanup() {
     // actually delete the procedures if it is not the one with the max procedure id.
     List<Cell> cells = new ArrayList<Cell>();
-    try (RegionScanner scanner =
+    try (RegionScanner scanner = region
-      region.getScanner(new Scan().addColumn(PROC_FAMILY, PROC_QUALIFIER).setReversed(true))) {
+      .getRegionScanner(new Scan().addColumn(PROC_FAMILY, PROC_QUALIFIER).setReversed(true))) {
       // skip the row with max procedure id
       boolean moreRows = scanner.next(cells);
       if (cells.isEmpty()) {
@@ -442,8 +442,9 @@ public class RegionProcedureStore extends ProcedureStoreBase {
         Cell cell = cells.get(0);
         cells.clear();
         if (cell.getValueLength() == 0) {
-          region.update(r -> r
+          region.update(
-            .delete(new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())));
+            r -> r.delete(new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())
+              .addFamily(PROC_FAMILY)));
         }
       }
     } catch (IOException e) {

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java

@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
 import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
 import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
 import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
@@ -129,16 +130,18 @@ public class TestFailedMetaReplicaAssigment {
     }
 
     @Override
-    public AssignmentManager createAssignmentManager(MasterServices master) {
+    public AssignmentManager createAssignmentManager(MasterServices master,
-      return new BrokenMasterMetaAssignmentManager(master);
+      MasterRegion masterRegion) {
+      return new BrokenMasterMetaAssignmentManager(master, masterRegion);
     }
   }
 
   public static class BrokenMasterMetaAssignmentManager extends AssignmentManager {
     MasterServices master;
 
-    public BrokenMasterMetaAssignmentManager(final MasterServices master) {
+    public BrokenMasterMetaAssignmentManager(final MasterServices master,
-      super(master);
+      MasterRegion masterRegion) {
+      super(master, masterRegion);
       this.master = master;
     }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCloseAnOpeningRegion.java

@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -69,12 +70,13 @@ public class TestCloseAnOpeningRegion {
     }
 
     @Override
-    protected AssignmentManager createAssignmentManager(MasterServices master) {
+    protected AssignmentManager createAssignmentManager(MasterServices master,
-      return new AssignmentManager(master) {
+      MasterRegion masterRegion) {
+      return new AssignmentManager(master, masterRegion) {
 
         @Override
         public ReportRegionStateTransitionResponse reportRegionStateTransition(
-            ReportRegionStateTransitionRequest req) throws PleaseHoldException {
+          ReportRegionStateTransitionRequest req) throws PleaseHoldException {
           ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(req);
           TransitionCode code = req.getTransition(0).getTransitionCode();
           if (code == TransitionCode.OPENED && ARRIVE != null) {

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java

@@ -27,7 +27,6 @@ import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CountDownLatch;
 import java.util.stream.Collectors;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.CompatibilityFactory;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -40,6 +39,7 @@ import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
 import org.apache.hadoop.hbase.master.assignment.ServerState;
 import org.apache.hadoop.hbase.master.assignment.ServerStateNode;
 import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -166,15 +166,16 @@ public class TestClusterRestartFailover extends AbstractTestRestartCluster {
     }
 
     @Override
-    protected AssignmentManager createAssignmentManager(MasterServices master) {
+    protected AssignmentManager createAssignmentManager(MasterServices master,
-      return new AssignmentManagerForTest(master);
+      MasterRegion masterRegion) {
+      return new AssignmentManagerForTest(master, masterRegion);
     }
   }
 
   private static final class AssignmentManagerForTest extends AssignmentManager {
 
-    public AssignmentManagerForTest(MasterServices master) {
+    public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) {
-      super(master);
+      super(master, masterRegion);
     }
 
     @Override

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMigrateAndMirrorMetaLocations.java

@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.lengthOfPBMagic;
+import static org.apache.hadoop.hbase.zookeeper.ZKMetadata.removeMetaData;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.hbase.CatalogFamilyFormat;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.RegionLocations;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.StartTestingClusterOption;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos;
+
+/**
+ * Testcase for HBASE-26193.
+ */
+@Category({ MasterTests.class, LargeTests.class })
+public class TestMigrateAndMirrorMetaLocations {
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+    HBaseClassTestRule.forClass(TestMigrateAndMirrorMetaLocations.class);
+
+  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    UTIL.startMiniCluster(3);
+    HBaseTestingUtil.setReplicas(UTIL.getAdmin(), TableName.META_TABLE_NAME, 2);
+  }
+
+  @AfterClass
+  public static void tearDown() throws IOException {
+    UTIL.shutdownMiniCluster();
+  }
+
+  private void assertLocationEquals(Result result, int replicaCount) throws Exception {
+    RegionLocations locs = CatalogFamilyFormat.getRegionLocations(result);
+    assertEquals(replicaCount, locs.size());
+    for (int i = 0; i < replicaCount; i++) {
+      String znode = UTIL.getZooKeeperWatcher().getZNodePaths().getZNodeForReplica(i);
+      byte[] data = ZKUtil.getData(UTIL.getZooKeeperWatcher(), znode);
+      data = removeMetaData(data);
+      int prefixLen = lengthOfPBMagic();
+      ZooKeeperProtos.MetaRegionServer zkProto = ZooKeeperProtos.MetaRegionServer.parser()
+        .parseFrom(data, prefixLen, data.length - prefixLen);
+      ServerName sn = ProtobufUtil.toServerName(zkProto.getServer());
+      assertEquals(locs.getRegionLocation(i).getServerName(), sn);
+    }
+    assertEquals(replicaCount, UTIL.getZooKeeperWatcher().getMetaReplicaNodes().size());
+  }
+
+  private void checkMirrorLocation(int replicaCount) throws Exception {
+    MasterRegion masterRegion = UTIL.getMiniHBaseCluster().getMaster().getMasterRegion();
+    try (RegionScanner scanner =
+      masterRegion.getRegionScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
+      List<Cell> cells = new ArrayList<>();
+      boolean moreRows = scanner.next(cells);
+      // should only have one row as we have only one meta region, different replicas will be in the
+      // same row
+      assertFalse(moreRows);
+      assertFalse(cells.isEmpty());
+      Result result = Result.create(cells);
+      // make sure we publish the correct location to zookeeper too
+      assertLocationEquals(result, replicaCount);
+    }
+  }
+
+  private void waitUntilNoSCP() throws IOException {
+    UTIL.waitFor(30000, () -> UTIL.getMiniHBaseCluster().getMaster().getProcedures().stream()
+      .filter(p -> p instanceof ServerCrashProcedure).allMatch(Procedure::isSuccess));
+  }
+
+  @Test
+  public void test() throws Exception {
+    checkMirrorLocation(2);
+    MasterRegion masterRegion = UTIL.getMiniHBaseCluster().getMaster().getMasterRegion();
+    try (RegionScanner scanner =
+      masterRegion.getRegionScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
+      List<Cell> cells = new ArrayList<>();
+      scanner.next(cells);
+      Cell cell = cells.get(0);
+      // delete the only row
+      masterRegion.update(
+        r -> r.delete(new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())
+          .addFamily(HConstants.CATALOG_FAMILY)));
+      masterRegion.flush(true);
+    }
+    // restart the whole cluster, to see if we can migrate the data on zookeeper to master local
+    // region
+    UTIL.shutdownMiniHBaseCluster();
+    UTIL.startMiniHBaseCluster(StartTestingClusterOption.builder().numRegionServers(3).build());
+    masterRegion = UTIL.getMiniHBaseCluster().getMaster().getMasterRegion();
+    try (RegionScanner scanner =
+      masterRegion.getRegionScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
+      List<Cell> cells = new ArrayList<>();
+      boolean moreRows = scanner.next(cells);
+      assertFalse(moreRows);
+      // should have the migrated data
+      assertFalse(cells.isEmpty());
+    }
+    // wait until all meta regions have been assigned
+    UTIL.waitFor(30000,
+      () -> UTIL.getMiniHBaseCluster().getRegions(TableName.META_TABLE_NAME).size() == 2);
+    // make sure all the SCPs are finished
+    waitUntilNoSCP();
+    checkMirrorLocation(2);
+
+    // increase replica count to 3
+    HBaseTestingUtil.setReplicas(UTIL.getAdmin(), TableName.META_TABLE_NAME, 3);
+    checkMirrorLocation(3);
+
+    byte[] replica2Data = ZKUtil.getData(UTIL.getZooKeeperWatcher(),
+      UTIL.getZooKeeperWatcher().getZNodePaths().getZNodeForReplica(2));
+
+    // decrease replica count to 1
+    HBaseTestingUtil.setReplicas(UTIL.getAdmin(), TableName.META_TABLE_NAME, 1);
+    checkMirrorLocation(1);
+
+    // restart the whole cluster, put an extra replica znode on zookeeper, to see if we will remove
+    // it
+    UTIL.shutdownMiniHBaseCluster();
+    ZKUtil.createAndFailSilent(UTIL.getZooKeeperWatcher(),
+      UTIL.getZooKeeperWatcher().getZNodePaths().getZNodeForReplica(2), replica2Data);
+    UTIL.startMiniHBaseCluster(StartTestingClusterOption.builder().numRegionServers(3).build());
+    // should have removed the extra replica znode as it is part of the start up process, when
+    // initializing AM
+    assertEquals(1, UTIL.getZooKeeperWatcher().getMetaReplicaNodes().size());
+    // make sure all the SCPs are finished
+    waitUntilNoSCP();
+    checkMirrorLocation(1);
+  }
+}

hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java

@@ -50,6 +50,8 @@ import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
+import org.apache.hadoop.hbase.master.region.MasterRegionFactory;
 import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
@@ -85,6 +87,7 @@ public class MockMasterServices extends MockNoopMasterServices {
   private final SplitWALManager splitWALManager;
   private final AssignmentManager assignmentManager;
   private final TableStateManager tableStateManager;
+  private final MasterRegion masterRegion;
 
   private MasterProcedureEnv procedureEnv;
   private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
@@ -107,9 +110,10 @@ public class MockMasterServices extends MockNoopMasterServices {
     this.splitWALManager =
       conf.getBoolean(HBASE_SPLIT_WAL_COORDINATED_BY_ZK, DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK)?
         null: new SplitWALManager(this);
-
+    this.masterRegion = MasterRegionFactory.create(this);
     // Mock an AM.
-    this.assignmentManager = new AssignmentManager(this, new MockRegionStateStore(this));
+    this.assignmentManager =
+      new AssignmentManager(this, masterRegion, new MockRegionStateStore(this, masterRegion));
     this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
     this.serverManager = new ServerManager(this);
     this.tableStateManager = Mockito.mock(TableStateManager.class);
@@ -290,8 +294,8 @@ public class MockMasterServices extends MockNoopMasterServices {
   }
 
   private static class MockRegionStateStore extends RegionStateStore {
-    public MockRegionStateStore(final MasterServices master) {
+    public MockRegionStateStore(MasterServices master, MasterRegion masterRegion) {
-      super(master);
+      super(master, masterRegion);
     }
 
     @Override

hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java

@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -55,8 +56,8 @@ public class TestOpenRegionProcedureBackoff {
 
   private static final class AssignmentManagerForTest extends AssignmentManager {
 
-    public AssignmentManagerForTest(MasterServices master) {
+    public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) {
-      super(master);
+      super(master, masterRegion);
     }
 
     @Override
@@ -75,8 +76,9 @@ public class TestOpenRegionProcedureBackoff {
     }
 
     @Override
-    protected AssignmentManager createAssignmentManager(MasterServices master) {
+    protected AssignmentManager createAssignmentManager(MasterServices master,
-      return new AssignmentManagerForTest(master);
+      MasterRegion masterRegion) {
+      return new AssignmentManagerForTest(master, masterRegion);
     }
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureHang.java

@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -74,8 +75,8 @@ public class TestOpenRegionProcedureHang {
 
   private static final class AssignmentManagerForTest extends AssignmentManager {
 
-    public AssignmentManagerForTest(MasterServices master) {
+    public AssignmentManagerForTest(MasterServices master,MasterRegion masterRegion) {
-      super(master);
+      super(master, masterRegion);
     }
 
     @Override
@@ -110,8 +111,9 @@ public class TestOpenRegionProcedureHang {
     }
 
     @Override
-    protected AssignmentManager createAssignmentManager(MasterServices master) {
+    protected AssignmentManager createAssignmentManager(MasterServices master,
-      return new AssignmentManagerForTest(master);
+      MasterRegion masterRegion) {
+      return new AssignmentManagerForTest(master, masterRegion);
     }
 
     @Override

hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java

@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
 import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -69,8 +70,8 @@ public class TestRaceBetweenSCPAndDTP {
 
   private static final class AssignmentManagerForTest extends AssignmentManager {
 
-    public AssignmentManagerForTest(MasterServices master) {
+    public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) {
-      super(master);
+      super(master,masterRegion);
     }
 
     @Override
@@ -95,8 +96,9 @@ public class TestRaceBetweenSCPAndDTP {
     }
 
     @Override
-    protected AssignmentManager createAssignmentManager(MasterServices master) {
+    protected AssignmentManager createAssignmentManager(MasterServices master,
-      return new AssignmentManagerForTest(master);
+      MasterRegion masterRegion) {
+      return new AssignmentManagerForTest(master, masterRegion);
     }
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java

@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -69,8 +70,8 @@ public class TestRaceBetweenSCPAndTRSP {
 
   private static final class AssignmentManagerForTest extends AssignmentManager {
 
-    public AssignmentManagerForTest(MasterServices master) {
+    public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) {
-      super(master);
+      super(master, masterRegion);
     }
 
     @Override
@@ -108,8 +109,9 @@ public class TestRaceBetweenSCPAndTRSP {
     }
 
     @Override
-    protected AssignmentManager createAssignmentManager(MasterServices master) {
+    protected AssignmentManager createAssignmentManager(MasterServices master,
-      return new AssignmentManagerForTest(master);
+      MasterRegion masterRegion) {
+      return new AssignmentManagerForTest(master, masterRegion);
     }
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionAssignedToMultipleRegionServers.java

@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -80,8 +81,8 @@ public class TestRegionAssignedToMultipleRegionServers {
 
   private static final class AssignmentManagerForTest extends AssignmentManager {
 
-    public AssignmentManagerForTest(MasterServices master) {
+    public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) {
-      super(master);
+      super(master, masterRegion);
     }
 
     @Override
@@ -114,8 +115,9 @@ public class TestRegionAssignedToMultipleRegionServers {
     }
 
     @Override
-    protected AssignmentManager createAssignmentManager(MasterServices master) {
+    protected AssignmentManager createAssignmentManager(MasterServices master,
-      return new AssignmentManagerForTest(master);
+      MasterRegion masterRegion) {
+      return new AssignmentManagerForTest(master, masterRegion);
     }
 
     @Override

hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportOnlineRegionsRace.java

@@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.master.RegionState;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -68,8 +69,8 @@ public class TestReportOnlineRegionsRace {
 
   private static final class AssignmentManagerForTest extends AssignmentManager {
 
-    public AssignmentManagerForTest(MasterServices master) {
+    public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) {
-      super(master);
+      super(master, masterRegion);
     }
 
     @Override
@@ -110,8 +111,9 @@ public class TestReportOnlineRegionsRace {
     }
 
     @Override
-    protected AssignmentManager createAssignmentManager(MasterServices master) {
+    protected AssignmentManager createAssignmentManager(MasterServices master,
-      return new AssignmentManagerForTest(master);
+      MasterRegion masterRegion) {
+      return new AssignmentManagerForTest(master, masterRegion);
     }
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionFromDeadServer.java

@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -82,8 +83,8 @@ public class TestReportRegionStateTransitionFromDeadServer {
 
   private static final class AssignmentManagerForTest extends AssignmentManager {
 
-    public AssignmentManagerForTest(MasterServices master) {
+    public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) {
-      super(master);
+      super(master, masterRegion);
     }
 
     @Override
@@ -121,8 +122,9 @@ public class TestReportRegionStateTransitionFromDeadServer {
     }
 
     @Override
-    protected AssignmentManager createAssignmentManager(MasterServices master) {
+    protected AssignmentManager createAssignmentManager(MasterServices master,
-      return new AssignmentManagerForTest(master);
+      MasterRegion masterRegion) {
+      return new AssignmentManagerForTest(master, masterRegion);
     }
 
     @Override

hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestReportRegionStateTransitionRetry.java

@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -60,8 +61,8 @@ public class TestReportRegionStateTransitionRetry {
 
   private static final class AssignmentManagerForTest extends AssignmentManager {
 
-    public AssignmentManagerForTest(MasterServices master) {
+    public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) {
-      super(master);
+      super(master, masterRegion);
     }
 
     @Override
@@ -88,8 +89,9 @@ public class TestReportRegionStateTransitionRetry {
     }
 
     @Override
-    protected AssignmentManager createAssignmentManager(MasterServices master) {
+    protected AssignmentManager createAssignmentManager(MasterServices master,
-      return new AssignmentManagerForTest(master);
+      MasterRegion masterRegion) {
+      return new AssignmentManagerForTest(master, masterRegion);
     }
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSCPGetRegionsRace.java

@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -89,8 +90,8 @@ public class TestSCPGetRegionsRace {
 
   private static final class AssignmentManagerForTest extends AssignmentManager {
 
-    public AssignmentManagerForTest(MasterServices master) {
+    public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) {
-      super(master);
+      super(master, masterRegion);
     }
 
     @Override
@@ -134,8 +135,9 @@ public class TestSCPGetRegionsRace {
     }
 
     @Override
-    protected AssignmentManager createAssignmentManager(MasterServices master) {
+    protected AssignmentManager createAssignmentManager(MasterServices master,
-      return new AssignmentManagerForTest(master);
+      MasterRegion masterRegion) {
+      return new AssignmentManagerForTest(master, masterRegion);
     }
 
     @Override

hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestWakeUpUnexpectedProcedure.java

@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.master.RegionState;
 import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.region.MasterRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -135,8 +136,8 @@ public class TestWakeUpUnexpectedProcedure {
 
   private static final class AMForTest extends AssignmentManager {
 
-    public AMForTest(MasterServices master) {
+    public AMForTest(MasterServices master, MasterRegion masterRegion) {
-      super(master);
+      super(master, masterRegion);
     }
 
     @Override
@@ -202,8 +203,9 @@ public class TestWakeUpUnexpectedProcedure {
     }
 
     @Override
-    protected AssignmentManager createAssignmentManager(MasterServices master) {
+    protected AssignmentManager createAssignmentManager(MasterServices master,
-      return new AMForTest(master);
+      MasterRegion masterRegion) {
+      return new AMForTest(master, masterRegion);
     }
 
     @Override

hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionOnTwoFileSystems.java

@@ -222,7 +222,7 @@ public class TestMasterRegionOnTwoFileSystems {
       region.close(true);
       region = createMasterRegion(
         ServerName.valueOf("localhost", 12345, EnvironmentEdgeManager.currentTime() + round + 1));
-      try (RegionScanner scanner = region.getScanner(new Scan())) {
+      try (RegionScanner scanner = region.getRegionScanner(new Scan())) {
         List<Cell> cells = new ArrayList<>();
         boolean moreValues = true;
         for (int i = 0; i < (round + 1) * countPerRound; i++) {

hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStoreMigration.java

@@ -80,7 +80,7 @@ public class TestRegionProcedureStoreMigration {
     Configuration conf = htu.getConfiguration();
     conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, false);
     // Runs on local filesystem. Test does not need sync. Turn off checks.
-    htu.getConfiguration().setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false);
+    conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false);
     Path testDir = htu.getDataTestDir();
     CommonFSUtils.setRootDir(conf, testDir);
     walStore = new WALProcedureStore(conf, new LeaseRecovery() {