HBASE-24720: Meta replicas not cleaned when disabled (#2057)

- make sure to always clean up excess meta replicas not just when their
number get decreased
- make sure NotServingRegionException is handled properly even when
wrapped
- add test

Signed-off-by: Peter Somogyi <psomogyi@apache.org>

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java

@@ -56,10 +56,6 @@ class MasterMetaBootstrap {
       throws IOException, InterruptedException, KeeperException {
     int numReplicas = master.getConfiguration().getInt(HConstants.META_REPLICAS_NUM,
            HConstants.DEFAULT_META_REPLICA_NUM);
-    if (numReplicas <= 1) {
-      // No replicaas to assign. Return.
-      return;
-    }
     final AssignmentManager assignmentManager = master.getAssignmentManager();
     if (!assignmentManager.isMetaLoaded()) {
       throw new IllegalStateException("hbase:meta must be initialized first before we can " +

hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java

@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.client.ClusterConnection;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.RetriesExhaustedException;
 import org.apache.hadoop.hbase.ipc.HBaseRpcController;
+import org.apache.hadoop.hbase.ipc.RemoteWithExtrasException;
 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.procedure2.Procedure;
@@ -708,8 +709,13 @@ public class ServerManager {
           ProtobufUtil.getRegionInfo(controller, rs, region.getRegionName());
         if (rsRegion == null) return;
       } catch (IOException ioe) {
-        if (ioe instanceof NotServingRegionException) // no need to retry again
+        if (ioe instanceof NotServingRegionException ||
+          (ioe instanceof RemoteWithExtrasException &&
+            ((RemoteWithExtrasException)ioe).unwrapRemoteException()
+              instanceof NotServingRegionException)) {
+          // no need to retry again
           return;
+        }
         LOG.warn("Exception when retrieving regioninfo from: "
           + region.getRegionNameAsString(), ioe);
       }

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasBasic.java

@@ -20,15 +20,17 @@ package org.apache.hadoop.hbase.client;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
-
+import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JVMClusterUtil;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
 import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
@@ -80,6 +82,27 @@ public class TestMetaWithReplicasBasic extends MetaWithReplicasTestBase {
     }
   }
 
+  @Test
+  public void testReplicaCleanup() throws Exception {
+    ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
+    List<String> metaReplicaZnodes = zkw.getMetaReplicaNodes();
+    assertEquals(3, metaReplicaZnodes.size());
+
+    final HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
+    master.stop("Restarting");
+    TEST_UTIL.waitFor(30000, () -> master.isStopped());
+    TEST_UTIL.getMiniHBaseCluster().getConfiguration().setInt(HConstants.META_REPLICAS_NUM, 1);
+
+    JVMClusterUtil.MasterThread newMasterThread = TEST_UTIL.getMiniHBaseCluster().startMaster();
+    final HMaster newMaster = newMasterThread.getMaster();
+
+    //wait until new master finished meta replica assignment logic
+    TEST_UTIL.waitFor(30000, () -> newMaster.getMasterQuotaManager() != null);
+    zkw = TEST_UTIL.getZooKeeperWatcher();
+    metaReplicaZnodes = zkw.getMetaReplicaNodes();
+    assertEquals(1, metaReplicaZnodes.size());
+
+  }
   @Test
   public void testAccessingUnknownTables() throws Exception {
     Configuration conf = new Configuration(TEST_UTIL.getConfiguration());