HBASE-24135 TableStateNotFoundException happends when table creation if rsgroup is enable (#1550)

Signed-off-by: Lijin Bin <binlijin@apache.org>
Signed-off-by: Duo Zhang <zhangduo@apache.org>

hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java

@@ -109,48 +109,6 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
   /** Provider for mapping principal names to Users */
   private UserProvider userProvider;
 
-  /** Get rsgroup table mapping script */
-  private RSGroupMappingScript script;
-
-  // Package visibility for testing
-  static class RSGroupMappingScript {
-
-    static final String RS_GROUP_MAPPING_SCRIPT = "hbase.rsgroup.table.mapping.script";
-    static final String RS_GROUP_MAPPING_SCRIPT_TIMEOUT =
-      "hbase.rsgroup.table.mapping.script.timeout";
-
-    private ShellCommandExecutor rsgroupMappingScript;
-
-    RSGroupMappingScript(Configuration conf) {
-      String script = conf.get(RS_GROUP_MAPPING_SCRIPT);
-      if (script == null || script.isEmpty()) {
-        return;
-      }
-
-      rsgroupMappingScript = new ShellCommandExecutor(
-        new String[] { script, "", "" }, null, null,
-        conf.getLong(RS_GROUP_MAPPING_SCRIPT_TIMEOUT, 5000) // 5 seconds
-      );
-    }
-
-    String getRSGroup(String namespace, String tablename) {
-      if (rsgroupMappingScript == null) {
-        return RSGroupInfo.DEFAULT_GROUP;
-      }
-      String[] exec = rsgroupMappingScript.getExecString();
-      exec[1] = namespace;
-      exec[2] = tablename;
-      try {
-        rsgroupMappingScript.execute();
-      } catch (IOException e) {
-        LOG.error(e.getMessage() + " placing back to default rsgroup");
-        return RSGroupInfo.DEFAULT_GROUP;
-      }
-      return rsgroupMappingScript.getOutput().trim();
-    }
-
-  }
-
   @Override
   public void start(CoprocessorEnvironment env) throws IOException {
     if (!(env instanceof HasMasterServices)) {
@@ -169,7 +127,6 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
 
     // set the user-provider.
     this.userProvider = UserProvider.instantiate(env.getConfiguration());
-    this.script = new RSGroupMappingScript(env.getConfiguration());
   }
 
   @Override
@@ -500,30 +457,14 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
   }
 
   void assignTableToGroup(TableDescriptor desc) throws IOException {
-    String groupName =
-        master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString())
-                .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
-    if (groupName == null) {
-      groupName = RSGroupInfo.DEFAULT_GROUP;
-    }
-
-    if (groupName.equals(RSGroupInfo.DEFAULT_GROUP)) {
-      TableName tableName = desc.getTableName();
-      groupName = script.getRSGroup(
-        tableName.getNamespaceAsString(),
-        tableName.getQualifierAsString()
-      );
-      LOG.info("rsgroup for " + tableName + " is " + groupName);
-    }
-
-    RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName);
+    RSGroupInfo rsGroupInfo = groupInfoManager.determineRSGroupInfoForTable(desc.getTableName());
     if (rsGroupInfo == null) {
-      throw new ConstraintException("Default RSGroup (" + groupName + ") for this table's "
-          + "namespace does not exist.");
+      throw new ConstraintException("Default RSGroup for this table " + desc.getTableName()
+        + " does not exist.");
     }
     if (!rsGroupInfo.containsTable(desc.getTableName())) {
-      LOG.debug("Pre-moving table " + desc.getTableName() + " to RSGroup " + groupName);
-      groupAdminServer.moveTables(Sets.newHashSet(desc.getTableName()), groupName);
+      LOG.debug("Pre-moving table " + desc.getTableName() + " to RSGroup " + rsGroupInfo.getName());
+      groupAdminServer.moveTables(Sets.newHashSet(desc.getTableName()), rsGroupInfo.getName());
     }
   }
 
@@ -536,17 +477,22 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
       final ObserverContext<MasterCoprocessorEnvironment> ctx,
       final TableDescriptor desc,
       final RegionInfo[] regions) throws IOException {
-    if (!desc.getTableName().isSystemTable() && !rsgroupHasServersOnline(desc)) {
-      throw new HBaseIOException("No online servers in the rsgroup, which table " +
-          desc.getTableName().getNameAsString() + " belongs to");
+    if (desc.getTableName().isSystemTable()) {
+      return;
+    }
+    RSGroupInfo rsGroupInfo = groupInfoManager.determineRSGroupInfoForTable(desc.getTableName());
+    if (rsGroupInfo == null) {
+      throw new ConstraintException("Default RSGroup for this table " + desc.getTableName()
+        + " does not exist.");
+    }
+    if (!RSGroupUtil.rsGroupHasOnlineServer(master, rsGroupInfo)) {
+      throw new HBaseIOException("No online servers in the rsgroup " + rsGroupInfo.getName()
+        + " which table " + desc.getTableName().getNameAsString() + " belongs to");
+    }
+    synchronized (groupInfoManager) {
+      groupInfoManager.moveTables(
+        Collections.singleton(desc.getTableName()), rsGroupInfo.getName());
     }
-  }
-
-  // Assign table to default RSGroup.
-  @Override
-  public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
-      TableDescriptor desc, RegionInfo[] regions) throws IOException {
-    assignTableToGroup(desc);
   }
 
   // Remove table from its RSGroup.

hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java

@@ -132,4 +132,11 @@ public interface RSGroupInfoManager {
    * @param newName new rsgroup name
    */
   void renameRSGroup(String oldName, String newName) throws IOException;
+
+  /**
+   * Determine {@code RSGroupInfo} for the given table.
+   * @param tableName table name
+   * @return {@link RSGroupInfo} which table should belong to
+   */
+  RSGroupInfo determineRSGroupInfoForTable(TableName tableName) throws IOException;
 }

hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java

@@ -32,8 +32,10 @@ import java.util.OptionalLong;
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
@@ -53,6 +55,7 @@ import org.apache.hadoop.hbase.constraint.ConstraintException;
 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
+import org.apache.hadoop.hbase.master.ClusterSchema;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.ServerListener;
 import org.apache.hadoop.hbase.master.TableStateManager;
@@ -70,11 +73,13 @@ import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
 import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.apache.hadoop.util.Shell;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
 import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
 import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
@@ -133,14 +138,56 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
   private final ServerEventsListenerThread serverEventsListenerThread =
     new ServerEventsListenerThread();
 
+  /** Get rsgroup table mapping script */
+  @VisibleForTesting
+  RSGroupMappingScript script;
+
+  // Package visibility for testing
+  static class RSGroupMappingScript {
+
+    static final String RS_GROUP_MAPPING_SCRIPT = "hbase.rsgroup.table.mapping.script";
+    static final String RS_GROUP_MAPPING_SCRIPT_TIMEOUT =
+      "hbase.rsgroup.table.mapping.script.timeout";
+
+    private Shell.ShellCommandExecutor rsgroupMappingScript;
+
+    RSGroupMappingScript(Configuration conf) {
+      String script = conf.get(RS_GROUP_MAPPING_SCRIPT);
+      if (script == null || script.isEmpty()) {
+        return;
+      }
+
+      rsgroupMappingScript = new Shell.ShellCommandExecutor(
+        new String[] { script, "", "" }, null, null,
+        conf.getLong(RS_GROUP_MAPPING_SCRIPT_TIMEOUT, 5000) // 5 seconds
+      );
+    }
+
+    String getRSGroup(String namespace, String tablename) {
+      if (rsgroupMappingScript == null) {
+        return null;
+      }
+      String[] exec = rsgroupMappingScript.getExecString();
+      exec[1] = namespace;
+      exec[2] = tablename;
+      try {
+        rsgroupMappingScript.execute();
+      } catch (IOException e) {
+        LOG.error("Failed to get RSGroup from script for table {}:{}", namespace, tablename, e);
+        return null;
+      }
+      return rsgroupMappingScript.getOutput().trim();
+    }
+  }
+
   private RSGroupInfoManagerImpl(MasterServices masterServices) throws IOException {
     this.masterServices = masterServices;
     this.watcher = masterServices.getZooKeeper();
     this.conn = masterServices.getConnection();
     this.rsGroupStartupWorker = new RSGroupStartupWorker();
+    script = new RSGroupMappingScript(masterServices.getConfiguration());
   }
 
-
   private synchronized void init() throws IOException {
     refresh();
     serverEventsListenerThread.start();
@@ -366,6 +413,45 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
     flushConfig(newGroupMap);
   }
 
+  /**
+   * Will try to get the rsgroup from {@code tableMap} first
+   * then try to get the rsgroup from {@code script}
+   * try to get the rsgroup from the {@link NamespaceDescriptor} lastly.
+   * If still not present, return default group.
+   */
+  @Override
+  public RSGroupInfo determineRSGroupInfoForTable(TableName tableName)
+    throws IOException {
+    RSGroupInfo groupFromOldRSGroupInfo = getRSGroup(getRSGroupOfTable(tableName));
+    if (groupFromOldRSGroupInfo != null) {
+      return groupFromOldRSGroupInfo;
+    }
+    // RSGroup information determined by administrator.
+    RSGroupInfo groupDeterminedByAdmin = getRSGroup(
+      script.getRSGroup(tableName.getNamespaceAsString(), tableName.getQualifierAsString()));
+    if (groupDeterminedByAdmin != null) {
+      return groupDeterminedByAdmin;
+    }
+    // Finally, we will try to fall back to namespace as rsgroup if exists
+    ClusterSchema clusterSchema = masterServices.getClusterSchema();
+    if (clusterSchema == null) {
+      if (TableName.isMetaTableName(tableName)) {
+        LOG.info("Can not get the namespace rs group config for meta table, since the" +
+          " meta table is not online yet, will use default group to assign meta first");
+      } else {
+        LOG.warn("ClusterSchema is null, can only use default rsgroup, should not happen?");
+      }
+    } else {
+      NamespaceDescriptor nd = clusterSchema.getNamespace(tableName.getNamespaceAsString());
+      RSGroupInfo groupNameOfNs =
+        getRSGroup(nd.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP));
+      if (groupNameOfNs != null) {
+        return groupNameOfNs;
+      }
+    }
+    return getRSGroup(RSGroupInfo.DEFAULT_GROUP);
+  }
+
   List<RSGroupInfo> retrieveGroupListFromGroupTable() throws IOException {
     List<RSGroupInfo> rsGroupInfoList = Lists.newArrayList();
     try (Table table = conn.getTable(RSGROUP_TABLE_NAME);

hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupUtil.java

@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.rsgroup;
+
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Helper class for RSGroup implementation
+ */
+@InterfaceAudience.Private
+public final class RSGroupUtil {
+
+  public static boolean rsGroupHasOnlineServer(MasterServices master, RSGroupInfo rsGroupInfo) {
+    for (ServerName onlineServer : master.getServerManager().createDestinationServersList()) {
+      if (rsGroupInfo.getServers().contains(onlineServer.getAddress())) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+}

hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestDetermineRSGroupInfoForTable.java

@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rsgroup;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManagerImpl.RSGroupMappingScript;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test {@link RSGroupInfoManager#determineRSGroupInfoForTable(TableName)}
+ */
+@Category({ MediumTests.class })
+public class TestDetermineRSGroupInfoForTable {
+
+  private static final Logger LOG = LoggerFactory.getLogger(TestDetermineRSGroupInfoForTable.class);
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+    HBaseClassTestRule.forClass(TestDetermineRSGroupInfoForTable.class);
+
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  private static HMaster master;
+
+  private static Admin admin;
+
+  private static RSGroupInfoManager rsGroupInfoManager;
+
+  private static RSGroupAdminClient rsGroupAdminClient;
+
+  private static final String GROUP_NAME = "rsg";
+
+  private static final String NAMESPACE_NAME = "ns";
+  private static final String OTHER_NAMESPACE_NAME = "other";
+
+  private static final TableName TABLE_NAME = TableName.valueOf(NAMESPACE_NAME, "tb");
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    UTIL.getConfiguration().set(
+      HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
+      RSGroupBasedLoadBalancer.class.getName());
+    UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
+      RSGroupAdminEndpoint.class.getName());
+    UTIL.startMiniCluster(5);
+    master = UTIL.getMiniHBaseCluster().getMaster();
+    admin = UTIL.getAdmin();
+    rsGroupAdminClient = new RSGroupAdminClient(UTIL.getConnection());
+
+    HRegionServer rs = UTIL.getHBaseCluster().getRegionServer(0);
+    rsGroupAdminClient.addRSGroup(GROUP_NAME);
+    rsGroupAdminClient.moveServers(
+      Collections.singleton(rs.getServerName().getAddress()), GROUP_NAME);
+    admin.createNamespace(NamespaceDescriptor.create(NAMESPACE_NAME)
+      .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, GROUP_NAME)
+      .build());
+    admin.createNamespace(NamespaceDescriptor.create(OTHER_NAMESPACE_NAME).build());
+  }
+
+  @AfterClass
+  public static void tearDown() throws IOException {
+    admin.deleteNamespace(NAMESPACE_NAME);
+
+    UTIL.shutdownMiniCluster();
+  }
+
+  @Before
+  public void setUpBeforeMethod() throws IOException {
+    rsGroupInfoManager = RSGroupInfoManagerImpl.getInstance(master);
+    rsGroupInfoManager.start();
+  }
+
+  @After
+  public void tearDownAfterMethod() throws IOException {
+    rsGroupInfoManager = RSGroupInfoManagerImpl.getInstance(master);
+  }
+
+  @Test
+  public void testByDefault() throws IOException {
+    RSGroupInfo group =
+      rsGroupInfoManager.determineRSGroupInfoForTable(TableName.valueOf("tb"));
+    assertEquals(group.getName(), RSGroupInfo.DEFAULT_GROUP);
+  }
+
+  @Test
+  public void testDetermineByNamespaceConfig() throws IOException {
+    RSGroupInfo group = rsGroupInfoManager.determineRSGroupInfoForTable(TABLE_NAME);
+    assertEquals(group.getName(), GROUP_NAME);
+
+    group = rsGroupInfoManager.determineRSGroupInfoForTable(
+      TableName.valueOf(OTHER_NAMESPACE_NAME, "tb"));
+    assertEquals(group.getName(), RSGroupInfo.DEFAULT_GROUP);
+  }
+
+  /**
+   * determine by script
+   */
+  @Test
+  public void testDetermineByScript() throws IOException {
+    RSGroupMappingScript script = mock(RSGroupMappingScript.class);
+    when(script.getRSGroup(anyString(), anyString())).thenReturn(GROUP_NAME);
+    ((RSGroupInfoManagerImpl) rsGroupInfoManager).script = script;
+
+    RSGroupInfo group = rsGroupInfoManager.determineRSGroupInfoForTable(TABLE_NAME);
+    assertEquals(group.getName(), GROUP_NAME);
+  }
+
+}

hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMappingScript.java

@@ -26,7 +26,7 @@ import java.io.PrintWriter;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint.RSGroupMappingScript;
+import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManagerImpl.RSGroupMappingScript;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.junit.After;
 import org.junit.Assert;

hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupUtil.java

@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rsgroup;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestRSGroupUtil {
+
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  private static Admin admin;
+
+  private static HMaster master;
+
+  private static RSGroupAdminClient rsGroupAdminClient;
+
+  private static final String GROUP_NAME = "rsg";
+
+  private static final String NAMESPACE_NAME = "ns";
+
+  private static RSGroupInfoManager rsGroupInfoManager;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    UTIL.getConfiguration().set(
+      HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
+      RSGroupBasedLoadBalancer.class.getName());
+    UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
+      RSGroupAdminEndpoint.class.getName());
+    UTIL.startMiniCluster(5);
+    master = UTIL.getMiniHBaseCluster().getMaster();
+    admin = UTIL.getAdmin();
+    rsGroupAdminClient = new RSGroupAdminClient(UTIL.getConnection());
+
+    HRegionServer rs = UTIL.getHBaseCluster().getRegionServer(0);
+    rsGroupAdminClient.addRSGroup(GROUP_NAME);
+    rsGroupAdminClient.moveServers(Collections.singleton(rs.getServerName().getAddress()), GROUP_NAME);
+    admin.createNamespace(NamespaceDescriptor.create(NAMESPACE_NAME)
+      .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, GROUP_NAME)
+      .build());
+
+    rsGroupInfoManager = RSGroupInfoManagerImpl.getInstance(master);
+    rsGroupInfoManager.start();
+  }
+
+  @AfterClass
+  public static void tearDown() throws IOException {
+    admin.deleteNamespace(NAMESPACE_NAME);
+
+    UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void rsGroupHasOnlineServer() throws IOException {
+    rsGroupInfoManager.refresh();
+    RSGroupInfo defaultGroup = rsGroupInfoManager.getRSGroup(RSGroupInfo.DEFAULT_GROUP);
+    Assert.assertTrue(RSGroupUtil.rsGroupHasOnlineServer(master, defaultGroup));
+
+    RSGroupInfo rsGroup = rsGroupInfoManager.getRSGroup(GROUP_NAME);
+    Assert.assertTrue(RSGroupUtil.rsGroupHasOnlineServer(master, rsGroup));
+
+    rsGroupAdminClient.addRSGroup("empty");
+    rsGroupInfoManager.refresh();
+    RSGroupInfo emptyGroup = rsGroupInfoManager.getRSGroup("empty");
+    Assert.assertFalse(RSGroupUtil.rsGroupHasOnlineServer(master, emptyGroup));
+  }
+}