HBASE-13084 Add labels to VisibilityLabelsCache asynchronously causes TestShell flakey

Signed-off-by: stack <stack@apache.org>

hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java

@@ -99,6 +99,7 @@ import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
 import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache;
 import org.apache.hadoop.hbase.tool.Canary;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
@@ -3851,6 +3852,37 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
     waitFor(timeout, predicateNoRegionsInTransition());
   }
 
+  /**
+   * Wait until labels is ready in VisibilityLabelsCache.
+   * @param timeoutMillis
+   * @param labels
+   */
+  public void waitLabelAvailable(long timeoutMillis, final String... labels) {
+    final VisibilityLabelsCache labelsCache = VisibilityLabelsCache.get();
+    waitFor(timeoutMillis, new Waiter.ExplainingPredicate<RuntimeException>() {
+
+      @Override
+      public boolean evaluate() {
+        for (String label : labels) {
+          if (labelsCache.getLabelOrdinal(label) == 0) {
+            return false;
+          }
+        }
+        return true;
+      }
+
+      @Override
+      public String explainFailure() {
+        for (String label : labels) {
+          if (labelsCache.getLabelOrdinal(label) == 0) {
+            return label + " is not available yet";
+          }
+        }
+        return "";
+      }
+    });
+  }
+
   /**
    * Create a set of column descriptors with the combination of compression,
    * encoding, bloom codecs available.

hbase-shell/src/test/ruby/hbase/visibility_labels_admin_test.rb

@@ -54,6 +54,7 @@ module Hbase
       label = 'TEST_AUTHS'
       user = org.apache.hadoop.hbase.security.User.getCurrent().getName();
       visibility_admin.add_labels(label)
+      $TEST_CLUSTER.waitLabelAvailable(10000, label)
       count = visibility_admin.get_auths(user).length
 
       # verifying the set functionality
@@ -74,6 +75,7 @@ module Hbase
       label = 'TEST_VISIBILITY'
       user = org.apache.hadoop.hbase.security.User.getCurrent().getName();
       visibility_admin.add_labels(label)
+      $TEST_CLUSTER.waitLabelAvailable(10000, label)
       visibility_admin.set_auths(user, label)
 
       # verifying put functionality