HBASE-11878 TestVisibilityLabelsWithDistributedLogReplay#testAddVisibilityLabelsOnRSRestart sometimes fails due to VisibilityController not yet initialized

hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityControllerNotReadyException.java

@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.visibility;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/*
+ * This exception indicates that VisibilityController hasn't finished initialization.
+ */
+@InterfaceAudience.Public
+public class VisibilityControllerNotReadyException extends IOException {
+
+  private static final long serialVersionUID = 1725986525207989173L;
+
+  public VisibilityControllerNotReadyException(String msg) {
+    super(msg);
+  }
+}

hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java

@@ -249,6 +249,7 @@ public class VisibilityController extends BaseMasterAndRegionObserver implements
   public void postLogReplay(ObserverContext<RegionCoprocessorEnvironment> e) {
     if (this.labelsRegion) {
       initVisibilityLabelService(e.getEnvironment());
+      LOG.debug("post labels region log replay");
     }
   }
 
@@ -411,7 +412,9 @@ public class VisibilityController extends BaseMasterAndRegionObserver implements
   @Override
   public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
       RegionScanner s) throws IOException {
-    if (!initialized) throw new IOException("VisibilityController not yet initialized!!");
+    if (!initialized) {
+      throw new VisibilityControllerNotReadyException("VisibilityController not yet initialized!");
+    }
     HRegion region = e.getEnvironment().getRegion();
     Authorizations authorizations = null;
     try {
@@ -507,7 +510,9 @@ public class VisibilityController extends BaseMasterAndRegionObserver implements
   @Override
   public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
       throws IOException {
-    if (!initialized) throw new IOException("VisibilityController not yet initialized!!");
+    if (!initialized) {
+      throw new VisibilityControllerNotReadyException("VisibilityController not yet initialized!");
+    }
     HRegion region = e.getEnvironment().getRegion();
     Authorizations authorizations = null;
     try {
@@ -635,8 +640,9 @@ public class VisibilityController extends BaseMasterAndRegionObserver implements
     VisibilityLabelsResponse.Builder response = VisibilityLabelsResponse.newBuilder();
     List<VisibilityLabel> visLabels = request.getVisLabelList();
     if (!initialized) {
-      setExceptionResults(visLabels.size(), new CoprocessorException(
+      setExceptionResults(visLabels.size(),
-          "VisibilityController not yet initialized"), response);
+        new VisibilityControllerNotReadyException("VisibilityController not yet initialized!"),
+        response);
     } else {
       try {
         checkCallingUserAuth();
@@ -688,8 +694,9 @@ public class VisibilityController extends BaseMasterAndRegionObserver implements
     VisibilityLabelsResponse.Builder response = VisibilityLabelsResponse.newBuilder();
     List<ByteString> auths = request.getAuthList();
     if (!initialized) {
-      setExceptionResults(auths.size(), new CoprocessorException(
+      setExceptionResults(auths.size(),
-          "VisibilityController not yet initialized"), response);
+        new VisibilityControllerNotReadyException("VisibilityController not yet initialized!"),
+        response);
     } else {
       try {
         checkCallingUserAuth();

hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDefaultVisLabelService.java

@@ -25,18 +25,25 @@ import static org.junit.Assert.fail;
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Threads;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -44,6 +51,7 @@ import org.junit.experimental.categories.Category;
 
 @Category(MediumTests.class)
 public class TestVisibilityLabelsWithDefaultVisLabelService extends TestVisibilityLabels {
+  final Log LOG = LogFactory.getLog(getClass());
 
   @BeforeClass
   public static void setupBeforeClass() throws Exception {
@@ -104,19 +112,33 @@ public class TestVisibilityLabelsWithDefaultVisLabelService extends TestVisibili
     // Start one new RS
     RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
     waitForLabelsRegionAvailability(rs.getRegionServer());
-    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
+    final AtomicBoolean vcInitialized = new AtomicBoolean(true);
-        new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+    do {
-      public VisibilityLabelsResponse run() throws Exception {
+      PrivilegedExceptionAction<VisibilityLabelsResponse> action =
-        String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, "ABC", "XYZ" };
+          new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
-        try {
+        public VisibilityLabelsResponse run() throws Exception {
-          VisibilityClient.addLabels(conf, labels);
+          String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, "ABC", "XYZ" };
-        } catch (Throwable t) {
+          try {
-          throw new IOException(t);
+            VisibilityLabelsResponse resp = VisibilityClient.addLabels(conf, labels);
+            List<RegionActionResult> results = resp.getResultList();
+            if (results.get(0).hasException()) {
+              NameBytesPair pair = results.get(0).getException();
+              Throwable t = ProtobufUtil.toException(pair);
+              LOG.debug("Got exception writing labels", t);
+              if (t instanceof VisibilityControllerNotReadyException) {
+                vcInitialized.set(false);
+                LOG.warn("VisibilityController was not yet initialized");
+                Threads.sleep(10);
+              }
+            } else LOG.debug("new labels added: " + resp);
+          } catch (Throwable t) {
+            throw new IOException(t);
+          }
+          return null;
         }
-        return null;
+      };
-      }
+      SUPERUSER.runAs(action);
-    };
+    } while (!vcInitialized.get());
-    SUPERUSER.runAs(action);
     // Scan the visibility label
     Scan s = new Scan();
     s.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL));