HBASE-12373 Provide a command to list visibility labels (Jerry He)

hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityClient.java

@@ -21,6 +21,7 @@ import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LA
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
@@ -34,6 +35,8 @@ import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
 import org.apache.hadoop.hbase.ipc.ServerRpcController;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsRequest;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse;
+import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.ListLabelsRequest;
+import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.ListLabelsResponse;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.SetAuthsRequest;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest;
@@ -161,6 +164,57 @@ public class VisibilityClient {
     }
   }
 
+  /**
+   * Retrieve the list of visibility labels defined in the system.
+   * @param conf
+   * @param regex  The regular expression to filter which labels are returned.
+   * @return labels The list of visibility labels defined in the system.
+   * @throws Throwable
+   */
+  public static ListLabelsResponse listLabels(Configuration conf, final String regex)
+      throws Throwable {
+    Connection connection = null;
+    Table table = null;
+    try {
+      connection = ConnectionFactory.createConnection(conf);
+      table = connection.getTable(LABELS_TABLE_NAME);
+      Batch.Call<VisibilityLabelsService, ListLabelsResponse> callable =
+          new Batch.Call<VisibilityLabelsService, ListLabelsResponse>() {
+        ServerRpcController controller = new ServerRpcController();
+        BlockingRpcCallback<ListLabelsResponse> rpcCallback =
+            new BlockingRpcCallback<ListLabelsResponse>();
+
+        public ListLabelsResponse call(VisibilityLabelsService service) throws IOException {
+          ListLabelsRequest.Builder listAuthLabelsReqBuilder = ListLabelsRequest.newBuilder();
+          if (regex != null) {
+            // Compile the regex here to catch any regex exception earlier.
+            Pattern pattern = Pattern.compile(regex);
+            listAuthLabelsReqBuilder.setRegex(pattern.toString());
+          }
+          service.listLabels(controller, listAuthLabelsReqBuilder.build(), rpcCallback);
+          ListLabelsResponse response = rpcCallback.get();
+          if (controller.failedOnException()) {
+            throw controller.getFailedOn();
+          }
+          return response;
+        }
+      };
+      Map<byte[], ListLabelsResponse> result =
+          table.coprocessorService(VisibilityLabelsService.class, HConstants.EMPTY_BYTE_ARRAY,
+            HConstants.EMPTY_BYTE_ARRAY, callable);
+      return result.values().iterator().next(); // There will be exactly one region for labels
+      // table and so one entry in result Map.
+    }
+    finally {
+      if (table != null) {
+        table.close();
+      }
+      if (connection != null) {
+        connection.close();
+      }
+    }
+  }
+
   /**
    * Removes given labels from user's globally authorized list of labels.
    * @param conf

hbase-protocol/src/main/protobuf/VisibilityLabels.proto

@@ -60,6 +60,14 @@ message GetAuthsResponse {
   repeated bytes auth = 2;
 }
 
+message ListLabelsRequest {
+  optional string regex = 1;
+}
+
+message ListLabelsResponse {
+  repeated bytes label = 1;
+}
+
 service VisibilityLabelsService {
   rpc addLabels(VisibilityLabelsRequest)
     returns (VisibilityLabelsResponse);
@@ -69,4 +77,6 @@ service VisibilityLabelsService {
     returns (VisibilityLabelsResponse);
   rpc getAuths(GetAuthsRequest)
     returns (GetAuthsResponse);
+  rpc listLabels(ListLabelsRequest)
+    returns (ListLabelsResponse);
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java

@@ -36,6 +36,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -371,6 +372,26 @@ public class DefaultVisibilityLabelServiceImpl implements VisibilityLabelService
     return auths;
   }
 
+  @Override
+  public List<String> listLabels(String regex) throws IOException {
+    assert (labelsRegion != null);
+    Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths =
+        extractLabelsAndAuths(getExistingLabelsWithAuths());
+    Map<String, Integer> labels = labelsAndUserAuths.getFirst();
+    labels.remove(SYSTEM_LABEL);
+    if (regex != null) {
+      Pattern pattern = Pattern.compile(regex);
+      ArrayList<String> matchedLabels = new ArrayList<String>();
+      for (String label : labels.keySet()) {
+        if (pattern.matcher(label).matches()) {
+          matchedLabels.add(label);
+        }
+      }
+      return matchedLabels;
+    }
+    return new ArrayList<String>(labels.keySet());
+  }
+
   @Override
   public List<Tag> createVisibilityExpTags(String visExpression, boolean withSerializationFormat,
       boolean checkAuths) throws IOException {

hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java

@@ -78,6 +78,8 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResul
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsRequest;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse;
+import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.ListLabelsRequest;
+import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.ListLabelsResponse;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.SetAuthsRequest;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest;
@@ -881,6 +883,37 @@ public class VisibilityController extends BaseMasterAndRegionObserver implements
     done.run(response.build());
   }
 
+  @Override
+  public synchronized void listLabels(RpcController controller, ListLabelsRequest request,
+      RpcCallback<ListLabelsResponse> done) {
+    ListLabelsResponse.Builder response = ListLabelsResponse.newBuilder();
+    if (!initialized) {
+      controller.setFailed("VisibilityController not yet initialized");
+    } else {
+      List<String> labels = null;
+      try {
+        // We do ACL check here as we create scanner directly on region. It will not make calls to
+        // AccessController CP methods.
+        if (this.acOn && !isSystemOrSuperUser()) {
+          User requestingUser = VisibilityUtils.getActiveUser();
+          throw new AccessDeniedException("User '"
+              + (requestingUser != null ? requestingUser.getShortName() : "null")
+              + "' is not authorized to perform this action.");
+        }
+        String regex = request.hasRegex() ? request.getRegex() : null;
+        labels = this.visibilityLabelService.listLabels(regex);
+      } catch (IOException e) {
+        ResponseConverter.setControllerException(controller, e);
+      }
+      if (labels != null && !labels.isEmpty()) {
+        for (String label : labels) {
+          response.addLabel(ByteStringer.wrap(Bytes.toBytes(label)));
+        }
+      }
+    }
+    done.run(response.build());
+  }
+
   private void checkCallingUserAuth() throws IOException {
     if (!this.acOn) {
       User user = VisibilityUtils.getActiveUser();

hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelService.java

@@ -81,6 +81,13 @@ public interface VisibilityLabelService extends Configurable {
    */
   List<String> getAuths(byte[] user, boolean systemCall) throws IOException;
 
+  /**
+   * Retrieve the list of visibility labels defined in the system.
+   * @param regex  The regular expression to filter which labels are returned.
+   * @return List of visibility labels
+   */
+  List<String> listLabels(String regex) throws IOException;
+
   /**
    * Creates tags corresponding to given visibility expression.
    * <br>

hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/ExpAsStringVisibilityLabelServiceImpl.java

@@ -167,6 +167,12 @@ public class ExpAsStringVisibilityLabelServiceImpl implements VisibilityLabelSer
     return auths;
   }
 
+  @Override
+  public List<String> listLabels(String regex) throws IOException {
+    // return an empty list for this implementation.
+    return new ArrayList<String>();
+  }
+
   @Override
   public List<Tag> createVisibilityExpTags(String visExpression, boolean withSerializationFormat,
       boolean checkAuths) throws IOException {

hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDefaultVisLabelService.java

@@ -18,8 +18,10 @@
 package org.apache.hadoop.hbase.security.visibility;
 
 import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
+import static org.apache.hadoop.hbase.security.visibility.VisibilityUtils.SYSTEM_LABEL;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
@@ -38,6 +40,7 @@ import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
+import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.ListLabelsResponse;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -50,6 +53,8 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import com.google.protobuf.ByteString;
+
 @Category({SecurityTests.class, MediumTests.class})
 public class TestVisibilityLabelsWithDefaultVisLabelService extends TestVisibilityLabels {
   final Log LOG = LogFactory.getLog(getClass());
@@ -164,4 +169,56 @@ public class TestVisibilityLabelsWithDefaultVisLabelService extends TestVisibili
     // One label is the "system" label.
     Assert.assertEquals("The count should be 13", 13, i);
   }
+
+  @Test
+  public void testListLabels() throws Throwable {
+    PrivilegedExceptionAction<ListLabelsResponse> action =
+        new PrivilegedExceptionAction<ListLabelsResponse>() {
+      public ListLabelsResponse run() throws Exception {
+        ListLabelsResponse response = null;
+        try {
+          response = VisibilityClient.listLabels(conf, null);
+        } catch (Throwable e) {
+          fail("Should not have thrown exception");
+        }
+        // The addLabels() in setup added:
+        // { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, COPYRIGHT, ACCENT,
+        //  UNICODE_VIS_TAG, UC1, UC2 };
+        // The previous tests added 2 more labels: ABC, XYZ
+        // The 'system' label is excluded.
+        List<ByteString> labels = response.getLabelList();
+        assertEquals(12, labels.size());
+        assertTrue(labels.contains(ByteString.copyFrom(SECRET.getBytes())));
+        assertTrue(labels.contains(ByteString.copyFrom(TOPSECRET.getBytes())));
+        assertTrue(labels.contains(ByteString.copyFrom(CONFIDENTIAL.getBytes())));
+        assertTrue(labels.contains(ByteString.copyFrom("ABC".getBytes())));
+        assertTrue(labels.contains(ByteString.copyFrom("XYZ".getBytes())));
+        assertFalse(labels.contains(ByteString.copyFrom(SYSTEM_LABEL.getBytes())));
+        return null;
+      }
+    };
+    SUPERUSER.runAs(action);
+  }
+
+  @Test
+  public void testListLabelsWithRegEx() throws Throwable {
+    PrivilegedExceptionAction<ListLabelsResponse> action =
+        new PrivilegedExceptionAction<ListLabelsResponse>() {
+      public ListLabelsResponse run() throws Exception {
+        ListLabelsResponse response = null;
+        try {
+          response = VisibilityClient.listLabels(conf, ".*secret");
+        } catch (Throwable e) {
+          fail("Should not have thrown exception");
+        }
+        // Only return the labels that end with 'secret'
+        List<ByteString> labels = response.getLabelList();
+        assertEquals(2, labels.size());
+        assertTrue(labels.contains(ByteString.copyFrom(SECRET.getBytes())));
+        assertTrue(labels.contains(ByteString.copyFrom(TOPSECRET.getBytes())));
+        return null;
+      }
+    };
+    SUPERUSER.runAs(action);
+  }
 }

hbase-shell/src/main/ruby/hbase/visibility_labels.rb

@@ -100,6 +100,20 @@ module Hbase
       end
     end
 
+    def list_labels(regex = ".*")
+      lables_table_available?
+      begin
+        response = VisibilityClient.listLabels(@config, regex)
+        if response.nil?
+          raise(ArgumentError, "DISABLED: Visibility labels feature is not available")
+        end
+        if response.getLabelList.empty?
+          raise(ArgumentError, "No auth label defined")
+        end
+        return response.getLabelList
+      end
+    end
+
     def clear_auths(user, *args)
       lables_table_available?
       # Normalize args
@@ -136,4 +150,4 @@ module Hbase
       @admin.tableExists(table_name)
     end
   end
-end
+end

hbase-shell/src/main/ruby/shell.rb

@@ -400,6 +400,7 @@ Shell.load_command_group(
   :comment => "NOTE: Above commands are only applicable if running with the VisibilityController coprocessor",
   :commands => %w[
     add_labels
+    list_labels
     set_auths
     get_auths
     clear_auths

hbase-shell/src/main/ruby/shell/commands/list_labels.rb

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class ListLabels < Command
+      def help
+        return <<-EOF
+List the visibility labels defined in the system.
+Optional regular expression parameter could be used to filter the labels being returned.
+Syntax : list_labels
+
+For example:
+
+    hbase> list_labels 'secret.*'
+    hbase> list_labels
+EOF
+      end
+
+      def command(regex = ".*")
+        format_simple_command do
+          list = visibility_labels_admin.list_labels(regex)
+          list.each do |label|
+            formatter.row([org.apache.hadoop.hbase.util.Bytes::toStringBinary(label.toByteArray)])
+          end
+        end
+      end
+    end
+  end
+end