HBASE-8663 a HBase Shell command to list the tables replicated from

current cluster (Demai Ni via JD)


git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1512488 13f79535-47bb-0310-9956-ffa450edef68

hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java

@@ -30,10 +30,16 @@ import org.apache.hadoop.hbase.replication.ReplicationPeers;
 import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.lang.Integer;
 
 /**
  * <p>
@@ -61,6 +67,15 @@ import java.util.Map;
 public class ReplicationAdmin implements Closeable {
   private static final Log LOG = LogFactory.getLog(ReplicationAdmin.class);
 
+  public static final String TNAME = "tableName";
+  public static final String CFNAME = "columnFamlyName";
+
+  // only Global for now, can add other type
+  // such as, 1) no global replication, or 2) the table is replicated to this cluster, etc.
+  public static final String REPLICATIONTYPE = "replicationType";
+  public static final String REPLICATIONGLOBAL = Integer
+      .toString(HConstants.REPLICATION_SCOPE_GLOBAL);
+
   private final HConnection connection;
   private final ReplicationQueuesClient replicationQueuesClient;
   private final ReplicationPeers replicationPeers;
@@ -166,4 +181,38 @@ public class ReplicationAdmin implements Closeable {
       this.connection.close();
     }
   }
+
+  
+  /**
+   * Find all column families that are replicated from this cluster
+   * @return the full list of the replicated column families of this cluster as:
+   *        tableName, family name, replicationType
+   *
+   * Currently replicationType is Global. In the future, more replication
+   * types may be extended here. For example
+   *  1) the replication may only apply to selected peers instead of all peers
+   *  2) the replicationType may indicate the host Cluster servers as Slave
+   *     for the table:columnFam.         
+   */
+  public List<HashMap<String, String>> listReplicated() throws IOException {
+    List<HashMap<String, String>> replicationColFams = new ArrayList<HashMap<String, String>>();
+    HTableDescriptor[] tables = this.connection.listTables();
+
+    for (HTableDescriptor table : tables) {
+      HColumnDescriptor[] columns = table.getColumnFamilies();
+      String tableName = table.getNameAsString();
+      for (HColumnDescriptor column : columns) {
+        if (column.getScope() != HConstants.REPLICATION_SCOPE_LOCAL) {
+          // At this moment, the columfam is replicated to all peers
+          HashMap<String, String> replicationEntry = new HashMap<String, String>();
+          replicationEntry.put(TNAME, tableName);
+          replicationEntry.put(CFNAME, column.getNameAsString());
+          replicationEntry.put(REPLICATIONTYPE, REPLICATIONGLOBAL);
+          replicationColFams.add(replicationEntry);
+        }
+      }
+    }
+
+    return replicationColFams;
+  } 
 }

hbase-server/src/main/ruby/hbase/replication_admin.rb

@@ -42,6 +42,13 @@ module Hbase
       @replication_admin.removePeer(id)
     end
 
+
+    #---------------------------------------------------------------------------------------------
+    # Show replcated tables/column families, and their ReplicationType
+    def list_replicated_tables
+       @replication_admin.listReplicated()
+    end
+
     #----------------------------------------------------------------------------------------------
     # List all peer clusters
     def list_peers

hbase-server/src/main/ruby/shell.rb

@@ -317,6 +317,7 @@ Shell.load_command_group(
     list_peers
     enable_peer
     disable_peer
+    list_replicated_tables
   ]
 )
 

hbase-server/src/main/ruby/shell/commands/list_replicated_tables.rb

@@ -0,0 +1,52 @@
+#
+# Copyright The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class ListReplicatedTables< Command
+      def help
+        return <<-EOF
+List all the tables and column families replicated from this cluster
+
+  hbase> list_replicated_tables
+  hbase> list_replicated_tables 'abc.*'
+EOF
+      end
+
+      def command(regex = ".*")
+        now = Time.now
+
+        formatter.header([ "TABLE:COLUMNFAMILY", "ReplicationType" ], [ 32 ])
+        list = replication_admin.list_replicated_tables
+        regex = /#{regex}/ unless regex.is_a?(Regexp)
+        list = list.select {|s| regex.match(s.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME))}
+        list.each do |e|
+          if e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONTYPE) == org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONGLOBAL
+             replicateType = "GLOBAL"
+          else
+             replicateType = "unknown"
+          end
+          formatter.row([e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME) + ":" + e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::CFNAME), replicateType], true, [32])
+        end
+        formatter.footer(now)
+      end
+    end
+  end
+end

hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java

@@ -18,11 +18,19 @@
 
 package org.apache.hadoop.hbase.replication;
 
+import java.util.HashMap;
+import java.util.List;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
 import org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
@@ -39,6 +47,7 @@ import org.junit.experimental.categories.Category;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
 
 @Category(LargeTests.class)
 public class TestReplicationSmallTests extends TestReplicationBase {
@@ -477,5 +486,59 @@ public class TestReplicationSmallTests extends TestReplicationBase {
     WALEdit edit = WALEdit.createCompaction(compactionDescriptor);
     Replication.scopeWALEdits(htable1.getTableDescriptor(), new HLogKey(), edit);
   }
+  
+  /**
+   * Test for HBASE-8663
+   * Create two new Tables with colfamilies enabled for replication then run
+   * ReplicationAdmin.listReplicated(). Finally verify the table:colfamilies. Note:
+   * TestReplicationAdmin is a better place for this testing but it would need mocks.
+   * @throws Exception
+   */
+  @Test(timeout = 300000)
+  public void testVerifyListReplicatedTable() throws Exception {
+	LOG.info("testVerifyListReplicatedTable");
+
+    final String tName = "VerifyListReplicated_";
+    final String colFam = "cf1";
+    final int numOfTables = 3;
+
+    HBaseAdmin hadmin = new HBaseAdmin(conf1);
+
+    // Create Tables
+    for (int i = 0; i < numOfTables; i++) {
+      HTableDescriptor ht = new HTableDescriptor(TableName.valueOf(tName + i));
+      HColumnDescriptor cfd = new HColumnDescriptor(colFam);
+      cfd.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
+      ht.addFamily(cfd);
+      hadmin.createTable(ht);
+    }
+
+    // verify the result
+    List<HashMap<String, String>> replicationColFams = admin.listReplicated();
+    int[] match = new int[numOfTables]; // array of 3 with init value of zero
+
+    for (int i = 0; i < replicationColFams.size(); i++) {
+      HashMap<String, String> replicationEntry = replicationColFams.get(i);
+      String tn = replicationEntry.get(ReplicationAdmin.TNAME);
+      if ((tn.startsWith(tName)) && replicationEntry.get(ReplicationAdmin.CFNAME).equals(colFam)) {
+        int m = Integer.parseInt(tn.substring(tn.length() - 1)); // get the last digit
+        match[m]++; // should only increase once
+      }
+    }
+
+    // check the matching result
+    for (int i = 0; i < match.length; i++) {
+      assertTrue("listReplicated() does not match table " + i, (match[i] == 1));
+    }
+
+    // drop tables
+    for (int i = 0; i < numOfTables; i++) {
+      String ht = tName + i;
+      hadmin.disableTable(ht);
+      hadmin.deleteTable(ht);
+    }
+
+    hadmin.close();
+  }
 
 }