HBASE-17752 Shell command to list snapshot sizes WRT quotas

hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java

@@ -31,6 +31,7 @@ import java.util.regex.Pattern;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellScanner;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
@@ -513,16 +514,50 @@ public class QuotaTableUtil {
     return QuotaProtos.SpaceQuotaSnapshot.parseFrom(bs).getQuotaUsage();
   }
 
-  static Scan createScanForSnapshotSizes(TableName table) {
+  static Scan createScanForSpaceSnapshotSizes() {
-    byte[] rowkey = getTableRowKey(table);
+    return createScanForSpaceSnapshotSizes(null);
-    return new Scan()
+  }
-        // Fetch just this one row
+
-        .withStartRow(rowkey)
+  static Scan createScanForSpaceSnapshotSizes(TableName table) {
-        .withStopRow(rowkey, true)
+    Scan s = new Scan();
-        // Just the usage family
+    if (null == table) {
-        .addFamily(QUOTA_FAMILY_USAGE)
+      // Read all tables, just look at the row prefix
-        // Only the snapshot size qualifiers
+      s.setRowPrefixFilter(QUOTA_TABLE_ROW_KEY_PREFIX);
-        .setFilter(new ColumnPrefixFilter(QUOTA_SNAPSHOT_SIZE_QUALIFIER));
+    } else {
+      // Fetch the exact row for the table
+      byte[] rowkey = getTableRowKey(table);
+      // Fetch just this one row
+      s.withStartRow(rowkey).withStopRow(rowkey, true);
+    }
+
+    // Just the usage family and only the snapshot size qualifiers
+    return s.addFamily(QUOTA_FAMILY_USAGE).setFilter(
+        new ColumnPrefixFilter(QUOTA_SNAPSHOT_SIZE_QUALIFIER));
+  }
+
+  /**
+   * Fetches any persisted HBase snapshot sizes stored in the quota table. The sizes here are
+   * computed relative to the table which the snapshot was created from. A snapshot's size will
+   * not include the size of files which the table still refers. These sizes, in bytes, are what
+   * is used internally to compute quota violation for tables and namespaces.
+   *
+   * @return A map of snapshot name to size in bytes per space quota computations
+   */
+  public static Map<String,Long> getObservedSnapshotSizes(Connection conn) throws IOException {
+    try (Table quotaTable = conn.getTable(QUOTA_TABLE_NAME);
+        ResultScanner rs = quotaTable.getScanner(createScanForSpaceSnapshotSizes())) {
+      final Map<String,Long> snapshotSizes = new HashMap<>();
+      for (Result r : rs) {
+        CellScanner cs = r.cellScanner();
+        while (cs.advance()) {
+          Cell c = cs.current();
+          final String snapshot = extractSnapshotNameFromSizeCell(c);
+          final long size = parseSnapshotSize(c);
+          snapshotSizes.put(snapshot, size);
+        }
+      }
+      return snapshotSizes;
+    }
   }
 
   /* =========================================================================
@@ -749,6 +784,12 @@ public class QuotaTableUtil {
     return Bytes.add(QUOTA_SNAPSHOT_SIZE_QUALIFIER, Bytes.toBytes(snapshotName));
   }
 
+  protected static String extractSnapshotNameFromSizeCell(Cell c) {
+    return Bytes.toString(
+        c.getQualifierArray(), c.getQualifierOffset() + QUOTA_SNAPSHOT_SIZE_QUALIFIER.length,
+        c.getQualifierLength() - QUOTA_SNAPSHOT_SIZE_QUALIFIER.length);
+  }
+
   protected static long extractSnapshotSize(
       byte[] data, int offset, int length) throws InvalidProtocolBufferException {
     ByteString byteStr = UnsafeByteOperations.unsafeWrap(data, offset, length);

hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java

@@ -114,7 +114,7 @@ public class TableQuotaSnapshotStore implements QuotaSnapshotStore<TableName> {
    */
   long getSnapshotSizesForTable(TableName tn) throws IOException {
     try (Table quotaTable = conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
-      Scan s = QuotaTableUtil.createScanForSnapshotSizes(tn);
+      Scan s = QuotaTableUtil.createScanForSpaceSnapshotSizes(tn);
       ResultScanner rs = quotaTable.getScanner(s);
       try {
         long size = 0L;

hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotasWithSnapshots.java

@@ -173,6 +173,13 @@ public class TestSpaceQuotasWithSnapshots {
         return expectedFinalSize == snapshot.getUsage();
       }
     });
+
+    Map<String,Long> snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn);
+    Long size = snapshotSizes.get(snapshot1);
+    assertNotNull("Did not observe the size of the snapshot", size);
+    assertEquals(
+        "The recorded size of the HBase snapshot was not the size we expected", actualInitialSize,
+        size.longValue());
   }
 
   @Test
@@ -265,6 +272,13 @@ public class TestSpaceQuotasWithSnapshots {
         return expectedFinalSize == snapshot.getUsage();
       }
     });
+
+    Map<String,Long> snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn);
+    Long size = snapshotSizes.get(snapshot1);
+    assertNotNull("Did not observe the size of the snapshot", size);
+    assertEquals(
+        "The recorded size of the HBase snapshot was not the size we expected", actualInitialSize,
+        size.longValue());
   }
 
   @Test

hbase-shell/src/main/ruby/hbase/quotas.rb

@@ -241,6 +241,10 @@ module Hbase
       return count
     end
 
+    def list_snapshot_sizes()
+      QuotaTableUtil.getObservedSnapshotSizes(@admin.getConnection())
+    end
+
     def _parse_size(str_limit)
       str_limit = str_limit.downcase
       match = /(\d+)([bkmgtp%]*)/.match(str_limit)

hbase-shell/src/main/ruby/shell.rb

@@ -423,6 +423,7 @@ Shell.load_command_group(
     list_quotas
     list_quota_table_sizes
     list_quota_snapshots
+    list_snapshot_sizes
   ]
 )
 

hbase-shell/src/main/ruby/shell/commands/list_snapshot_sizes.rb

@@ -0,0 +1,42 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class ListSnapshotSizes < Command
+      def help
+        return <<-EOF
+Lists the size of every HBase snapshot given the space quota size computation
+algorithms. An HBase snapshot only "owns" the size of a file when the table
+from which the snapshot was created no longer refers to that file.
+EOF
+      end
+
+      def command(args = {})
+        formatter.header(["SNAPSHOT", "SIZE"])
+        count = 0
+        quotas_admin.list_snapshot_sizes().each do |snapshot,size|
+          formatter.row([snapshot.to_s, size.to_s])
+          count += 1
+        end
+        formatter.footer(count)
+      end
+    end
+  end
+end

hbase-shell/src/test/ruby/hbase/quotas_test.rb

@@ -32,7 +32,7 @@ module Hbase
     def setup
       setup_hbase
       # Create test table if it does not exist
-      @test_name = "hbase_shell_tests_table"
+      @test_name = "hbase_shell_quota_tests_table"
       create_test_table(@test_name)
     end
 
@@ -109,5 +109,32 @@ module Hbase
       output = capture_stdout{ command(:list_quotas) }
       assert(output.include?("0 row(s)"))
     end
+
+    define_test 'can view size of snapshots' do
+      snapshot1 = "#{@test_name}_1"
+      snapshot2 = "#{@test_name}_2"
+      # Set a quota on our table
+      command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS, TABLE => @test_name)
+      (1..10).each{|i| command(:put, @test_name, 'a', "x:#{i}", "#{i}")}
+      command(:flush, @test_name)
+      command(:snapshot, @test_name, snapshot1)
+      (1..10).each{|i| command(:put, @test_name, 'b', "x:#{i}", "#{i}")}
+      command(:flush, @test_name)
+      command(:snapshot, @test_name, snapshot2)
+      duration_to_check = 1000 * 30
+      start = current = Time.now.to_i
+      # Poor man's Waiter from Java test classes
+      while current - start < duration_to_check
+        output = capture_stdout{ command(:list_snapshot_sizes) }
+        if output.include? snapshot1 and output.include? snapshot2
+          break
+        end
+        sleep 5
+        current = Time.now.to_i
+      end
+      output = capture_stdout{ command(:list_snapshot_sizes) }
+      assert(output.include? snapshot1)
+      assert(output.include? snapshot2)
+    end
   end
 end

hbase-shell/src/test/ruby/tests_runner.rb

@@ -37,6 +37,8 @@ unless defined?($TEST_CLUSTER)
   $TEST_CLUSTER.configuration.setInt("hbase.regionserver.msginterval", 100)
   $TEST_CLUSTER.configuration.setInt("hbase.client.pause", 250)
   $TEST_CLUSTER.configuration.set("hbase.quota.enabled", "true")
+  $TEST_CLUSTER.configuration.set('hbase.master.quotas.snapshot.chore.period', 5000)
+  $TEST_CLUSTER.configuration.set('hbase.master.quotas.snapshot.chore.delay', 5000)
   $TEST_CLUSTER.configuration.setInt(org.apache.hadoop.hbase.HConstants::HBASE_CLIENT_RETRIES_NUMBER, 6)
   $TEST_CLUSTER.startMiniCluster
   @own_cluster = true