From 3cd929eea253afd632b6def7c24df434872a2d7d Mon Sep 17 00:00:00 2001 From: tedyu Date: Thu, 9 Apr 2015 17:57:57 -0700 Subject: [PATCH] HBASE-13381 Expand TestSizeFailures to include small scans (Josh Elser) --- .../hadoop/hbase/client/TestSizeFailures.java | 194 ++++++++++-------- 1 file changed, 104 insertions(+), 90 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java index a77c50c56a7..a0b0f70b442 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java @@ -20,12 +20,10 @@ package org.apache.hadoop.hbase.client; import static org.junit.Assert.assertEquals; -import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; -import java.util.TreeSet; +import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -41,16 +39,17 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import com.google.common.collect.Maps; + @Category(LargeTests.class) public class TestSizeFailures { - final Log LOG = LogFactory.getLog(getClass()); + static final Log LOG = LogFactory.getLog(TestSizeFailures.class); protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static byte [] FAMILY = Bytes.toBytes("testFamily"); protected static int SLAVES = 1; + private static TableName TABLENAME; + private static final int NUM_ROWS = 1000 * 1000, NUM_COLS = 10; - /** - * @throws java.lang.Exception - */ @BeforeClass public static void setUpBeforeClass() throws Exception { // Uncomment the following lines if more verbosity is needed for @@ -61,11 +60,49 @@ public class TestSizeFailures { Configuration conf = TEST_UTIL.getConfiguration(); conf.setBoolean("hbase.table.sanity.checks", true); // ignore sanity checks in the server TEST_UTIL.startMiniCluster(SLAVES); + + // Write a bunch of data + TABLENAME = TableName.valueOf("testSizeFailures"); + List qualifiers = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + qualifiers.add(Bytes.toBytes(Integer.toString(i))); + } + + HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); + HTableDescriptor desc = new HTableDescriptor(TABLENAME); + desc.addFamily(hcd); + byte[][] splits = new byte[9][2]; + for (int i = 1; i < 10; i++) { + int split = 48 + i; + splits[i - 1][0] = (byte) (split >>> 8); + splits[i - 1][0] = (byte) (split); + } + TEST_UTIL.getHBaseAdmin().createTable(desc, splits); + Connection conn = TEST_UTIL.getConnection(); + + try (Table table = conn.getTable(TABLENAME)) { + List puts = new LinkedList<>(); + for (int i = 0; i < NUM_ROWS; i++) { + Put p = new Put(Bytes.toBytes(Integer.toString(i))); + for (int j = 0; j < NUM_COLS; j++) { + byte[] value = new byte[50]; + Bytes.random(value); + p.addColumn(FAMILY, Bytes.toBytes(Integer.toString(j)), value); + } + puts.add(p); + + if (puts.size() == 1000) { + table.batch(puts, new Object[1000]); + puts.clear(); + } + } + + if (puts.size() > 0) { + table.batch(puts, new Object[puts.size()]); + } + } } - /** - * @throws java.lang.Exception - */ @AfterClass public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); @@ -74,89 +111,66 @@ public class TestSizeFailures { /** * Basic client side validation of HBASE-13262 */ - @Test - public void testScannerSeesAllRecords() throws Exception { - final int NUM_ROWS = 1000 * 1000, NUM_COLS = 10; - final TableName TABLENAME = TableName.valueOf("testScannerSeesAllRecords"); - List qualifiers = new ArrayList<>(); - for (int i = 1; i <= 10; i++) { - qualifiers.add(Bytes.toBytes(Integer.toString(i))); - } + @Test + public void testScannerSeesAllRecords() throws Exception { + Connection conn = TEST_UTIL.getConnection(); + try (Table table = conn.getTable(TABLENAME)) { + Scan s = new Scan(); + s.addFamily(FAMILY); + s.setMaxResultSize(-1); + s.setBatch(-1); + s.setCaching(500); + Entry entry = sumTable(table.getScanner(s)); + long rowsObserved = entry.getKey(); + long entriesObserved = entry.getValue(); - HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); - HTableDescriptor desc = new HTableDescriptor(TABLENAME); - desc.addFamily(hcd); - byte[][] splits = new byte[9][2]; - for (int i = 1; i < 10; i++) { - int split = 48 + i; - splits[i - 1][0] = (byte) (split >>> 8); - splits[i - 1][0] = (byte) (split); - } - TEST_UTIL.getHBaseAdmin().createTable(desc, splits); - Connection conn = TEST_UTIL.getConnection(); + // Verify that we see 1M rows and 10M cells + assertEquals(NUM_ROWS, rowsObserved); + assertEquals(NUM_ROWS * NUM_COLS, entriesObserved); + } + } - try (Table table = conn.getTable(TABLENAME)) { - List puts = new LinkedList<>(); - for (int i = 0; i < NUM_ROWS; i++) { - Put p = new Put(Bytes.toBytes(Integer.toString(i))); - for (int j = 0; j < NUM_COLS; j++) { - byte[] value = new byte[50]; - Bytes.random(value); - p.addColumn(FAMILY, Bytes.toBytes(Integer.toString(j)), value); - } - puts.add(p); + /** + * Basic client side validation of HBASE-13262 + */ + @Test + public void testSmallScannerSeesAllRecords() throws Exception { + Connection conn = TEST_UTIL.getConnection(); + try (Table table = conn.getTable(TABLENAME)) { + Scan s = new Scan(); + s.setSmall(true); + s.addFamily(FAMILY); + s.setMaxResultSize(-1); + s.setBatch(-1); + s.setCaching(500); + Entry entry = sumTable(table.getScanner(s)); + long rowsObserved = entry.getKey(); + long entriesObserved = entry.getValue(); - if (puts.size() == 1000) { - Object[] results = new Object[1000]; - try { - table.batch(puts, results); - } catch (IOException e) { - LOG.error("Failed to write data", e); - LOG.debug("Errors: " + Arrays.toString(results)); - } + // Verify that we see 1M rows and 10M cells + assertEquals(NUM_ROWS, rowsObserved); + assertEquals(NUM_ROWS * NUM_COLS, entriesObserved); + } + } - puts.clear(); - } - } + /** + * Count the number of rows and the number of entries from a scanner + * + * @param scanner + * The Scanner + * @return An entry where the first item is rows observed and the second is entries observed. + */ + private Entry sumTable(ResultScanner scanner) { + long rowsObserved = 0l; + long entriesObserved = 0l; - if (puts.size() > 0) { - Object[] results = new Object[puts.size()]; - try { - table.batch(puts, results); - } catch (IOException e) { - LOG.error("Failed to write data", e); - LOG.debug("Errors: " + Arrays.toString(results)); - } - } - - // Flush the memstore to disk - TEST_UTIL.getHBaseAdmin().flush(TABLENAME); - - TreeSet rows = new TreeSet<>(); - long rowsObserved = 0l; - long entriesObserved = 0l; - Scan s = new Scan(); - s.addFamily(FAMILY); - s.setMaxResultSize(-1); - s.setBatch(-1); - s.setCaching(500); - ResultScanner scanner = table.getScanner(s); - // Read all the records in the table - for (Result result : scanner) { - rowsObserved++; - String row = new String(result.getRow()); - rows.add(Integer.parseInt(row)); - while (result.advance()) { - entriesObserved++; - // result.current(); - } - } - - // Verify that we see 1M rows and 10M cells - assertEquals(NUM_ROWS, rowsObserved); - assertEquals(NUM_ROWS * NUM_COLS, entriesObserved); - } - - conn.close(); - } + // Read all the records in the table + for (Result result : scanner) { + rowsObserved++; + while (result.advance()) { + entriesObserved++; + } + } + return Maps.immutableEntry(rowsObserved,entriesObserved); + } }