Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-16776 Remove duplicated versions of countRow() in tests

This commit is contained in:
Matteo Bertozzi 2016-10-05 19:45:50 -07:00
parent 06758bf630
commit b548d4978b
7 changed files with 72 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
View File

@ -2219,13 +2219,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
for (byte[] family: families) {
scan.addFamily(family);
}
ResultScanner results = table.getScanner(scan);
int count = 0;
for (@SuppressWarnings("unused") Result res : results) {
count++;
}
results.close();
return count;
return countRows(table, scan);
}
/**
@ -2240,6 +2234,32 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
}
}
public int countRows(final Region region) throws IOException {
return countRows(region, new Scan());
}
public int countRows(final Region region, final Scan scan) throws IOException {
InternalScanner scanner = region.getScanner(scan);
try {
return countRows(scanner);
} finally {
scanner.close();
}
}
public int countRows(final InternalScanner scanner) throws IOException {
// Do not retrieve the mob data when scanning
int scannedCount = 0;
List<Cell> results = new ArrayList<Cell>();
boolean hasMore = true;
while (hasMore) {
hasMore = scanner.next(results);
scannedCount += results.size();
results.clear();
}
return scannedCount;
}
/**
* Return an md5 digest of the entire contents of a table.
*/

37
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
View File

@ -504,7 +504,7 @@ public class TestFromClientSide {
byte [] endKey = regions.get(0).getRegionInfo().getEndKey();
// Count rows with a filter that stops us before passed 'endKey'.
// Should be count of rows in first region.
int endKeyCount = countRows(t, createScanWithRowFilter(endKey));
int endKeyCount = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey));
assertTrue(endKeyCount < rowCount);
// How do I know I did not got to second region? Thats tough. Can't really
@ -516,29 +516,29 @@ public class TestFromClientSide {
// New test. Make it so scan goes into next region by one and then two.
// Make sure count comes out right.
byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)};
int plusOneCount = countRows(t, createScanWithRowFilter(key));
int plusOneCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key));
assertEquals(endKeyCount + 1, plusOneCount);
key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)};
int plusTwoCount = countRows(t, createScanWithRowFilter(key));
int plusTwoCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key));
assertEquals(endKeyCount + 2, plusTwoCount);
// New test. Make it so I scan one less than endkey.
key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)};
int minusOneCount = countRows(t, createScanWithRowFilter(key));
int minusOneCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key));
assertEquals(endKeyCount - 1, minusOneCount);
// For above test... study logs. Make sure we do "Finished with scanning.."
// in first region and that we do not fall into the next region.
key = new byte [] {'a', 'a', 'a'};
int countBBB = countRows(t,
int countBBB = TEST_UTIL.countRows(t,
createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL));
assertEquals(1, countBBB);
int countGreater = countRows(t, createScanWithRowFilter(endKey, null,
int countGreater = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey, null,
CompareFilter.CompareOp.GREATER_OR_EQUAL));
// Because started at start of table.
assertEquals(0, countGreater);
countGreater = countRows(t, createScanWithRowFilter(endKey, endKey,
countGreater = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey, endKey,
CompareFilter.CompareOp.GREATER_OR_EQUAL));
assertEquals(rowCount - endKeyCount, countGreater);
}
@ -602,7 +602,7 @@ public class TestFromClientSide {
try (Table t = TEST_UTIL.getConnection().getTable(name)) {
int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
TEST_UTIL.getAdmin().flush(name);
int actualRowCount = countRows(t, new Scan().addColumn(FAMILY, FAMILY));
int actualRowCount = TEST_UTIL.countRows(t, new Scan().addColumn(FAMILY, FAMILY));
assertEquals(rowCount, actualRowCount);
}
assertTrue(ExceptionInReseekRegionObserver.reqCount.get() > 0);
@ -635,28 +635,9 @@ public class TestFromClientSide {
return s;
}
/*
* @param t
* @param s
* @return Count of rows in table.
* @throws IOException
*/
private int countRows(final Table t, final Scan s)
throws IOException {
// Assert all rows in table.
ResultScanner scanner = t.getScanner(s);
int count = 0;
for (Result result: scanner) {
count++;
assertTrue(result.size() > 0);
// LOG.info("Count=" + count + ", row=" + Bytes.toString(result.getRow()));
}
return count;
}
private void assertRowCount(final Table t, final int expected)
throws IOException {
assertEquals(expected, countRows(t, new Scan()));
assertEquals(expected, TEST_UTIL.countRows(t, new Scan()));
}
/*

12
hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerCrashProcedure.java
View File

@ -95,7 +95,7 @@ public class TestServerCrashProcedure {
try (Table t = this.util.getConnection().getTable(tableName)) {
// Load the table with a bit of data so some logs to split and some edits in each region.
this.util.loadTable(t, HBaseTestingUtility.COLUMNS[0]);
int count = countRows(t);
int count = util.countRows(t);
// Run the procedure executor outside the master so we can mess with it. Need to disable
// Master's running of the server crash processing.
HMaster master = this.util.getHBaseCluster().getMaster();
@ -123,15 +123,7 @@ public class TestServerCrashProcedure {
// Now run through the procedure twice crashing the executor on each step...
MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
// Assert all data came back.
assertEquals(count, countRows(t));
assertEquals(count, util.countRows(t));
}
}
int countRows(final Table t) throws IOException {
int count = 0;
try (ResultScanner scanner = t.getScanner(new Scan())) {
while(scanner.next() != null) count++;
}
return count;
}
}

39
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
View File

@ -132,7 +132,7 @@ public class TestMobStoreCompaction {
}
assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
assertEquals("Before compaction: mob file count", 0, countMobFiles());
assertEquals("Before compaction: rows", compactionThreshold, countRows());
assertEquals("Before compaction: rows", compactionThreshold, UTIL.countRows(region));
assertEquals("Before compaction: mob rows", 0, countMobRows());
region.compactStores();
@ -140,7 +140,7 @@ public class TestMobStoreCompaction {
assertEquals("After compaction: store files", 1, countStoreFiles());
assertEquals("After compaction: mob file count", 0, countMobFiles());
assertEquals("After compaction: referenced mob file count", 0, countReferencedMobFiles());
assertEquals("After compaction: rows", compactionThreshold, countRows());
assertEquals("After compaction: rows", compactionThreshold, UTIL.countRows(region));
assertEquals("After compaction: mob rows", 0, countMobRows());
}
@ -159,7 +159,7 @@ public class TestMobStoreCompaction {
}
assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
assertEquals("Before compaction: mob file count", compactionThreshold, countMobFiles());
assertEquals("Before compaction: rows", compactionThreshold, countRows());
assertEquals("Before compaction: rows", compactionThreshold, UTIL.countRows(region));
assertEquals("Before compaction: mob rows", compactionThreshold, countMobRows());
assertEquals("Before compaction: number of mob cells", compactionThreshold,
countMobCellsInMetadata());
@ -171,7 +171,7 @@ public class TestMobStoreCompaction {
assertEquals("After compaction: store files", 1, countStoreFiles());
assertEquals("After compaction: mob file count", compactionThreshold, countMobFiles());
assertEquals("After compaction: referenced mob file count", 0, countReferencedMobFiles());
assertEquals("After compaction: rows", compactionThreshold, countRows());
assertEquals("After compaction: rows", compactionThreshold, UTIL.countRows(region));
assertEquals("After compaction: mob rows", 0, countMobRows());
}
@ -200,7 +200,7 @@ public class TestMobStoreCompaction {
assertTrue("Bulkload result:", result);
assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
assertEquals("Before compaction: mob file count", 0, countMobFiles());
assertEquals("Before compaction: rows", compactionThreshold, countRows());
assertEquals("Before compaction: rows", compactionThreshold, UTIL.countRows(region));
assertEquals("Before compaction: mob rows", 0, countMobRows());
assertEquals("Before compaction: referenced mob file count", 0, countReferencedMobFiles());
@ -208,7 +208,7 @@ public class TestMobStoreCompaction {
assertEquals("After compaction: store files", 1, countStoreFiles());
assertEquals("After compaction: mob file count:", 1, countMobFiles());
assertEquals("After compaction: rows", compactionThreshold, countRows());
assertEquals("After compaction: rows", compactionThreshold, UTIL.countRows(region));
assertEquals("After compaction: mob rows", compactionThreshold, countMobRows());
assertEquals("After compaction: referenced mob file count", 1, countReferencedMobFiles());
assertEquals("After compaction: number of mob cells", compactionThreshold,
@ -230,7 +230,7 @@ public class TestMobStoreCompaction {
}
assertEquals("Before compaction: store files", numHfiles, countStoreFiles());
assertEquals("Before compaction: mob file count", numHfiles, countMobFiles());
assertEquals("Before compaction: rows", numHfiles, countRows());
assertEquals("Before compaction: rows", numHfiles, UTIL.countRows(region));
assertEquals("Before compaction: mob rows", numHfiles, countMobRows());
assertEquals("Before compaction: number of mob cells", numHfiles, countMobCellsInMetadata());
// now let's delete some cells that contain mobs
@ -351,29 +351,6 @@ public class TestMobStoreCompaction {
return scannedCount;
}
private int countRows() throws IOException {
Scan scan = new Scan();
InternalScanner scanner = region.getScanner(scan);
try {
return countRows(scanner);
} finally {
scanner.close();
}
}
private int countRows(InternalScanner scanner) throws IOException {
// Do not retrieve the mob data when scanning
int scannedCount = 0;
List<Cell> results = new ArrayList<Cell>();
boolean hasMore = true;
while (hasMore) {
hasMore = scanner.next(results);
scannedCount += results.size();
results.clear();
}
return scannedCount;
}
private byte[] makeDummyData(int size) {
byte[] dummyData = new byte[size];
new Random().nextBytes(dummyData);
@ -448,7 +425,7 @@ public class TestMobStoreCompaction {
StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_DROP_DELETES, null,
scanners, 0L, HConstants.LATEST_TIMESTAMP);
try {
size += countRows(scanner);
size += UTIL.countRows(scanner);
} finally {
scanner.close();
}

37
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java
View File

@ -244,8 +244,8 @@ public class TestRegionMergeTransaction {
final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
assertEquals(rowCountOfRegionA, countRows(this.region_a));
assertEquals(rowCountOfRegionB, countRows(this.region_b));
assertEquals(rowCountOfRegionA, TEST_UTIL.countRows(this.region_a));
assertEquals(rowCountOfRegionB, TEST_UTIL.countRows(this.region_b));
// Start transaction.
RegionMergeTransactionImpl mt = prepareOnGoodRegions();
@ -272,7 +272,7 @@ public class TestRegionMergeTransaction {
mergedRegion.getRegionInfo().getEndKey()));
// Count rows. merged region are already open
try {
int mergedRegionRowCount = countRows(mergedRegion);
int mergedRegionRowCount = TEST_UTIL.countRows(mergedRegion);
assertEquals((rowCountOfRegionA + rowCountOfRegionB),
mergedRegionRowCount);
} finally {
@ -288,8 +288,8 @@ public class TestRegionMergeTransaction {
final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
assertEquals(rowCountOfRegionA, countRows(this.region_a));
assertEquals(rowCountOfRegionB, countRows(this.region_b));
assertEquals(rowCountOfRegionA, TEST_UTIL.countRows(this.region_a));
assertEquals(rowCountOfRegionB, TEST_UTIL.countRows(this.region_b));
// Start transaction.
RegionMergeTransactionImpl mt = prepareOnGoodRegions();
@ -314,9 +314,9 @@ public class TestRegionMergeTransaction {
assertTrue(mt.rollback(null, null));
// Assert I can scan region_a and region_b.
int rowCountOfRegionA2 = countRows(this.region_a);
int rowCountOfRegionA2 = TEST_UTIL.countRows(this.region_a);
assertEquals(rowCountOfRegionA, rowCountOfRegionA2);
int rowCountOfRegionB2 = countRows(this.region_b);
int rowCountOfRegionB2 = TEST_UTIL.countRows(this.region_b);
assertEquals(rowCountOfRegionB, rowCountOfRegionB2);
// Assert rollback cleaned up stuff in fs
@ -332,7 +332,7 @@ public class TestRegionMergeTransaction {
// Count rows. daughters are already open
// Count rows. merged region are already open
try {
int mergedRegionRowCount = countRows(mergedRegion);
int mergedRegionRowCount = TEST_UTIL.countRows(mergedRegion);
assertEquals((rowCountOfRegionA + rowCountOfRegionB),
mergedRegionRowCount);
} finally {
@ -348,8 +348,8 @@ public class TestRegionMergeTransaction {
final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
assertEquals(rowCountOfRegionA, countRows(this.region_a));
assertEquals(rowCountOfRegionB, countRows(this.region_b));
assertEquals(rowCountOfRegionA, TEST_UTIL.countRows(this.region_a));
assertEquals(rowCountOfRegionB, TEST_UTIL.countRows(this.region_b));
// Start transaction.
RegionMergeTransactionImpl mt = prepareOnGoodRegions();
@ -448,23 +448,6 @@ public class TestRegionMergeTransaction {
TEST_UTIL.getConfiguration());
}
private int countRows(final HRegion r) throws IOException {
int rowcount = 0;
InternalScanner scanner = r.getScanner(new Scan());
try {
List<Cell> kvs = new ArrayList<Cell>();
boolean hasNext = true;
while (hasNext) {
hasNext = scanner.next(kvs);
if (!kvs.isEmpty())
rowcount++;
}
} finally {
scanner.close();
}
return rowcount;
}
/**
* Load region with rows from 'aaa' to 'zzz', skip the rows which are out of
* range of the region

42
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
View File

@ -80,10 +80,10 @@ public class TestSplitTransaction {
private static final byte [] ENDROW = new byte [] {'{', '{', '{'};
private static final byte [] GOOD_SPLIT_ROW = new byte [] {'d', 'd', 'd'};
private static final byte [] CF = HConstants.CATALOG_FAMILY;
private static boolean preRollBackCalled = false;
private static boolean postRollBackCalled = false;
@Before public void setup() throws IOException {
this.fs = FileSystem.get(TEST_UTIL.getConfiguration());
TEST_UTIL.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
@ -91,7 +91,7 @@ public class TestSplitTransaction {
final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
FSUtils.setRootDir(walConf, this.testdir);
this.wals = new WALFactory(walConf, null, this.getClass().getName());
this.parent = createRegion(this.testdir, this.wals);
RegionCoprocessorHost host = new RegionCoprocessorHost(this.parent, null, TEST_UTIL.getConfiguration());
this.parent.setCoprocessorHost(host);
@ -113,7 +113,7 @@ public class TestSplitTransaction {
@Test public void testFailAfterPONR() throws IOException, KeeperException {
final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
assertTrue(rowcount > 0);
int parentRowCount = countRows(this.parent);
int parentRowCount = TEST_UTIL.countRows(this.parent);
assertEquals(rowcount, parentRowCount);
// Start transaction.
@ -229,7 +229,7 @@ public class TestSplitTransaction {
@Test public void testWholesomeSplit() throws IOException {
final int rowcount = TEST_UTIL.loadRegion(this.parent, CF, true);
assertTrue(rowcount > 0);
int parentRowCount = countRows(this.parent);
int parentRowCount = TEST_UTIL.countRows(this.parent);
assertEquals(rowcount, parentRowCount);
// Pretend region's blocks are not in the cache, used for
@ -263,7 +263,7 @@ public class TestSplitTransaction {
int daughtersRowCount = 0;
for (Region openRegion: daughters) {
try {
int count = countRows(openRegion);
int count = TEST_UTIL.countRows(openRegion);
assertTrue(count > 0 && count != rowcount);
daughtersRowCount += count;
} finally {
@ -279,7 +279,7 @@ public class TestSplitTransaction {
public void testCountReferencesFailsSplit() throws IOException {
final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
assertTrue(rowcount > 0);
int parentRowCount = countRows(this.parent);
int parentRowCount = TEST_UTIL.countRows(this.parent);
assertEquals(rowcount, parentRowCount);
// Start transaction.
@ -307,7 +307,7 @@ public class TestSplitTransaction {
@Test public void testRollback() throws IOException {
final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
assertTrue(rowcount > 0);
int parentRowCount = countRows(this.parent);
int parentRowCount = TEST_UTIL.countRows(this.parent);
assertEquals(rowcount, parentRowCount);
// Start transaction.
@ -332,7 +332,7 @@ public class TestSplitTransaction {
assertTrue(spiedUponSt.rollback(null, null));
// Assert I can scan parent.
int parentRowCount2 = countRows(this.parent);
int parentRowCount2 = TEST_UTIL.countRows(this.parent);
assertEquals(parentRowCount, parentRowCount2);
// Assert rollback cleaned up stuff in fs
@ -347,7 +347,7 @@ public class TestSplitTransaction {
int daughtersRowCount = 0;
for (Region openRegion: daughters) {
try {
int count = countRows(openRegion);
int count = TEST_UTIL.countRows(openRegion);
assertTrue(count > 0 && count != rowcount);
daughtersRowCount += count;
} finally {
@ -359,7 +359,7 @@ public class TestSplitTransaction {
assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread());
assertTrue("Rollback hooks should be called.", wasRollBackHookCalled());
}
private boolean wasRollBackHookCalled(){
return (preRollBackCalled && postRollBackCalled);
}
@ -371,22 +371,6 @@ public class TestSplitTransaction {
private class MockedFailedDaughterCreation extends IOException {}
private class MockedFailedDaughterOpen extends IOException {}
private int countRows(final Region r) throws IOException {
int rowcount = 0;
InternalScanner scanner = r.getScanner(new Scan());
try {
List<Cell> kvs = new ArrayList<Cell>();
boolean hasNext = true;
while (hasNext) {
hasNext = scanner.next(kvs);
if (!kvs.isEmpty()) rowcount++;
}
} finally {
scanner.close();
}
return rowcount;
}
HRegion createRegion(final Path testdir, final WALFactory wals)
throws IOException {
// Make a region with start and end keys. Use 'aaa', to 'AAA'. The load
@ -402,14 +386,14 @@ public class TestSplitTransaction {
wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace()),
TEST_UTIL.getConfiguration());
}
public static class CustomObserver extends BaseRegionObserver{
@Override
public void preRollBackSplit(
ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException {
preRollBackCalled = true;
}
@Override
public void postRollBackSplit(
ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException {

18
hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
View File

@ -296,28 +296,16 @@ public class BaseTestHBaseFsck {
* Counts the number of rows to verify data loss or non-dataloss.
*/
int countRows() throws IOException {
Scan s = new Scan();
ResultScanner rs = tbl.getScanner(s);
int i = 0;
while(rs.next() !=null) {
i++;
}
return i;
return TEST_UTIL.countRows(tbl);
}
/**
* Counts the number of rows to verify data loss or non-dataloss.
*/
int countRows(byte[] start, byte[] end) throws IOException {
Scan s = new Scan(start, end);
ResultScanner rs = tbl.getScanner(s);
int i = 0;
while (rs.next() != null) {
i++;
}
return i;
return TEST_UTIL.countRows(tbl, new Scan(start, end));
}
/**
* delete table in preparation for next test
*