Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-10706 Disable writeToWal in tests where possible 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1577780 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
larsh 2014-03-15 04:22:19 +00:00
parent 8d7a4fcbdb
commit f3cb34588b
15 changed files with 57 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
View File

@ -31,7 +31,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
@ -218,12 +217,10 @@ public abstract class HBaseTestCase extends TestCase {
* @param r
* @param columnFamily
* @param column
* @param writeToWAL
* @throws IOException
* @return count of what we added.
*/
public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column,
boolean writeToWAL)
public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
throws IOException {
byte [] startKey = r.getRegionInfo().getStartKey();
byte [] endKey = r.getRegionInfo().getEndKey();
@ -232,32 +229,12 @@ public abstract class HBaseTestCase extends TestCase {
startKeyBytes = START_KEY_BYTES;
}
return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
startKeyBytes, endKey, -1, writeToWAL);
}
public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
throws IOException {
return addContent(r, columnFamily, column, true);
}
/**
* Add content to region <code>r</code> on the passed column
* <code>column</code>.
* Adds data of the from 'aaa', 'aab', etc where key and value are the same.
* @param r
* @param columnFamily
* @param writeToWAL
* @throws IOException
* @return count of what we added.
*/
public static long addContent(final HRegion r, final byte [] columnFamily, boolean writeToWAL)
throws IOException {
return addContent(r, columnFamily, null, writeToWAL);
startKeyBytes, endKey, -1);
}
public static long addContent(final HRegion r, final byte [] columnFamily)
throws IOException {
return addContent(r, columnFamily, null, true);
return addContent(r, columnFamily, null);
}
/**
@ -270,24 +247,14 @@ public abstract class HBaseTestCase extends TestCase {
* @throws IOException
* @return count of what we added.
*/
public static long addContent(final Incommon updater,
final String columnFamily, final boolean writeToWAL) throws IOException {
return addContent(updater, columnFamily, START_KEY_BYTES, null, writeToWAL);
}
public static long addContent(final Incommon updater,
final String columnFamily) throws IOException {
return addContent(updater, columnFamily, START_KEY_BYTES, null, true);
}
public static long addContent(final Incommon updater, final String family,
final String column, final boolean writeToWAL) throws IOException {
return addContent(updater, family, column, START_KEY_BYTES, null, writeToWAL);
return addContent(updater, columnFamily, START_KEY_BYTES, null);
}
public static long addContent(final Incommon updater, final String family,
final String column) throws IOException {
return addContent(updater, family, column, START_KEY_BYTES, null, true);
return addContent(updater, family, column, START_KEY_BYTES, null);
}
/**
@ -303,21 +270,14 @@ public abstract class HBaseTestCase extends TestCase {
* @throws IOException
*/
public static long addContent(final Incommon updater, final String columnFamily,
final byte [] startKeyBytes, final byte [] endKey, final boolean writeToWAL)
final byte [] startKeyBytes, final byte [] endKey)
throws IOException {
return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1, writeToWAL);
}
public static long addContent(final Incommon updater, final String family,
final String column, final byte [] startKeyBytes,
final byte [] endKey,
final boolean writeToWAL) throws IOException {
return addContent(updater, family, column, startKeyBytes, endKey, -1, writeToWAL);
return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
}
public static long addContent(final Incommon updater, final String family, String column,
final byte [] startKeyBytes, final byte [] endKey) throws IOException {
return addContent(updater, family, column, startKeyBytes, endKey, -1, true);
return addContent(updater, family, column, startKeyBytes, endKey, -1);
}
/**
@ -336,8 +296,7 @@ public abstract class HBaseTestCase extends TestCase {
public static long addContent(final Incommon updater,
final String columnFamily,
final String column,
final byte [] startKeyBytes, final byte [] endKey, final long ts,
final boolean writeToWAL)
final byte [] startKeyBytes, final byte [] endKey, final long ts)
throws IOException {
long count = 0;
// Add rows of three characters. The first character starts with the
@ -383,7 +342,7 @@ public abstract class HBaseTestCase extends TestCase {
} else {
put.add(split[0], split[1], t);
}
put.setDurability(writeToWAL ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
put.setDurability(Durability.SKIP_WAL);
updater.put(put);
count++;
} catch (RuntimeException ex) {
@ -409,13 +368,6 @@ public abstract class HBaseTestCase extends TestCase {
return count;
}
public static long addContent(final Incommon updater,
final String columnFamily,
final String column,
final byte [] startKeyBytes, final byte [] endKey, final long ts) throws IOException {
return addContent(updater, columnFamily, column, startKeyBytes, endKey, ts, true);
}
/**
* Implementors can flushcache.
*/
@ -586,7 +538,6 @@ public abstract class HBaseTestCase extends TestCase {
scanner.close();
}
@SuppressWarnings("unchecked")
public Iterator<Result> iterator() {
return scanner.iterator();
}

25
hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
View File

@ -1629,6 +1629,17 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
return loadTable(t, new byte[][] {f});
}
/**
* Load table with rows from 'aaa' to 'zzz'.
* @param t Table
* @param f Family
* @return Count of rows loaded.
* @throws IOException
*/
public int loadTable(final HTable t, final byte[] f, boolean writeToWAL) throws IOException {
return loadTable(t, new byte[][] {f}, null, writeToWAL);
}
/**
* Load table of multiple column families with rows from 'aaa' to 'zzz'.
* @param t Table
@ -1649,10 +1660,23 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* @throws IOException
*/
public int loadTable(final HTable t, final byte[][] f, byte[] value) throws IOException {
return loadTable(t, f, value, true);
}
/**
* Load table of multiple column families with rows from 'aaa' to 'zzz'.
* @param t Table
* @param f Array of Families to load
* @param value the values of the cells. If null is passed, the row key is used as value
* @return Count of rows loaded.
* @throws IOException
*/
public int loadTable(final HTable t, final byte[][] f, byte[] value, boolean writeToWAL) throws IOException {
t.setAutoFlush(false);
int rowCount = 0;
for (byte[] row : HBaseTestingUtility.ROWS) {
Put put = new Put(row);
put.setDurability(writeToWAL ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
for (int i = 0; i < f.length; i++) {
put.add(f[i], null, value != null ? value : row);
}
@ -1737,6 +1761,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
k[1] = b2;
k[2] = b3;
Put put = new Put(k);
put.setDurability(Durability.SKIP_WAL);
put.add(f, null, k);
if (r.getLog() == null) put.setDurability(Durability.SKIP_WAL);

2
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
View File

@ -546,7 +546,7 @@ public class TestFromClientSide {
throws IOException, InterruptedException {
byte [] name = Bytes.toBytes("testFilterAcrossMutlipleRegions");
HTable t = TEST_UTIL.createTable(name, FAMILY);
int rowCount = TEST_UTIL.loadTable(t, FAMILY);
int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
assertRowCount(t, rowCount);
// Split the table. Should split on a reasonable key; 'lqj'
Map<HRegionInfo, ServerName> regions = splitTable(t);

2
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
View File

@ -178,7 +178,7 @@ public class TestSnapshotFromClient {
// put some stuff in the table
HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
UTIL.loadTable(table, TEST_FAM);
UTIL.loadTable(table, TEST_FAM, false);
// get the name of all the regionservers hosting the snapshotted table
Set<String> snapshotServers = new HashSet<String>();

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
View File

@ -66,7 +66,7 @@ public class TestMultithreadedTableMapper {
UTIL.startMiniCluster();
HTable table = UTIL.createTable(MULTI_REGION_TABLE_NAME, new byte[][] {INPUT_FAMILY, OUTPUT_FAMILY});
UTIL.createMultiRegions(table, INPUT_FAMILY);
UTIL.loadTable(table, INPUT_FAMILY);
UTIL.loadTable(table, INPUT_FAMILY, false);
UTIL.startMiniMapReduceCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
View File

@ -79,7 +79,7 @@ public abstract class TestTableInputFormatScanBase {
// create and fill table
table = TEST_UTIL.createTable(TABLE_NAME, INPUT_FAMILY);
TEST_UTIL.createMultiRegions(table, INPUT_FAMILY);
TEST_UTIL.loadTable(table, INPUT_FAMILY);
TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
// start MR cluster
TEST_UTIL.startMiniMapReduceCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
View File

@ -77,7 +77,7 @@ public abstract class TestTableMapReduceBase {
HTable table =
UTIL.createTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY, OUTPUT_FAMILY });
UTIL.createMultiRegions(table, INPUT_FAMILY);
UTIL.loadTable(table, INPUT_FAMILY);
UTIL.loadTable(table, INPUT_FAMILY, false);
UTIL.startMiniMapReduceCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
View File

@ -68,7 +68,7 @@ public class TestMaster {
HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILYNAME);
assertTrue(m.assignmentManager.getZKTable().isEnabledTable(TABLENAME));
TEST_UTIL.loadTable(ht, FAMILYNAME);
TEST_UTIL.loadTable(ht, FAMILYNAME, false);
ht.close();
List<Pair<HRegionInfo, ServerName>> tableRegions =

4
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
View File

@ -146,7 +146,7 @@ public class TestCompaction {
for (int j = 0; j < jmax; j++) {
p.add(COLUMN_FAMILY, Bytes.toBytes(j), pad);
}
HBaseTestCase.addContent(loader, Bytes.toString(COLUMN_FAMILY), false);
HBaseTestCase.addContent(loader, Bytes.toString(COLUMN_FAMILY));
loader.put(p);
loader.flushcache();
}
@ -223,7 +223,7 @@ public class TestCompaction {
private void createStoreFile(final HRegion region, String family) throws IOException {
HRegionIncommon loader = new HRegionIncommon(region);
HBaseTestCase.addContent(loader, family, false);
HBaseTestCase.addContent(loader, family);
loader.flushcache();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
View File

@ -93,7 +93,7 @@ public class TestEndToEndSplitTransaction {
TableName.valueOf("TestSplit");
byte[] familyName = Bytes.toBytes("fam");
HTable ht = TEST_UTIL.createTable(tableName, familyName);
TEST_UTIL.loadTable(ht, familyName);
TEST_UTIL.loadTable(ht, familyName, false);
ht.close();
HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(0);
byte []firstRow = Bytes.toBytes("aaa");

2
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
View File

@ -202,7 +202,7 @@ public class TestFSErrorsExposed {
HTable table = new HTable(new Configuration(util.getConfiguration()), tableName);
// Load some data
util.loadTable(table, fam);
util.loadTable(table, fam, false);
table.flushCommits();
util.flush();
util.countRows(table);

6
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
View File

@ -199,7 +199,7 @@ public class TestMajorCompaction {
createStoreFile(r);
}
// Add more content.
HBaseTestCase.addContent(new HRegionIncommon(r), Bytes.toString(COLUMN_FAMILY), false);
HBaseTestCase.addContent(new HRegionIncommon(r), Bytes.toString(COLUMN_FAMILY));
// Now there are about 5 versions of each column.
// Default is that there only 3 (MAXVERSIONS) versions allowed per column.
@ -390,14 +390,14 @@ public class TestMajorCompaction {
private void createStoreFile(final HRegion region, String family) throws IOException {
HRegionIncommon loader = new HRegionIncommon(region);
HBaseTestCase.addContent(loader, family, false);
HBaseTestCase.addContent(loader, family);
loader.flushcache();
}
private void createSmallerStoreFile(final HRegion region) throws IOException {
HRegionIncommon loader = new HRegionIncommon(region);
HBaseTestCase.addContent(loader, Bytes.toString(COLUMN_FAMILY), ("" +
"bbb").getBytes(), null, false);
"bbb").getBytes(), null);
loader.flushcache();
}

8
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
View File

@ -164,13 +164,13 @@ public class TestMinorCompaction {
HRegionIncommon loader = new HRegionIncommon(r);
for (int i = 0; i < compactionThreshold + 1; i++) {
HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col1), firstRowBytes,
thirdRowBytes, i, false);
thirdRowBytes, i);
HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col2), firstRowBytes,
thirdRowBytes, i, false);
thirdRowBytes, i);
HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(col1), firstRowBytes,
thirdRowBytes, i, false);
thirdRowBytes, i);
HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(col2), firstRowBytes,
thirdRowBytes, i, false);
thirdRowBytes, i);
r.flushcache();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionFavoredNodes.java
View File

@ -130,7 +130,7 @@ public class TestRegionFavoredNodes {
// Write some data to each region and flush. Repeat some number of times to
// get multiple files for each region.
for (int i = 0; i < FLUSHES; i++) {
TEST_UTIL.loadTable(table, COLUMN_FAMILY);
TEST_UTIL.loadTable(table, COLUMN_FAMILY, false);
TEST_UTIL.flush();
}

8
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
View File

@ -389,7 +389,7 @@ public class TestSplitTransactionOnCluster {
cluster.getMaster().setCatalogJanitorEnabled(false);
try {
// Add a bit of load up into the table so splittable.
TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
// Get region pre-split.
HRegionServer server = cluster.getRegionServer(tableRegionIndex);
printOutRegions(server, "Initial regions: ");
@ -462,7 +462,7 @@ public class TestSplitTransactionOnCluster {
cluster.getMaster().setCatalogJanitorEnabled(false);
try {
// Add a bit of load up into the table so splittable.
TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
// Get region pre-split.
HRegionServer server = cluster.getRegionServer(tableRegionIndex);
printOutRegions(server, "Initial regions: ");
@ -691,7 +691,7 @@ public class TestSplitTransactionOnCluster {
"testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable());
try {
// Add a bit of load up into the table so splittable.
TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
// Get region pre-split.
HRegionServer server = cluster.getRegionServer(tableRegionIndex);
printOutRegions(server, "Initial regions: ");
@ -781,7 +781,7 @@ public class TestSplitTransactionOnCluster {
"testMasterRestartAtRegionSplitPendingCatalogJanitor", new UselessTestAbortable());
try {
// Add a bit of load up into the table so splittable.
TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
// Get region pre-split.
HRegionServer server = cluster.getRegionServer(tableRegionIndex);
printOutRegions(server, "Initial regions: ");