Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-14781 Turn per cf flushing on for ITBLL by default

Browse Source
This commit is contained in:
Elliott Clark 2015-11-06 12:11:47 -08:00
parent 61e2566c1c
commit 17fd03ea8d
1 changed files with 7 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
View File

@ -405,8 +405,7 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase {
if (this.numNodes < this.wrap) { if (this.numNodes < this.wrap) {
this.wrap = this.numNodes; this.wrap = this.numNodes;
} }
this.multipleUnevenColumnFamilies = this.multipleUnevenColumnFamilies = isMultiUnevenColumnFamilies(context.getConfiguration());
context.getConfiguration().getBoolean(MULTIPLE_UNEVEN_COLUMNFAMILIES_KEY, false);
} }
protected void instantiateHTable() throws IOException { protected void instantiateHTable() throws IOException {
@ -605,11 +604,6 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase {
job.setOutputFormatClass(NullOutputFormat.class); job.setOutputFormatClass(NullOutputFormat.class);
job.getConfiguration().setBoolean("mapreduce.map.speculative", false); job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
String multipleUnevenColumnFamiliesStr = System.getProperty(MULTIPLE_UNEVEN_COLUMNFAMILIES_KEY);
if (multipleUnevenColumnFamiliesStr != null) {
job.getConfiguration().setBoolean(MULTIPLE_UNEVEN_COLUMNFAMILIES_KEY,
Boolean.parseBoolean(multipleUnevenColumnFamiliesStr));
}
TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.addDependencyJars(job);
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
TableMapReduceUtil.initCredentials(job); TableMapReduceUtil.initCredentials(job);
@ -820,9 +814,7 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase {
protected void setup( protected void setup(
Mapper<ImmutableBytesWritable, Result, BytesWritable, BytesWritable>.Context context) Mapper<ImmutableBytesWritable, Result, BytesWritable, BytesWritable>.Context context)
throws IOException, InterruptedException { throws IOException, InterruptedException {
this.multipleUnevenColumnFamilies = this.multipleUnevenColumnFamilies = isMultiUnevenColumnFamilies(context.getConfiguration());
context.getConfiguration().getBoolean(Generator.MULTIPLE_UNEVEN_COLUMNFAMILIES_KEY,
false);
} }
@Override @Override
@ -1091,10 +1083,9 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase {
scan.addColumn(FAMILY_NAME, COLUMN_PREV); scan.addColumn(FAMILY_NAME, COLUMN_PREV);
scan.setCaching(10000); scan.setCaching(10000);
scan.setCacheBlocks(false); scan.setCacheBlocks(false);
if (isMultiUnevenColumnFamilies()) { if (isMultiUnevenColumnFamilies(getConf())) {
scan.addColumn(BIG_FAMILY_NAME, BIG_FAMILY_NAME); scan.addColumn(BIG_FAMILY_NAME, BIG_FAMILY_NAME);
scan.addColumn(TINY_FAMILY_NAME, TINY_FAMILY_NAME); scan.addColumn(TINY_FAMILY_NAME, TINY_FAMILY_NAME);
job.getConfiguration().setBoolean(Generator.MULTIPLE_UNEVEN_COLUMNFAMILIES_KEY, true);
} }
TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan, TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan,
@ -1571,16 +1562,15 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase {
} }
} }
private static boolean isMultiUnevenColumnFamilies() { private static boolean isMultiUnevenColumnFamilies(Configuration conf) {
return Boolean.TRUE.toString().equalsIgnoreCase( return conf.getBoolean(Generator.MULTIPLE_UNEVEN_COLUMNFAMILIES_KEY,true);
System.getProperty(Generator.MULTIPLE_UNEVEN_COLUMNFAMILIES_KEY));
} }
@Test @Test
public void testContinuousIngest() throws IOException, Exception { public void testContinuousIngest() throws IOException, Exception {
//Loop <num iterations> <num mappers> <num nodes per mapper> <output dir> <num reducers> //Loop <num iterations> <num mappers> <num nodes per mapper> <output dir> <num reducers>
Configuration conf = getTestingUtil(getConf()).getConfiguration(); Configuration conf = getTestingUtil(getConf()).getConfiguration();
if (isMultiUnevenColumnFamilies()) { if (isMultiUnevenColumnFamilies(getConf())) {
// make sure per CF flush is on // make sure per CF flush is on
conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushLargeStoresPolicy.class.getName()); conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushLargeStoresPolicy.class.getName());
} }
@ -1665,7 +1655,7 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase {
@Override @Override
protected Set<String> getColumnFamilies() { protected Set<String> getColumnFamilies() {
if (isMultiUnevenColumnFamilies()) { if (isMultiUnevenColumnFamilies(getConf())) {
return Sets.newHashSet(Bytes.toString(FAMILY_NAME), Bytes.toString(BIG_FAMILY_NAME), return Sets.newHashSet(Bytes.toString(FAMILY_NAME), Bytes.toString(BIG_FAMILY_NAME),
Bytes.toString(TINY_FAMILY_NAME)); Bytes.toString(TINY_FAMILY_NAME));
} else { } else {