Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-12734 TestPerColumnFamilyFlush.testCompareStoreFileCount is flakey (zhangduo)

This commit is contained in:
stack 2014-12-22 08:56:33 -08:00
parent 5d664efe8b
commit 4d53fe5dcd
1 changed files with 26 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
View File

@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.hbase.regionserver; package org.apache.hadoop.hbase.regionserver;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
@ -40,22 +39,14 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.DefaultMemStore;
import org.apache.hadoop.hbase.regionserver.FlushAllStoresPolicy;
import org.apache.hadoop.hbase.regionserver.FlushLargeStoresPolicy;
import org.apache.hadoop.hbase.regionserver.FlushPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.wal.FSHLog; import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
@ -134,10 +125,8 @@ public class TestPerColumnFamilyFlush {
// Set up the configuration // Set up the configuration
Configuration conf = HBaseConfiguration.create(); Configuration conf = HBaseConfiguration.create();
conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 200 * 1024); conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 200 * 1024);
conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushLargeStoresPolicy.class.getName());
FlushLargeStoresPolicy.class.getName()); conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND, 100 * 1024);
conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND,
100 * 1024);
// Intialize the HRegion // Intialize the HRegion
initHRegion("testSelectiveFlushWhenEnabled", conf); initHRegion("testSelectiveFlushWhenEnabled", conf);
// Add 1200 entries for CF1, 100 for CF2 and 50 for CF3 // Add 1200 entries for CF1, 100 for CF2 and 50 for CF3
@ -342,8 +331,7 @@ public class TestPerColumnFamilyFlush {
Configuration conf = TEST_UTIL.getConfiguration(); Configuration conf = TEST_UTIL.getConfiguration();
conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 20000); conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 20000);
// Carefully chosen limits so that the memstore just flushes when we're done // Carefully chosen limits so that the memstore just flushes when we're done
conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushLargeStoresPolicy.class.getName());
FlushLargeStoresPolicy.class.getName());
conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND, 10000); conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND, 10000);
final int numRegionServers = 4; final int numRegionServers = 4;
TEST_UTIL.startMiniCluster(numRegionServers); TEST_UTIL.startMiniCluster(numRegionServers);
@ -440,8 +428,7 @@ public class TestPerColumnFamilyFlush {
public void testFlushingWhenLogRolling() throws Exception { public void testFlushingWhenLogRolling() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration(); Configuration conf = TEST_UTIL.getConfiguration();
conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 300000); conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 300000);
conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushLargeStoresPolicy.class.getName());
FlushLargeStoresPolicy.class.getName());
conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND, 100000); conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND, 100000);
// Also, let us try real hard to get a log roll to happen. // Also, let us try real hard to get a log roll to happen.
@ -503,7 +490,8 @@ public class TestPerColumnFamilyFlush {
TEST_UTIL.shutdownMiniCluster(); TEST_UTIL.shutdownMiniCluster();
} }
private void doPut(HTableInterface table) throws IOException { private void doPut(Table table, long memstoreFlushSize) throws IOException, InterruptedException {
HRegion region = getRegionWithName(table.getName()).getFirst();
// cf1 4B per row, cf2 40B per row and cf3 400B per row // cf1 4B per row, cf2 40B per row and cf3 400B per row
byte[] qf = Bytes.toBytes("qf"); byte[] qf = Bytes.toBytes("qf");
Random rand = new Random(); Random rand = new Random();
@ -520,6 +508,10 @@ public class TestPerColumnFamilyFlush {
put.add(FAMILY2, qf, value2); put.add(FAMILY2, qf, value2);
put.add(FAMILY3, qf, value3); put.add(FAMILY3, qf, value3);
table.put(put); table.put(put);
// slow down to let regionserver flush region.
while (region.getMemstoreSize().get() > memstoreFlushSize) {
Thread.sleep(100);
}
} }
} }
@ -527,11 +519,11 @@ public class TestPerColumnFamilyFlush {
// percolumnfamilyflush enabled. // percolumnfamilyflush enabled.
@Test @Test
public void testCompareStoreFileCount() throws Exception { public void testCompareStoreFileCount() throws Exception {
long memstoreFlushSize = 1024L * 1024;
Configuration conf = TEST_UTIL.getConfiguration(); Configuration conf = TEST_UTIL.getConfiguration();
conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024); conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, memstoreFlushSize);
conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushAllStoresPolicy.class.getName()); conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushAllStoresPolicy.class.getName());
conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND, conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND, 400 * 1024);
400 * 1024);
conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000); conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
ConstantSizeRegionSplitPolicy.class.getName()); ConstantSizeRegionSplitPolicy.class.getName());
@ -548,9 +540,9 @@ public class TestPerColumnFamilyFlush {
NamespaceDescriptor.create(TABLENAME.getNamespaceAsString()).build()); NamespaceDescriptor.create(TABLENAME.getNamespaceAsString()).build());
TEST_UTIL.getHBaseAdmin().createTable(htd); TEST_UTIL.getHBaseAdmin().createTable(htd);
getRegionWithName(TABLENAME).getFirst(); getRegionWithName(TABLENAME).getFirst();
HConnection conn = HConnectionManager.createConnection(conf); Connection conn = ConnectionFactory.createConnection(conf);
HTableInterface table = conn.getTable(TABLENAME); Table table = conn.getTable(TABLENAME);
doPut(table); doPut(table, memstoreFlushSize);
table.close(); table.close();
conn.close(); conn.close();
@ -561,15 +553,14 @@ public class TestPerColumnFamilyFlush {
TEST_UTIL.shutdownMiniCluster(); TEST_UTIL.shutdownMiniCluster();
LOG.info("==============Test with selective flush enabled==============="); LOG.info("==============Test with selective flush enabled===============");
conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushLargeStoresPolicy.class.getName());
FlushLargeStoresPolicy.class.getName());
TEST_UTIL.startMiniCluster(1); TEST_UTIL.startMiniCluster(1);
TEST_UTIL.getHBaseAdmin().createNamespace( TEST_UTIL.getHBaseAdmin().createNamespace(
NamespaceDescriptor.create(TABLENAME.getNamespaceAsString()).build()); NamespaceDescriptor.create(TABLENAME.getNamespaceAsString()).build());
TEST_UTIL.getHBaseAdmin().createTable(htd); TEST_UTIL.getHBaseAdmin().createTable(htd);
conn = HConnectionManager.createConnection(conf); conn = ConnectionFactory.createConnection(conf);
table = conn.getTable(TABLENAME); table = conn.getTable(TABLENAME);
doPut(table); doPut(table, memstoreFlushSize);
table.close(); table.close();
conn.close(); conn.close();
@ -602,8 +593,8 @@ public class TestPerColumnFamilyFlush {
htd.addFamily(new HColumnDescriptor(FAMILY3)); htd.addFamily(new HColumnDescriptor(FAMILY3));
Configuration conf = HBaseConfiguration.create(); Configuration conf = HBaseConfiguration.create();
HConnection conn = HConnectionManager.createConnection(conf); Connection conn = ConnectionFactory.createConnection(conf);
HBaseAdmin admin = new HBaseAdmin(conn); Admin admin = conn.getAdmin();
if (admin.tableExists(TABLENAME)) { if (admin.tableExists(TABLENAME)) {
admin.disableTable(TABLENAME); admin.disableTable(TABLENAME);
admin.deleteTable(TABLENAME); admin.deleteTable(TABLENAME);
@ -618,7 +609,7 @@ public class TestPerColumnFamilyFlush {
} }
admin.close(); admin.close();
HTableInterface table = conn.getTable(TABLENAME); Table table = conn.getTable(TABLENAME);
byte[] qf = Bytes.toBytes("qf"); byte[] qf = Bytes.toBytes("qf");
Random rand = new Random(); Random rand = new Random();
byte[] value1 = new byte[16]; byte[] value1 = new byte[16];