HBASE-15949 Cleanup TestRegionServerMetrics.
@Before and @After to setup/teardown tables using @Rule to set table name based on testname. Refactor out copy-pasted code fragments to single function. (Apekshit) Change-Id: Ic22e5027cc3952bab5ec30070ed20e98017db65a
This commit is contained in:
Apekshit
Apekshit Sharma
parent
bdb46f01b9
commit
b557f0bec6
|
|
@ -17,6 +17,8 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase.regionserver;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.*;
|
||||
import org.apache.hadoop.hbase.client.*;
|
||||
|
|
@ -27,11 +29,17 @@ import org.apache.hadoop.hbase.util.Bytes;
|
|||
import org.apache.hadoop.hbase.util.Threads;
|
||||
import org.apache.log4j.Level;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.junit.rules.TestName;
|
||||
import org.junit.rules.TestRule;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
|
|
@ -39,23 +47,35 @@ import java.io.IOException;
|
|||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
@Category({RegionServerTests.class, MediumTests.class})
|
||||
public class TestRegionServerMetrics {
|
||||
private static MetricsAssertHelper metricsHelper;
|
||||
private static final Log LOG = LogFactory.getLog(TestRegionServerMetrics.class);
|
||||
|
||||
@Rule
|
||||
public TestName testName = new TestName();
|
||||
|
||||
@ClassRule
|
||||
public static TestRule timeout = CategoryBasedTimeout.forClass(TestRegionServerMetrics.class);
|
||||
|
||||
static {
|
||||
Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
|
||||
}
|
||||
|
||||
private static MetricsAssertHelper metricsHelper;
|
||||
private static MiniHBaseCluster cluster;
|
||||
private static HRegionServer rs;
|
||||
private static Configuration conf;
|
||||
private static HBaseTestingUtility TEST_UTIL;
|
||||
private static Connection connection;
|
||||
private static MetricsRegionServer metricsRegionServer;
|
||||
private static MetricsRegionServerSource serverSource;
|
||||
private static final int NUM_SCAN_NEXT = 30;
|
||||
private static int numScanNext = 0;
|
||||
private static byte[] cf = Bytes.toBytes("cf");
|
||||
private static byte[] row = Bytes.toBytes("row");
|
||||
private static byte[] qualifier = Bytes.toBytes("qual");
|
||||
private static byte[] val = Bytes.toBytes("val");
|
||||
private static Admin admin;
|
||||
|
||||
@BeforeClass
|
||||
public static void startCluster() throws Exception {
|
||||
|
|
@ -65,12 +85,16 @@ public class TestRegionServerMetrics {
|
|||
conf.getLong("hbase.splitlog.max.resubmit", 0);
|
||||
// Make the failure test faster
|
||||
conf.setInt("zookeeper.recovery.retry", 0);
|
||||
// testMobMetrics creates few hfiles and manages compaction manually.
|
||||
conf.setInt("hbase.hstore.compactionThreshold", 100);
|
||||
conf.setInt("hbase.hstore.compaction.max", 100);
|
||||
conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
|
||||
|
||||
TEST_UTIL.startMiniCluster(1, 1);
|
||||
cluster = TEST_UTIL.getHBaseCluster();
|
||||
|
||||
cluster.waitForActiveAndReadyMaster();
|
||||
admin = TEST_UTIL.getHBaseAdmin();
|
||||
connection = TEST_UTIL.getConnection();
|
||||
|
||||
while (cluster.getLiveRegionServerThreads().size() < 1) {
|
||||
Threads.sleep(100);
|
||||
|
|
@ -88,551 +112,370 @@ public class TestRegionServerMetrics {
|
|||
}
|
||||
}
|
||||
|
||||
@Test(timeout = 300000)
|
||||
TableName tableName;
|
||||
Table table;
|
||||
|
||||
@Before
|
||||
public void beforeTestMethod() throws Exception {
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
tableName = TableName.valueOf(testName.getMethodName());
|
||||
table = TEST_UTIL.createTable(tableName, cf);
|
||||
}
|
||||
|
||||
@After
|
||||
public void afterTestMethod() throws Exception {
|
||||
admin.disableTable(tableName);
|
||||
admin.deleteTable(tableName);
|
||||
}
|
||||
|
||||
public void waitTableDeleted(TableName name, long timeoutInMillis) throws Exception {
|
||||
long start = System.currentTimeMillis();
|
||||
while (true) {
|
||||
HTableDescriptor[] tables = admin.listTables();
|
||||
for (HTableDescriptor htd : tables) {
|
||||
if (htd.getNameAsString() == name.getNameAsString())
|
||||
return;
|
||||
}
|
||||
if (System.currentTimeMillis() - start > timeoutInMillis)
|
||||
return;
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
}
|
||||
|
||||
public void assertCounter(String metric, long expectedValue) {
|
||||
metricsHelper.assertCounter(metric, expectedValue, serverSource);
|
||||
}
|
||||
|
||||
public void assertGauge(String metric, long expectedValue) {
|
||||
metricsHelper.assertGauge(metric, expectedValue, serverSource);
|
||||
}
|
||||
|
||||
// Aggregates metrics from regions and assert given list of metrics and expected values.
|
||||
public void assertRegionMetrics(String metric, long expectedValue) throws Exception {
|
||||
try (RegionLocator locator = connection.getRegionLocator(tableName)) {
|
||||
for ( HRegionLocation location: locator.getAllRegionLocations()) {
|
||||
HRegionInfo hri = location.getRegionInfo();
|
||||
MetricsRegionAggregateSource agg =
|
||||
rs.getRegion(hri.getRegionName()).getMetrics().getSource().getAggregateSource();
|
||||
String prefix = "namespace_" + NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR +
|
||||
"_table_" + tableName.getNameAsString() +
|
||||
"_region_" + hri.getEncodedName()+
|
||||
"_metric_";
|
||||
metricsHelper.assertCounter(prefix + metric, expectedValue, agg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void doNPuts(int n, boolean batch) throws Exception {
|
||||
if (batch) {
|
||||
List<Put> puts = new ArrayList<>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
Put p = new Put(Bytes.toBytes("" + i + "row")).addColumn(cf, qualifier, val);
|
||||
puts.add(p);
|
||||
}
|
||||
table.put(puts);
|
||||
} else {
|
||||
for (int i = 0; i < n; i++) {
|
||||
Put p = new Put(row).addColumn(cf, qualifier, val);
|
||||
table.put(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void doNGets(int n, boolean batch) throws Exception {
|
||||
if (batch) {
|
||||
List<Get> gets = new ArrayList<>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
gets.add(new Get(row));
|
||||
}
|
||||
table.get(gets);
|
||||
} else {
|
||||
for (int i = 0; i < n; i++) {
|
||||
table.get(new Get(row));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRegionCount() throws Exception {
|
||||
String regionMetricsKey = "regionCount";
|
||||
long regions = metricsHelper.getGaugeLong(regionMetricsKey, serverSource);
|
||||
// Creating a table should add one region
|
||||
TEST_UTIL.createTable(TableName.valueOf("table"), Bytes.toBytes("cf"));
|
||||
metricsHelper.assertGaugeGt(regionMetricsKey, regions, serverSource);
|
||||
metricsHelper.assertGauge("regionCount", 1, serverSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLocalFiles() throws Exception {
|
||||
metricsHelper.assertGauge("percentFilesLocal", 0, serverSource);
|
||||
metricsHelper.assertGauge("percentFilesLocalSecondaryRegions", 0, serverSource);
|
||||
assertGauge("percentFilesLocal", 0);
|
||||
assertGauge("percentFilesLocalSecondaryRegions", 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRequestCount() throws Exception {
|
||||
String tableNameString = "testRequestCount";
|
||||
TableName tName = TableName.valueOf(tableNameString);
|
||||
byte[] cfName = Bytes.toBytes("d");
|
||||
byte[] row = Bytes.toBytes("rk");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] initValue = Bytes.toBytes("Value");
|
||||
|
||||
TEST_UTIL.createTable(tName, cfName);
|
||||
|
||||
Connection connection = TEST_UTIL.getConnection();
|
||||
connection.getTable(tName).close(); //wait for the table to come up.
|
||||
|
||||
// Do a first put to be sure that the connection is established, meta is there and so on.
|
||||
Table table = connection.getTable(tName);
|
||||
Put p = new Put(row);
|
||||
p.addColumn(cfName, qualifier, initValue);
|
||||
table.put(p);
|
||||
doNPuts(1, false);
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
|
||||
long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
|
||||
long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
|
||||
|
||||
for (int i=0; i< 30; i++) {
|
||||
table.put(p);
|
||||
}
|
||||
doNPuts(30, false);
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource);
|
||||
metricsHelper.assertCounter("readRequestCount", readRequests, serverSource);
|
||||
metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
|
||||
assertCounter("totalRequestCount", requests + 30);
|
||||
assertCounter("readRequestCount", readRequests);
|
||||
assertCounter("writeRequestCount", writeRequests + 30);
|
||||
|
||||
Get g = new Get(row);
|
||||
for (int i=0; i< 10; i++) {
|
||||
table.get(g);
|
||||
}
|
||||
doNGets(10, false);
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource);
|
||||
metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource);
|
||||
metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
|
||||
assertCounter("totalRequestCount", requests + 40);
|
||||
assertCounter("readRequestCount", readRequests + 10);
|
||||
assertCounter("writeRequestCount", writeRequests + 30);
|
||||
|
||||
try (RegionLocator locator = connection.getRegionLocator(tName)) {
|
||||
for ( HRegionLocation location: locator.getAllRegionLocations()) {
|
||||
HRegionInfo i = location.getRegionInfo();
|
||||
MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
|
||||
.getMetrics()
|
||||
.getSource()
|
||||
.getAggregateSource();
|
||||
String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
|
||||
"_table_"+tableNameString +
|
||||
"_region_" + i.getEncodedName()+
|
||||
"_metric";
|
||||
metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg);
|
||||
metricsHelper.assertCounter(prefix + "_mutateCount", 31, agg);
|
||||
}
|
||||
}
|
||||
List<Get> gets = new ArrayList<Get>();
|
||||
for (int i=0; i< 10; i++) {
|
||||
gets.add(new Get(row));
|
||||
}
|
||||
table.get(gets);
|
||||
assertRegionMetrics("getNumOps", 10);
|
||||
assertRegionMetrics("mutateCount", 31);
|
||||
|
||||
doNGets(10, true); // true = batch
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
metricsHelper.assertCounter("totalRequestCount", requests + 50, serverSource);
|
||||
metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource);
|
||||
metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
|
||||
assertCounter("totalRequestCount", requests + 50);
|
||||
assertCounter("readRequestCount", readRequests + 20);
|
||||
assertCounter("writeRequestCount", writeRequests + 30);
|
||||
|
||||
List<Put> puts = new ArrayList<>();
|
||||
for (int i=0; i< 30; i++) {
|
||||
puts.add(p);
|
||||
}
|
||||
table.put(puts);
|
||||
doNPuts(30, true);
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
metricsHelper.assertCounter("totalRequestCount", requests + 80, serverSource);
|
||||
metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource);
|
||||
metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
|
||||
|
||||
table.close();
|
||||
assertCounter("totalRequestCount", requests + 80);
|
||||
assertCounter("readRequestCount", readRequests + 20);
|
||||
assertCounter("writeRequestCount", writeRequests + 60);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGet() throws Exception {
|
||||
String tableNameString = "testGet";
|
||||
TableName tName = TableName.valueOf(tableNameString);
|
||||
byte[] cfName = Bytes.toBytes("d");
|
||||
byte[] row = Bytes.toBytes("rk");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] initValue = Bytes.toBytes("Value");
|
||||
|
||||
TEST_UTIL.createTable(tName, cfName);
|
||||
|
||||
Connection connection = TEST_UTIL.getConnection();
|
||||
connection.getTable(tName).close(); //wait for the table to come up.
|
||||
|
||||
// Do a first put to be sure that the connection is established, meta is there and so on.
|
||||
Table table = connection.getTable(tName);
|
||||
Put p = new Put(row);
|
||||
p.addColumn(cfName, qualifier, initValue);
|
||||
table.put(p);
|
||||
|
||||
Get g = new Get(row);
|
||||
for (int i=0; i< 10; i++) {
|
||||
table.get(g);
|
||||
}
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
|
||||
try (RegionLocator locator = connection.getRegionLocator(tName)) {
|
||||
for ( HRegionLocation location: locator.getAllRegionLocations()) {
|
||||
HRegionInfo i = location.getRegionInfo();
|
||||
MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
|
||||
.getMetrics()
|
||||
.getSource()
|
||||
.getAggregateSource();
|
||||
String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
|
||||
"_table_"+tableNameString +
|
||||
"_region_" + i.getEncodedName()+
|
||||
"_metric";
|
||||
metricsHelper.assertCounter(prefix + "_getSizeNumOps", 10, agg);
|
||||
metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg);
|
||||
}
|
||||
metricsHelper.assertCounterGt("Get_num_ops", 10, serverSource);
|
||||
}
|
||||
table.close();
|
||||
doNPuts(1, false);
|
||||
doNGets(10, false);
|
||||
assertRegionMetrics("getNumOps", 10);
|
||||
assertRegionMetrics("getSizeNumOps", 10);
|
||||
metricsHelper.assertCounterGt("Get_num_ops", 10, serverSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMutationsWithoutWal() throws Exception {
|
||||
TableName tableName = TableName.valueOf("testMutationsWithoutWal");
|
||||
byte[] cf = Bytes.toBytes("d");
|
||||
byte[] row = Bytes.toBytes("rk");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] val = Bytes.toBytes("Value");
|
||||
Put p = new Put(row).addColumn(cf, qualifier, val)
|
||||
.setDurability(Durability.SKIP_WAL);
|
||||
table.put(p);
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
|
||||
Table t = TEST_UTIL.createTable(tableName, cf);
|
||||
|
||||
Put p = new Put(row);
|
||||
p.addColumn(cf, qualifier, val);
|
||||
p.setDurability(Durability.SKIP_WAL);
|
||||
|
||||
t.put(p);
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource);
|
||||
assertGauge("mutationsWithoutWALCount", 1);
|
||||
long minLength = row.length + cf.length + qualifier.length + val.length;
|
||||
metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
|
||||
|
||||
t.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStoreCount() throws Exception {
|
||||
TableName tableName = TableName.valueOf("testStoreCount");
|
||||
byte[] cf = Bytes.toBytes("d");
|
||||
byte[] row = Bytes.toBytes("rk");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] val = Bytes.toBytes("Value");
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
long stores = metricsHelper.getGaugeLong("storeCount", serverSource);
|
||||
long storeFiles = metricsHelper.getGaugeLong("storeFileCount", serverSource);
|
||||
|
||||
//Force a hfile.
|
||||
Table t = TEST_UTIL.createTable(tableName, cf);
|
||||
Put p = new Put(row);
|
||||
p.addColumn(cf, qualifier, val);
|
||||
t.put(p);
|
||||
doNPuts(1, false);
|
||||
TEST_UTIL.getHBaseAdmin().flush(tableName);
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
metricsHelper.assertGauge("storeCount", stores +1, serverSource);
|
||||
metricsHelper.assertGauge("storeFileCount", storeFiles + 1, serverSource);
|
||||
|
||||
t.close();
|
||||
assertGauge("storeCount", 1);
|
||||
assertGauge("storeFileCount", 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStoreFileAge() throws Exception {
|
||||
TableName tableName = TableName.valueOf("testStoreFileAge");
|
||||
byte[] cf = Bytes.toBytes("d");
|
||||
byte[] row = Bytes.toBytes("rk");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] val = Bytes.toBytes("Value");
|
||||
|
||||
//Force a hfile.
|
||||
Table t = TEST_UTIL.createTable(tableName, cf);
|
||||
Put p = new Put(row);
|
||||
p.addColumn(cf, qualifier, val);
|
||||
t.put(p);
|
||||
doNPuts(1, false);
|
||||
TEST_UTIL.getHBaseAdmin().flush(tableName);
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
assertTrue(metricsHelper.getGaugeLong("maxStoreFileAge", serverSource) > 0);
|
||||
assertTrue(metricsHelper.getGaugeLong("minStoreFileAge", serverSource) > 0);
|
||||
assertTrue(metricsHelper.getGaugeLong("avgStoreFileAge", serverSource) > 0);
|
||||
|
||||
t.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCheckAndPutCount() throws Exception {
|
||||
String tableNameString = "testCheckAndPutCount";
|
||||
TableName tableName = TableName.valueOf(tableNameString);
|
||||
byte[] cf = Bytes.toBytes("d");
|
||||
byte[] row = Bytes.toBytes("rk");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] valOne = Bytes.toBytes("Value");
|
||||
byte[] valTwo = Bytes.toBytes("ValueTwo");
|
||||
byte[] valThree = Bytes.toBytes("ValueThree");
|
||||
|
||||
Table t = TEST_UTIL.createTable(tableName, cf);
|
||||
Put p = new Put(row);
|
||||
p.addColumn(cf, qualifier, valOne);
|
||||
t.put(p);
|
||||
table.put(p);
|
||||
|
||||
Put pTwo = new Put(row);
|
||||
pTwo.addColumn(cf, qualifier, valTwo);
|
||||
t.checkAndPut(row, cf, qualifier, valOne, pTwo);
|
||||
table.checkAndPut(row, cf, qualifier, valOne, pTwo);
|
||||
|
||||
Put pThree = new Put(row);
|
||||
pThree.addColumn(cf, qualifier, valThree);
|
||||
t.checkAndPut(row, cf, qualifier, valOne, pThree);
|
||||
table.checkAndPut(row, cf, qualifier, valOne, pThree);
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
metricsHelper.assertCounter("checkMutateFailedCount", 1, serverSource);
|
||||
metricsHelper.assertCounter("checkMutatePassedCount", 1, serverSource);
|
||||
|
||||
t.close();
|
||||
assertCounter("checkMutateFailedCount", 1);
|
||||
assertCounter("checkMutatePassedCount", 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIncrement() throws Exception {
|
||||
String tableNameString = "testIncrement";
|
||||
TableName tableName = TableName.valueOf(tableNameString);
|
||||
byte[] cf = Bytes.toBytes("d");
|
||||
byte[] row = Bytes.toBytes("rk");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] val = Bytes.toBytes(0l);
|
||||
Put p = new Put(row).addColumn(cf, qualifier, Bytes.toBytes(0L));
|
||||
table.put(p);
|
||||
|
||||
|
||||
Table t = TEST_UTIL.createTable(tableName, cf);
|
||||
Put p = new Put(row);
|
||||
p.addColumn(cf, qualifier, val);
|
||||
t.put(p);
|
||||
|
||||
for(int count = 0; count< 13; count++) {
|
||||
for(int count = 0; count < 13; count++) {
|
||||
Increment inc = new Increment(row);
|
||||
inc.addColumn(cf, qualifier, 100);
|
||||
t.increment(inc);
|
||||
table.increment(inc);
|
||||
}
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
metricsHelper.assertCounter("incrementNumOps", 13, serverSource);
|
||||
|
||||
t.close();
|
||||
assertCounter("incrementNumOps", 13);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppend() throws Exception {
|
||||
String tableNameString = "testAppend";
|
||||
TableName tableName = TableName.valueOf(tableNameString);
|
||||
byte[] cf = Bytes.toBytes("d");
|
||||
byte[] row = Bytes.toBytes("rk");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] val = Bytes.toBytes("One");
|
||||
|
||||
|
||||
Table t = TEST_UTIL.createTable(tableName, cf);
|
||||
Put p = new Put(row);
|
||||
p.addColumn(cf, qualifier, val);
|
||||
t.put(p);
|
||||
doNPuts(1, false);
|
||||
|
||||
for(int count = 0; count< 73; count++) {
|
||||
Append append = new Append(row);
|
||||
append.add(cf, qualifier, Bytes.toBytes(",Test"));
|
||||
t.append(append);
|
||||
table.append(append);
|
||||
}
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
metricsHelper.assertCounter("appendNumOps", 73, serverSource);
|
||||
|
||||
t.close();
|
||||
assertCounter("appendNumOps", 73);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testScanSize() throws IOException {
|
||||
String tableNameString = "testScanSize";
|
||||
TableName tableName = TableName.valueOf(tableNameString);
|
||||
byte[] cf = Bytes.toBytes("d");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] val = Bytes.toBytes("One");
|
||||
public void testScanSize() throws Exception {
|
||||
doNPuts(100, true); // batch put
|
||||
Scan s = new Scan();
|
||||
s.setBatch(1);
|
||||
s.setCaching(1);
|
||||
ResultScanner resultScanners = table.getScanner(s);
|
||||
|
||||
List<Put> puts = new ArrayList<>();
|
||||
for (int insertCount =0; insertCount < 100; insertCount++) {
|
||||
Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
|
||||
p.addColumn(cf, qualifier, val);
|
||||
puts.add(p);
|
||||
}
|
||||
try (Table t = TEST_UTIL.createTable(tableName, cf)) {
|
||||
t.put(puts);
|
||||
|
||||
Scan s = new Scan();
|
||||
s.setBatch(1);
|
||||
s.setCaching(1);
|
||||
ResultScanner resultScanners = t.getScanner(s);
|
||||
|
||||
for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
|
||||
Result result = resultScanners.next();
|
||||
assertNotNull(result);
|
||||
assertEquals(1, result.size());
|
||||
}
|
||||
for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
|
||||
Result result = resultScanners.next();
|
||||
assertNotNull(result);
|
||||
assertEquals(1, result.size());
|
||||
}
|
||||
numScanNext += NUM_SCAN_NEXT;
|
||||
try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
|
||||
for ( HRegionLocation location: locator.getAllRegionLocations()) {
|
||||
HRegionInfo i = location.getRegionInfo();
|
||||
MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
|
||||
.getMetrics()
|
||||
.getSource()
|
||||
.getAggregateSource();
|
||||
String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
|
||||
"_table_"+tableNameString +
|
||||
"_region_" + i.getEncodedName()+
|
||||
"_metric";
|
||||
metricsHelper.assertCounter(prefix + "_scanSizeNumOps", NUM_SCAN_NEXT, agg);
|
||||
}
|
||||
metricsHelper.assertCounter("ScanSize_num_ops", numScanNext, serverSource);
|
||||
}
|
||||
try (Admin admin = TEST_UTIL.getHBaseAdmin()) {
|
||||
admin.disableTable(tableName);
|
||||
admin.deleteTable(tableName);
|
||||
}
|
||||
assertRegionMetrics("scanSizeNumOps", NUM_SCAN_NEXT);
|
||||
assertCounter("ScanSize_num_ops", numScanNext);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testScanTime() throws IOException {
|
||||
String tableNameString = "testScanTime";
|
||||
TableName tableName = TableName.valueOf(tableNameString);
|
||||
byte[] cf = Bytes.toBytes("d");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] val = Bytes.toBytes("One");
|
||||
public void testScanTime() throws Exception {
|
||||
doNPuts(100, true);
|
||||
Scan s = new Scan();
|
||||
s.setBatch(1);
|
||||
s.setCaching(1);
|
||||
ResultScanner resultScanners = table.getScanner(s);
|
||||
|
||||
List<Put> puts = new ArrayList<>();
|
||||
for (int insertCount =0; insertCount < 100; insertCount++) {
|
||||
Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
|
||||
p.addColumn(cf, qualifier, val);
|
||||
puts.add(p);
|
||||
}
|
||||
try (Table t = TEST_UTIL.createTable(tableName, cf)) {
|
||||
t.put(puts);
|
||||
|
||||
Scan s = new Scan();
|
||||
s.setBatch(1);
|
||||
s.setCaching(1);
|
||||
ResultScanner resultScanners = t.getScanner(s);
|
||||
|
||||
for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
|
||||
Result result = resultScanners.next();
|
||||
assertNotNull(result);
|
||||
assertEquals(1, result.size());
|
||||
}
|
||||
for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
|
||||
Result result = resultScanners.next();
|
||||
assertNotNull(result);
|
||||
assertEquals(1, result.size());
|
||||
}
|
||||
numScanNext += NUM_SCAN_NEXT;
|
||||
try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
|
||||
for ( HRegionLocation location: locator.getAllRegionLocations()) {
|
||||
HRegionInfo i = location.getRegionInfo();
|
||||
MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
|
||||
.getMetrics()
|
||||
.getSource()
|
||||
.getAggregateSource();
|
||||
String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
|
||||
"_table_"+tableNameString +
|
||||
"_region_" + i.getEncodedName()+
|
||||
"_metric";
|
||||
metricsHelper.assertCounter(prefix + "_scanTimeNumOps", NUM_SCAN_NEXT, agg);
|
||||
}
|
||||
metricsHelper.assertCounter("ScanTime_num_ops", numScanNext, serverSource);
|
||||
}
|
||||
try (Admin admin = TEST_UTIL.getHBaseAdmin()) {
|
||||
admin.disableTable(tableName);
|
||||
admin.deleteTable(tableName);
|
||||
}
|
||||
assertRegionMetrics("scanTimeNumOps", NUM_SCAN_NEXT);
|
||||
assertCounter("ScanTime_num_ops", numScanNext);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testScanSizeForSmallScan() throws IOException {
|
||||
String tableNameString = "testScanSizeSmall";
|
||||
TableName tableName = TableName.valueOf(tableNameString);
|
||||
byte[] cf = Bytes.toBytes("d");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] val = Bytes.toBytes("One");
|
||||
public void testScanSizeForSmallScan() throws Exception {
|
||||
doNPuts(100, true);
|
||||
Scan s = new Scan();
|
||||
s.setSmall(true);
|
||||
s.setCaching(1);
|
||||
ResultScanner resultScanners = table.getScanner(s);
|
||||
|
||||
List<Put> puts = new ArrayList<>();
|
||||
for (int insertCount =0; insertCount < 100; insertCount++) {
|
||||
Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
|
||||
p.addColumn(cf, qualifier, val);
|
||||
puts.add(p);
|
||||
}
|
||||
try (Table t = TEST_UTIL.createTable(tableName, cf)) {
|
||||
t.put(puts);
|
||||
|
||||
Scan s = new Scan();
|
||||
s.setSmall(true);
|
||||
s.setCaching(1);
|
||||
ResultScanner resultScanners = t.getScanner(s);
|
||||
|
||||
for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
|
||||
Result result = resultScanners.next();
|
||||
assertNotNull(result);
|
||||
assertEquals(1, result.size());
|
||||
}
|
||||
for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
|
||||
Result result = resultScanners.next();
|
||||
assertNotNull(result);
|
||||
assertEquals(1, result.size());
|
||||
}
|
||||
numScanNext += NUM_SCAN_NEXT;
|
||||
try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
|
||||
for ( HRegionLocation location: locator.getAllRegionLocations()) {
|
||||
HRegionInfo i = location.getRegionInfo();
|
||||
MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
|
||||
.getMetrics()
|
||||
.getSource()
|
||||
.getAggregateSource();
|
||||
String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
|
||||
"_table_"+tableNameString +
|
||||
"_region_" + i.getEncodedName()+
|
||||
"_metric";
|
||||
metricsHelper.assertCounter(prefix + "_scanSizeNumOps", NUM_SCAN_NEXT, agg);
|
||||
}
|
||||
metricsHelper.assertCounter("ScanSize_num_ops", numScanNext, serverSource);
|
||||
}
|
||||
try (Admin admin = TEST_UTIL.getHBaseAdmin()) {
|
||||
admin.disableTable(tableName);
|
||||
admin.deleteTable(tableName);
|
||||
}
|
||||
assertRegionMetrics("scanSizeNumOps", NUM_SCAN_NEXT);
|
||||
assertCounter("ScanSize_num_ops", numScanNext);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMobMetrics() throws IOException, InterruptedException {
|
||||
String tableNameString = "testMobMetrics";
|
||||
TableName tableName = TableName.valueOf(tableNameString);
|
||||
byte[] cf = Bytes.toBytes("d");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] val = Bytes.toBytes("mobdata");
|
||||
int numHfiles = conf.getInt("hbase.hstore.compactionThreshold", 3) - 1;
|
||||
TableName tableName = TableName.valueOf("testMobMetricsLocal");
|
||||
int numHfiles = 5;
|
||||
HTableDescriptor htd = new HTableDescriptor(tableName);
|
||||
HColumnDescriptor hcd = new HColumnDescriptor(cf);
|
||||
hcd.setMobEnabled(true);
|
||||
hcd.setMobThreshold(0);
|
||||
htd.addFamily(hcd);
|
||||
Connection connection = ConnectionFactory.createConnection(conf);
|
||||
Admin admin = connection.getAdmin();
|
||||
Table t = TEST_UTIL.createTable(htd, new byte[0][0], conf);
|
||||
Region region = rs.getOnlineRegions(tableName).get(0);
|
||||
for (int insertCount = 0; insertCount < numHfiles; insertCount++) {
|
||||
Put p = new Put(Bytes.toBytes(insertCount));
|
||||
p.addColumn(cf, qualifier, val);
|
||||
t.put(p);
|
||||
admin.flush(tableName);
|
||||
byte[] val = Bytes.toBytes("mobdata");
|
||||
try {
|
||||
Table table = TEST_UTIL.createTable(htd, new byte[0][0], conf);
|
||||
Region region = rs.getOnlineRegions(tableName).get(0);
|
||||
for (int insertCount = 0; insertCount < numHfiles; insertCount++) {
|
||||
Put p = new Put(Bytes.toBytes(insertCount));
|
||||
p.addColumn(cf, qualifier, val);
|
||||
table.put(p);
|
||||
admin.flush(tableName);
|
||||
}
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
assertCounter("mobFlushCount", numHfiles);
|
||||
|
||||
Scan scan = new Scan(Bytes.toBytes(0), Bytes.toBytes(numHfiles));
|
||||
ResultScanner scanner = table.getScanner(scan);
|
||||
scanner.next(100);
|
||||
numScanNext++; // this is an ugly construct
|
||||
scanner.close();
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
assertCounter("mobScanCellsCount", numHfiles);
|
||||
|
||||
region.getTableDesc().getFamily(cf).setMobThreshold(100);
|
||||
// metrics are reset by the region initialization
|
||||
((HRegion) region).initialize();
|
||||
region.compact(true);
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
assertCounter("cellsCountCompactedFromMob", numHfiles);
|
||||
assertCounter("cellsCountCompactedToMob", 0);
|
||||
|
||||
scanner = table.getScanner(scan);
|
||||
scanner.next(100);
|
||||
numScanNext++; // this is an ugly construct
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
assertCounter("mobScanCellsCount", 0);
|
||||
|
||||
for (int insertCount = numHfiles; insertCount < 2 * numHfiles; insertCount++) {
|
||||
Put p = new Put(Bytes.toBytes(insertCount));
|
||||
p.addColumn(cf, qualifier, val);
|
||||
table.put(p);
|
||||
admin.flush(tableName);
|
||||
}
|
||||
region.getTableDesc().getFamily(cf).setMobThreshold(0);
|
||||
// metrics are reset by the region initialization
|
||||
((HRegion) region).initialize();
|
||||
region.compact(true);
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
// metrics are reset by the region initialization
|
||||
assertCounter("cellsCountCompactedFromMob", 0);
|
||||
assertCounter("cellsCountCompactedToMob", 2 * numHfiles);
|
||||
} finally {
|
||||
admin.disableTable(tableName);
|
||||
admin.deleteTable(tableName);
|
||||
}
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
metricsHelper.assertCounter("mobFlushCount", numHfiles, serverSource);
|
||||
Scan scan = new Scan(Bytes.toBytes(0), Bytes.toBytes(2));
|
||||
ResultScanner scanner = t.getScanner(scan);
|
||||
scanner.next(100);
|
||||
numScanNext++; // this is an ugly construct
|
||||
scanner.close();
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
metricsHelper.assertCounter("mobScanCellsCount", 2, serverSource);
|
||||
region.getTableDesc().getFamily(cf).setMobThreshold(100);
|
||||
((HRegion)region).initialize();
|
||||
region.compact(true);
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
metricsHelper.assertCounter("cellsCountCompactedFromMob", numHfiles,
|
||||
serverSource);
|
||||
metricsHelper.assertCounter("cellsCountCompactedToMob", 0, serverSource);
|
||||
scanner = t.getScanner(scan);
|
||||
scanner.next(100);
|
||||
numScanNext++; // this is an ugly construct
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
// metrics are reset by the region initialization
|
||||
metricsHelper.assertCounter("mobScanCellsCount", 0, serverSource);
|
||||
for (int insertCount = numHfiles;
|
||||
insertCount < 2 * numHfiles - 1; insertCount++) {
|
||||
Put p = new Put(Bytes.toBytes(insertCount));
|
||||
p.addColumn(cf, qualifier, val);
|
||||
t.put(p);
|
||||
admin.flush(tableName);
|
||||
}
|
||||
region.getTableDesc().getFamily(cf).setMobThreshold(0);
|
||||
((HRegion)region).initialize();
|
||||
region.compact(true);
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
// metrics are reset by the region initialization
|
||||
metricsHelper.assertCounter("cellsCountCompactedFromMob", 0, serverSource);
|
||||
metricsHelper.assertCounter("cellsCountCompactedToMob", 2 * numHfiles - 1,
|
||||
serverSource);
|
||||
t.close();
|
||||
admin.close();
|
||||
connection.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
@Ignore
|
||||
public void testRangeCountMetrics() throws Exception {
|
||||
String tableNameString = "testRangeCountMetrics";
|
||||
final long[] timeranges =
|
||||
{ 1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000, 60000, 120000, 300000, 600000 };
|
||||
final String timeRangeType = "TimeRangeCount";
|
||||
final String timeRangeMetricName = "Mutate";
|
||||
boolean timeRangeCountUpdated = false;
|
||||
|
||||
TableName tName = TableName.valueOf(tableNameString);
|
||||
byte[] cfName = Bytes.toBytes("d");
|
||||
byte[] row = Bytes.toBytes("rk");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] initValue = Bytes.toBytes("Value");
|
||||
|
||||
TEST_UTIL.createTable(tName, cfName);
|
||||
|
||||
Connection connection = TEST_UTIL.getConnection();
|
||||
connection.getTable(tName).close(); // wait for the table to come up.
|
||||
|
||||
// Do a first put to be sure that the connection is established, meta is there and so on.
|
||||
Table table = connection.getTable(tName);
|
||||
Put p = new Put(row);
|
||||
p.addColumn(cfName, qualifier, initValue);
|
||||
p.addColumn(cf, qualifier, val);
|
||||
table.put(p);
|
||||
|
||||
// do some puts and gets
|
||||
|
|
@ -672,28 +515,15 @@ public class TestRegionServerMetrics {
|
|||
}
|
||||
}
|
||||
assertEquals(true, timeRangeCountUpdated);
|
||||
|
||||
table.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAverageRegionSize() throws Exception {
|
||||
TableName tableName = TableName.valueOf("testAverageRegionSize");
|
||||
byte[] cf = Bytes.toBytes("d");
|
||||
byte[] row = Bytes.toBytes("rk");
|
||||
byte[] qualifier = Bytes.toBytes("qual");
|
||||
byte[] val = Bytes.toBytes("Value");
|
||||
|
||||
//Force a hfile.
|
||||
Table t = TEST_UTIL.createTable(tableName, cf);
|
||||
Put p = new Put(row);
|
||||
p.addColumn(cf, qualifier, val);
|
||||
t.put(p);
|
||||
doNPuts(1, false);
|
||||
TEST_UTIL.getHBaseAdmin().flush(tableName);
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
assertTrue(metricsHelper.getGaugeDouble("averageRegionSize", serverSource) > 0.0);
|
||||
|
||||
t.close();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue