Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-15949 Cleanup TestRegionServerMetrics. 

@Before and @After to setup/teardown tables using @Rule to set table name based on testname.
Refactor out copy-pasted code fragments to single function.
(Apekshit)

Change-Id: Ic22e5027cc3952bab5ec30070ed20e98017db65a
This commit is contained in:
Apekshit 2016-06-01 21:52:29 -07:00 committed by Apekshit Sharma
parent bdb46f01b9
commit b557f0bec6
1 changed files with 253 additions and 423 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

578
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
View File

@ -17,6 +17,8 @@
*/ */
package org.apache.hadoop.hbase.regionserver; package org.apache.hadoop.hbase.regionserver;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.client.*;
@ -27,11 +29,17 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.Threads;
import org.apache.log4j.Level; import org.apache.log4j.Level;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Ignore; import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.junit.rules.TestRule;
import static org.junit.Assert.*; import static org.junit.Assert.*;
@ -39,23 +47,35 @@ import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@Category({RegionServerTests.class, MediumTests.class}) @Category({RegionServerTests.class, MediumTests.class})
public class TestRegionServerMetrics { public class TestRegionServerMetrics {
private static MetricsAssertHelper metricsHelper; private static final Log LOG = LogFactory.getLog(TestRegionServerMetrics.class);
@Rule
public TestName testName = new TestName();
@ClassRule
public static TestRule timeout = CategoryBasedTimeout.forClass(TestRegionServerMetrics.class);
static { static {
Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG); Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
} }
private static MetricsAssertHelper metricsHelper;
private static MiniHBaseCluster cluster; private static MiniHBaseCluster cluster;
private static HRegionServer rs; private static HRegionServer rs;
private static Configuration conf; private static Configuration conf;
private static HBaseTestingUtility TEST_UTIL; private static HBaseTestingUtility TEST_UTIL;
private static Connection connection;
private static MetricsRegionServer metricsRegionServer; private static MetricsRegionServer metricsRegionServer;
private static MetricsRegionServerSource serverSource; private static MetricsRegionServerSource serverSource;
private static final int NUM_SCAN_NEXT = 30; private static final int NUM_SCAN_NEXT = 30;
private static int numScanNext = 0; private static int numScanNext = 0;
private static byte[] cf = Bytes.toBytes("cf");
private static byte[] row = Bytes.toBytes("row");
private static byte[] qualifier = Bytes.toBytes("qual");
private static byte[] val = Bytes.toBytes("val");
private static Admin admin;
@BeforeClass @BeforeClass
public static void startCluster() throws Exception { public static void startCluster() throws Exception {
@ -65,12 +85,16 @@ public class TestRegionServerMetrics {
conf.getLong("hbase.splitlog.max.resubmit", 0); conf.getLong("hbase.splitlog.max.resubmit", 0);
// Make the failure test faster // Make the failure test faster
conf.setInt("zookeeper.recovery.retry", 0); conf.setInt("zookeeper.recovery.retry", 0);
// testMobMetrics creates few hfiles and manages compaction manually.
conf.setInt("hbase.hstore.compactionThreshold", 100);
conf.setInt("hbase.hstore.compaction.max", 100);
conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1); conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
TEST_UTIL.startMiniCluster(1, 1); TEST_UTIL.startMiniCluster(1, 1);
cluster = TEST_UTIL.getHBaseCluster(); cluster = TEST_UTIL.getHBaseCluster();
cluster.waitForActiveAndReadyMaster(); cluster.waitForActiveAndReadyMaster();
admin = TEST_UTIL.getHBaseAdmin();
connection = TEST_UTIL.getConnection();
while (cluster.getLiveRegionServerThreads().size() < 1) { while (cluster.getLiveRegionServerThreads().size() < 1) {
Threads.sleep(100); Threads.sleep(100);
@ -88,551 +112,370 @@ public class TestRegionServerMetrics {
} }
} }
@Test(timeout = 300000) TableName tableName;
Table table;
@Before
public void beforeTestMethod() throws Exception {
metricsRegionServer.getRegionServerWrapper().forceRecompute();
tableName = TableName.valueOf(testName.getMethodName());
table = TEST_UTIL.createTable(tableName, cf);
}
@After
public void afterTestMethod() throws Exception {
admin.disableTable(tableName);
admin.deleteTable(tableName);
}
public void waitTableDeleted(TableName name, long timeoutInMillis) throws Exception {
long start = System.currentTimeMillis();
while (true) {
HTableDescriptor[] tables = admin.listTables();
for (HTableDescriptor htd : tables) {
if (htd.getNameAsString() == name.getNameAsString())
return;
}
if (System.currentTimeMillis() - start > timeoutInMillis)
return;
Thread.sleep(1000);
}
}
public void assertCounter(String metric, long expectedValue) {
metricsHelper.assertCounter(metric, expectedValue, serverSource);
}
public void assertGauge(String metric, long expectedValue) {
metricsHelper.assertGauge(metric, expectedValue, serverSource);
}
// Aggregates metrics from regions and assert given list of metrics and expected values.
public void assertRegionMetrics(String metric, long expectedValue) throws Exception {
try (RegionLocator locator = connection.getRegionLocator(tableName)) {
for ( HRegionLocation location: locator.getAllRegionLocations()) {
HRegionInfo hri = location.getRegionInfo();
MetricsRegionAggregateSource agg =
rs.getRegion(hri.getRegionName()).getMetrics().getSource().getAggregateSource();
String prefix = "namespace_" + NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR +
"_table_" + tableName.getNameAsString() +
"_region_" + hri.getEncodedName()+
"_metric_";
metricsHelper.assertCounter(prefix + metric, expectedValue, agg);
}
}
}
public void doNPuts(int n, boolean batch) throws Exception {
if (batch) {
List<Put> puts = new ArrayList<>();
for (int i = 0; i < n; i++) {
Put p = new Put(Bytes.toBytes("" + i + "row")).addColumn(cf, qualifier, val);
puts.add(p);
}
table.put(puts);
} else {
for (int i = 0; i < n; i++) {
Put p = new Put(row).addColumn(cf, qualifier, val);
table.put(p);
}
}
}
public void doNGets(int n, boolean batch) throws Exception {
if (batch) {
List<Get> gets = new ArrayList<>();
for (int i = 0; i < n; i++) {
gets.add(new Get(row));
}
table.get(gets);
} else {
for (int i = 0; i < n; i++) {
table.get(new Get(row));
}
}
}
@Test
public void testRegionCount() throws Exception { public void testRegionCount() throws Exception {
String regionMetricsKey = "regionCount"; metricsHelper.assertGauge("regionCount", 1, serverSource);
long regions = metricsHelper.getGaugeLong(regionMetricsKey, serverSource);
// Creating a table should add one region
TEST_UTIL.createTable(TableName.valueOf("table"), Bytes.toBytes("cf"));
metricsHelper.assertGaugeGt(regionMetricsKey, regions, serverSource);
} }
@Test @Test
public void testLocalFiles() throws Exception { public void testLocalFiles() throws Exception {
metricsHelper.assertGauge("percentFilesLocal", 0, serverSource); assertGauge("percentFilesLocal", 0);
metricsHelper.assertGauge("percentFilesLocalSecondaryRegions", 0, serverSource); assertGauge("percentFilesLocalSecondaryRegions", 0);
} }
@Test @Test
public void testRequestCount() throws Exception { public void testRequestCount() throws Exception {
String tableNameString = "testRequestCount";
TableName tName = TableName.valueOf(tableNameString);
byte[] cfName = Bytes.toBytes("d");
byte[] row = Bytes.toBytes("rk");
byte[] qualifier = Bytes.toBytes("qual");
byte[] initValue = Bytes.toBytes("Value");
TEST_UTIL.createTable(tName, cfName);
Connection connection = TEST_UTIL.getConnection();
connection.getTable(tName).close(); //wait for the table to come up.
// Do a first put to be sure that the connection is established, meta is there and so on. // Do a first put to be sure that the connection is established, meta is there and so on.
Table table = connection.getTable(tName); doNPuts(1, false);
Put p = new Put(row);
p.addColumn(cfName, qualifier, initValue);
table.put(p);
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
long requests = metricsHelper.getCounter("totalRequestCount", serverSource); long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
long readRequests = metricsHelper.getCounter("readRequestCount", serverSource); long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource); long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
for (int i=0; i< 30; i++) { doNPuts(30, false);
table.put(p);
}
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource); assertCounter("totalRequestCount", requests + 30);
metricsHelper.assertCounter("readRequestCount", readRequests, serverSource); assertCounter("readRequestCount", readRequests);
metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource); assertCounter("writeRequestCount", writeRequests + 30);
Get g = new Get(row); doNGets(10, false);
for (int i=0; i< 10; i++) {
table.get(g);
}
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource); assertCounter("totalRequestCount", requests + 40);
metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource); assertCounter("readRequestCount", readRequests + 10);
metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource); assertCounter("writeRequestCount", writeRequests + 30);
try (RegionLocator locator = connection.getRegionLocator(tName)) { assertRegionMetrics("getNumOps", 10);
for ( HRegionLocation location: locator.getAllRegionLocations()) { assertRegionMetrics("mutateCount", 31);
HRegionInfo i = location.getRegionInfo();
MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName()) doNGets(10, true); // true = batch
.getMetrics()
.getSource()
.getAggregateSource();
String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
"_table_"+tableNameString +
"_region_" + i.getEncodedName()+
"_metric";
metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg);
metricsHelper.assertCounter(prefix + "_mutateCount", 31, agg);
}
}
List<Get> gets = new ArrayList<Get>();
for (int i=0; i< 10; i++) {
gets.add(new Get(row));
}
table.get(gets);
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("totalRequestCount", requests + 50, serverSource); assertCounter("totalRequestCount", requests + 50);
metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource); assertCounter("readRequestCount", readRequests + 20);
metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource); assertCounter("writeRequestCount", writeRequests + 30);
List<Put> puts = new ArrayList<>(); doNPuts(30, true);
for (int i=0; i< 30; i++) {
puts.add(p);
}
table.put(puts);
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("totalRequestCount", requests + 80, serverSource); assertCounter("totalRequestCount", requests + 80);
metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource); assertCounter("readRequestCount", readRequests + 20);
metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource); assertCounter("writeRequestCount", writeRequests + 60);
table.close();
} }
@Test @Test
public void testGet() throws Exception { public void testGet() throws Exception {
String tableNameString = "testGet";
TableName tName = TableName.valueOf(tableNameString);
byte[] cfName = Bytes.toBytes("d");
byte[] row = Bytes.toBytes("rk");
byte[] qualifier = Bytes.toBytes("qual");
byte[] initValue = Bytes.toBytes("Value");
TEST_UTIL.createTable(tName, cfName);
Connection connection = TEST_UTIL.getConnection();
connection.getTable(tName).close(); //wait for the table to come up.
// Do a first put to be sure that the connection is established, meta is there and so on. // Do a first put to be sure that the connection is established, meta is there and so on.
Table table = connection.getTable(tName); doNPuts(1, false);
Put p = new Put(row); doNGets(10, false);
p.addColumn(cfName, qualifier, initValue); assertRegionMetrics("getNumOps", 10);
table.put(p); assertRegionMetrics("getSizeNumOps", 10);
Get g = new Get(row);
for (int i=0; i< 10; i++) {
table.get(g);
}
metricsRegionServer.getRegionServerWrapper().forceRecompute();
try (RegionLocator locator = connection.getRegionLocator(tName)) {
for ( HRegionLocation location: locator.getAllRegionLocations()) {
HRegionInfo i = location.getRegionInfo();
MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
.getMetrics()
.getSource()
.getAggregateSource();
String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
"_table_"+tableNameString +
"_region_" + i.getEncodedName()+
"_metric";
metricsHelper.assertCounter(prefix + "_getSizeNumOps", 10, agg);
metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg);
}
metricsHelper.assertCounterGt("Get_num_ops", 10, serverSource); metricsHelper.assertCounterGt("Get_num_ops", 10, serverSource);
} }
table.close();
}
@Test @Test
public void testMutationsWithoutWal() throws Exception { public void testMutationsWithoutWal() throws Exception {
TableName tableName = TableName.valueOf("testMutationsWithoutWal"); Put p = new Put(row).addColumn(cf, qualifier, val)
byte[] cf = Bytes.toBytes("d"); .setDurability(Durability.SKIP_WAL);
byte[] row = Bytes.toBytes("rk"); table.put(p);
byte[] qualifier = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("Value");
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
assertGauge("mutationsWithoutWALCount", 1);
Table t = TEST_UTIL.createTable(tableName, cf);
Put p = new Put(row);
p.addColumn(cf, qualifier, val);
p.setDurability(Durability.SKIP_WAL);
t.put(p);
metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource);
long minLength = row.length + cf.length + qualifier.length + val.length; long minLength = row.length + cf.length + qualifier.length + val.length;
metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource); metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
t.close();
} }
@Test @Test
public void testStoreCount() throws Exception { public void testStoreCount() throws Exception {
TableName tableName = TableName.valueOf("testStoreCount");
byte[] cf = Bytes.toBytes("d");
byte[] row = Bytes.toBytes("rk");
byte[] qualifier = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("Value");
metricsRegionServer.getRegionServerWrapper().forceRecompute();
long stores = metricsHelper.getGaugeLong("storeCount", serverSource);
long storeFiles = metricsHelper.getGaugeLong("storeFileCount", serverSource);
//Force a hfile. //Force a hfile.
Table t = TEST_UTIL.createTable(tableName, cf); doNPuts(1, false);
Put p = new Put(row);
p.addColumn(cf, qualifier, val);
t.put(p);
TEST_UTIL.getHBaseAdmin().flush(tableName); TEST_UTIL.getHBaseAdmin().flush(tableName);
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertGauge("storeCount", stores +1, serverSource); assertGauge("storeCount", 1);
metricsHelper.assertGauge("storeFileCount", storeFiles + 1, serverSource); assertGauge("storeFileCount", 1);
t.close();
} }
@Test @Test
public void testStoreFileAge() throws Exception { public void testStoreFileAge() throws Exception {
TableName tableName = TableName.valueOf("testStoreFileAge");
byte[] cf = Bytes.toBytes("d");
byte[] row = Bytes.toBytes("rk");
byte[] qualifier = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("Value");
//Force a hfile. //Force a hfile.
Table t = TEST_UTIL.createTable(tableName, cf); doNPuts(1, false);
Put p = new Put(row);
p.addColumn(cf, qualifier, val);
t.put(p);
TEST_UTIL.getHBaseAdmin().flush(tableName); TEST_UTIL.getHBaseAdmin().flush(tableName);
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
assertTrue(metricsHelper.getGaugeLong("maxStoreFileAge", serverSource) > 0); assertTrue(metricsHelper.getGaugeLong("maxStoreFileAge", serverSource) > 0);
assertTrue(metricsHelper.getGaugeLong("minStoreFileAge", serverSource) > 0); assertTrue(metricsHelper.getGaugeLong("minStoreFileAge", serverSource) > 0);
assertTrue(metricsHelper.getGaugeLong("avgStoreFileAge", serverSource) > 0); assertTrue(metricsHelper.getGaugeLong("avgStoreFileAge", serverSource) > 0);
t.close();
} }
@Test @Test
public void testCheckAndPutCount() throws Exception { public void testCheckAndPutCount() throws Exception {
String tableNameString = "testCheckAndPutCount";
TableName tableName = TableName.valueOf(tableNameString);
byte[] cf = Bytes.toBytes("d");
byte[] row = Bytes.toBytes("rk");
byte[] qualifier = Bytes.toBytes("qual");
byte[] valOne = Bytes.toBytes("Value"); byte[] valOne = Bytes.toBytes("Value");
byte[] valTwo = Bytes.toBytes("ValueTwo"); byte[] valTwo = Bytes.toBytes("ValueTwo");
byte[] valThree = Bytes.toBytes("ValueThree"); byte[] valThree = Bytes.toBytes("ValueThree");
Table t = TEST_UTIL.createTable(tableName, cf);
Put p = new Put(row); Put p = new Put(row);
p.addColumn(cf, qualifier, valOne); p.addColumn(cf, qualifier, valOne);
t.put(p); table.put(p);
Put pTwo = new Put(row); Put pTwo = new Put(row);
pTwo.addColumn(cf, qualifier, valTwo); pTwo.addColumn(cf, qualifier, valTwo);
t.checkAndPut(row, cf, qualifier, valOne, pTwo); table.checkAndPut(row, cf, qualifier, valOne, pTwo);
Put pThree = new Put(row); Put pThree = new Put(row);
pThree.addColumn(cf, qualifier, valThree); pThree.addColumn(cf, qualifier, valThree);
t.checkAndPut(row, cf, qualifier, valOne, pThree); table.checkAndPut(row, cf, qualifier, valOne, pThree);
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("checkMutateFailedCount", 1, serverSource); assertCounter("checkMutateFailedCount", 1);
metricsHelper.assertCounter("checkMutatePassedCount", 1, serverSource); assertCounter("checkMutatePassedCount", 1);
t.close();
} }
@Test @Test
public void testIncrement() throws Exception { public void testIncrement() throws Exception {
String tableNameString = "testIncrement"; Put p = new Put(row).addColumn(cf, qualifier, Bytes.toBytes(0L));
TableName tableName = TableName.valueOf(tableNameString); table.put(p);
byte[] cf = Bytes.toBytes("d");
byte[] row = Bytes.toBytes("rk");
byte[] qualifier = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes(0l);
Table t = TEST_UTIL.createTable(tableName, cf);
Put p = new Put(row);
p.addColumn(cf, qualifier, val);
t.put(p);
for(int count = 0; count < 13; count++) { for(int count = 0; count < 13; count++) {
Increment inc = new Increment(row); Increment inc = new Increment(row);
inc.addColumn(cf, qualifier, 100); inc.addColumn(cf, qualifier, 100);
t.increment(inc); table.increment(inc);
} }
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("incrementNumOps", 13, serverSource); assertCounter("incrementNumOps", 13);
t.close();
} }
@Test @Test
public void testAppend() throws Exception { public void testAppend() throws Exception {
String tableNameString = "testAppend"; doNPuts(1, false);
TableName tableName = TableName.valueOf(tableNameString);
byte[] cf = Bytes.toBytes("d");
byte[] row = Bytes.toBytes("rk");
byte[] qualifier = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("One");
Table t = TEST_UTIL.createTable(tableName, cf);
Put p = new Put(row);
p.addColumn(cf, qualifier, val);
t.put(p);
for(int count = 0; count< 73; count++) { for(int count = 0; count< 73; count++) {
Append append = new Append(row); Append append = new Append(row);
append.add(cf, qualifier, Bytes.toBytes(",Test")); append.add(cf, qualifier, Bytes.toBytes(",Test"));
t.append(append); table.append(append);
} }
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("appendNumOps", 73, serverSource); assertCounter("appendNumOps", 73);
t.close();
} }
@Test @Test
public void testScanSize() throws IOException { public void testScanSize() throws Exception {
String tableNameString = "testScanSize"; doNPuts(100, true); // batch put
TableName tableName = TableName.valueOf(tableNameString);
byte[] cf = Bytes.toBytes("d");
byte[] qualifier = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("One");
List<Put> puts = new ArrayList<>();
for (int insertCount =0; insertCount < 100; insertCount++) {
Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
p.addColumn(cf, qualifier, val);
puts.add(p);
}
try (Table t = TEST_UTIL.createTable(tableName, cf)) {
t.put(puts);
Scan s = new Scan(); Scan s = new Scan();
s.setBatch(1); s.setBatch(1);
s.setCaching(1); s.setCaching(1);
ResultScanner resultScanners = t.getScanner(s); ResultScanner resultScanners = table.getScanner(s);
for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) { for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
Result result = resultScanners.next(); Result result = resultScanners.next();
assertNotNull(result); assertNotNull(result);
assertEquals(1, result.size()); assertEquals(1, result.size());
} }
}
numScanNext += NUM_SCAN_NEXT; numScanNext += NUM_SCAN_NEXT;
try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { assertRegionMetrics("scanSizeNumOps", NUM_SCAN_NEXT);
for ( HRegionLocation location: locator.getAllRegionLocations()) { assertCounter("ScanSize_num_ops", numScanNext);
HRegionInfo i = location.getRegionInfo();
MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
.getMetrics()
.getSource()
.getAggregateSource();
String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
"_table_"+tableNameString +
"_region_" + i.getEncodedName()+
"_metric";
metricsHelper.assertCounter(prefix + "_scanSizeNumOps", NUM_SCAN_NEXT, agg);
}
metricsHelper.assertCounter("ScanSize_num_ops", numScanNext, serverSource);
}
try (Admin admin = TEST_UTIL.getHBaseAdmin()) {
admin.disableTable(tableName);
admin.deleteTable(tableName);
}
} }
@Test @Test
public void testScanTime() throws IOException { public void testScanTime() throws Exception {
String tableNameString = "testScanTime"; doNPuts(100, true);
TableName tableName = TableName.valueOf(tableNameString);
byte[] cf = Bytes.toBytes("d");
byte[] qualifier = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("One");
List<Put> puts = new ArrayList<>();
for (int insertCount =0; insertCount < 100; insertCount++) {
Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
p.addColumn(cf, qualifier, val);
puts.add(p);
}
try (Table t = TEST_UTIL.createTable(tableName, cf)) {
t.put(puts);
Scan s = new Scan(); Scan s = new Scan();
s.setBatch(1); s.setBatch(1);
s.setCaching(1); s.setCaching(1);
ResultScanner resultScanners = t.getScanner(s); ResultScanner resultScanners = table.getScanner(s);
for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) { for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
Result result = resultScanners.next(); Result result = resultScanners.next();
assertNotNull(result); assertNotNull(result);
assertEquals(1, result.size()); assertEquals(1, result.size());
} }
}
numScanNext += NUM_SCAN_NEXT; numScanNext += NUM_SCAN_NEXT;
try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { assertRegionMetrics("scanTimeNumOps", NUM_SCAN_NEXT);
for ( HRegionLocation location: locator.getAllRegionLocations()) { assertCounter("ScanTime_num_ops", numScanNext);
HRegionInfo i = location.getRegionInfo();
MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
.getMetrics()
.getSource()
.getAggregateSource();
String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
"_table_"+tableNameString +
"_region_" + i.getEncodedName()+
"_metric";
metricsHelper.assertCounter(prefix + "_scanTimeNumOps", NUM_SCAN_NEXT, agg);
}
metricsHelper.assertCounter("ScanTime_num_ops", numScanNext, serverSource);
}
try (Admin admin = TEST_UTIL.getHBaseAdmin()) {
admin.disableTable(tableName);
admin.deleteTable(tableName);
}
} }
@Test @Test
public void testScanSizeForSmallScan() throws IOException { public void testScanSizeForSmallScan() throws Exception {
String tableNameString = "testScanSizeSmall"; doNPuts(100, true);
TableName tableName = TableName.valueOf(tableNameString);
byte[] cf = Bytes.toBytes("d");
byte[] qualifier = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("One");
List<Put> puts = new ArrayList<>();
for (int insertCount =0; insertCount < 100; insertCount++) {
Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
p.addColumn(cf, qualifier, val);
puts.add(p);
}
try (Table t = TEST_UTIL.createTable(tableName, cf)) {
t.put(puts);
Scan s = new Scan(); Scan s = new Scan();
s.setSmall(true); s.setSmall(true);
s.setCaching(1); s.setCaching(1);
ResultScanner resultScanners = t.getScanner(s); ResultScanner resultScanners = table.getScanner(s);
for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) { for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
Result result = resultScanners.next(); Result result = resultScanners.next();
assertNotNull(result); assertNotNull(result);
assertEquals(1, result.size()); assertEquals(1, result.size());
} }
}
numScanNext += NUM_SCAN_NEXT; numScanNext += NUM_SCAN_NEXT;
try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { assertRegionMetrics("scanSizeNumOps", NUM_SCAN_NEXT);
for ( HRegionLocation location: locator.getAllRegionLocations()) { assertCounter("ScanSize_num_ops", numScanNext);
HRegionInfo i = location.getRegionInfo();
MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
.getMetrics()
.getSource()
.getAggregateSource();
String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
"_table_"+tableNameString +
"_region_" + i.getEncodedName()+
"_metric";
metricsHelper.assertCounter(prefix + "_scanSizeNumOps", NUM_SCAN_NEXT, agg);
}
metricsHelper.assertCounter("ScanSize_num_ops", numScanNext, serverSource);
}
try (Admin admin = TEST_UTIL.getHBaseAdmin()) {
admin.disableTable(tableName);
admin.deleteTable(tableName);
}
} }
@Test @Test
public void testMobMetrics() throws IOException, InterruptedException { public void testMobMetrics() throws IOException, InterruptedException {
String tableNameString = "testMobMetrics"; TableName tableName = TableName.valueOf("testMobMetricsLocal");
TableName tableName = TableName.valueOf(tableNameString); int numHfiles = 5;
byte[] cf = Bytes.toBytes("d");
byte[] qualifier = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("mobdata");
int numHfiles = conf.getInt("hbase.hstore.compactionThreshold", 3) - 1;
HTableDescriptor htd = new HTableDescriptor(tableName); HTableDescriptor htd = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor(cf); HColumnDescriptor hcd = new HColumnDescriptor(cf);
hcd.setMobEnabled(true); hcd.setMobEnabled(true);
hcd.setMobThreshold(0); hcd.setMobThreshold(0);
htd.addFamily(hcd); htd.addFamily(hcd);
Connection connection = ConnectionFactory.createConnection(conf); byte[] val = Bytes.toBytes("mobdata");
Admin admin = connection.getAdmin(); try {
Table t = TEST_UTIL.createTable(htd, new byte[0][0], conf); Table table = TEST_UTIL.createTable(htd, new byte[0][0], conf);
Region region = rs.getOnlineRegions(tableName).get(0); Region region = rs.getOnlineRegions(tableName).get(0);
for (int insertCount = 0; insertCount < numHfiles; insertCount++) { for (int insertCount = 0; insertCount < numHfiles; insertCount++) {
Put p = new Put(Bytes.toBytes(insertCount)); Put p = new Put(Bytes.toBytes(insertCount));
p.addColumn(cf, qualifier, val); p.addColumn(cf, qualifier, val);
t.put(p); table.put(p);
admin.flush(tableName); admin.flush(tableName);
} }
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("mobFlushCount", numHfiles, serverSource); assertCounter("mobFlushCount", numHfiles);
Scan scan = new Scan(Bytes.toBytes(0), Bytes.toBytes(2));
ResultScanner scanner = t.getScanner(scan); Scan scan = new Scan(Bytes.toBytes(0), Bytes.toBytes(numHfiles));
ResultScanner scanner = table.getScanner(scan);
scanner.next(100); scanner.next(100);
numScanNext++; // this is an ugly construct numScanNext++; // this is an ugly construct
scanner.close(); scanner.close();
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("mobScanCellsCount", 2, serverSource); assertCounter("mobScanCellsCount", numHfiles);
region.getTableDesc().getFamily(cf).setMobThreshold(100); region.getTableDesc().getFamily(cf).setMobThreshold(100);
// metrics are reset by the region initialization
((HRegion) region).initialize(); ((HRegion) region).initialize();
region.compact(true); region.compact(true);
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("cellsCountCompactedFromMob", numHfiles, assertCounter("cellsCountCompactedFromMob", numHfiles);
serverSource); assertCounter("cellsCountCompactedToMob", 0);
metricsHelper.assertCounter("cellsCountCompactedToMob", 0, serverSource);
scanner = t.getScanner(scan); scanner = table.getScanner(scan);
scanner.next(100); scanner.next(100);
numScanNext++; // this is an ugly construct numScanNext++; // this is an ugly construct
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
// metrics are reset by the region initialization assertCounter("mobScanCellsCount", 0);
metricsHelper.assertCounter("mobScanCellsCount", 0, serverSource);
for (int insertCount = numHfiles; for (int insertCount = numHfiles; insertCount < 2 * numHfiles; insertCount++) {
insertCount < 2 * numHfiles - 1; insertCount++) {
Put p = new Put(Bytes.toBytes(insertCount)); Put p = new Put(Bytes.toBytes(insertCount));
p.addColumn(cf, qualifier, val); p.addColumn(cf, qualifier, val);
t.put(p); table.put(p);
admin.flush(tableName); admin.flush(tableName);
} }
region.getTableDesc().getFamily(cf).setMobThreshold(0); region.getTableDesc().getFamily(cf).setMobThreshold(0);
// metrics are reset by the region initialization
((HRegion) region).initialize(); ((HRegion) region).initialize();
region.compact(true); region.compact(true);
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
// metrics are reset by the region initialization // metrics are reset by the region initialization
metricsHelper.assertCounter("cellsCountCompactedFromMob", 0, serverSource); assertCounter("cellsCountCompactedFromMob", 0);
metricsHelper.assertCounter("cellsCountCompactedToMob", 2 * numHfiles - 1, assertCounter("cellsCountCompactedToMob", 2 * numHfiles);
serverSource); } finally {
t.close(); admin.disableTable(tableName);
admin.close(); admin.deleteTable(tableName);
connection.close(); }
} }
@Test @Test
@Ignore @Ignore
public void testRangeCountMetrics() throws Exception { public void testRangeCountMetrics() throws Exception {
String tableNameString = "testRangeCountMetrics";
final long[] timeranges = final long[] timeranges =
{ 1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000, 60000, 120000, 300000, 600000 }; { 1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000, 60000, 120000, 300000, 600000 };
final String timeRangeType = "TimeRangeCount"; final String timeRangeType = "TimeRangeCount";
final String timeRangeMetricName = "Mutate"; final String timeRangeMetricName = "Mutate";
boolean timeRangeCountUpdated = false; boolean timeRangeCountUpdated = false;
TableName tName = TableName.valueOf(tableNameString);
byte[] cfName = Bytes.toBytes("d");
byte[] row = Bytes.toBytes("rk");
byte[] qualifier = Bytes.toBytes("qual");
byte[] initValue = Bytes.toBytes("Value");
TEST_UTIL.createTable(tName, cfName);
Connection connection = TEST_UTIL.getConnection();
connection.getTable(tName).close(); // wait for the table to come up.
// Do a first put to be sure that the connection is established, meta is there and so on. // Do a first put to be sure that the connection is established, meta is there and so on.
Table table = connection.getTable(tName);
Put p = new Put(row); Put p = new Put(row);
p.addColumn(cfName, qualifier, initValue); p.addColumn(cf, qualifier, val);
table.put(p); table.put(p);
// do some puts and gets // do some puts and gets
@ -672,28 +515,15 @@ public class TestRegionServerMetrics {
} }
} }
assertEquals(true, timeRangeCountUpdated); assertEquals(true, timeRangeCountUpdated);
table.close();
} }
@Test @Test
public void testAverageRegionSize() throws Exception { public void testAverageRegionSize() throws Exception {
TableName tableName = TableName.valueOf("testAverageRegionSize");
byte[] cf = Bytes.toBytes("d");
byte[] row = Bytes.toBytes("rk");
byte[] qualifier = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("Value");
//Force a hfile. //Force a hfile.
Table t = TEST_UTIL.createTable(tableName, cf); doNPuts(1, false);
Put p = new Put(row);
p.addColumn(cf, qualifier, val);
t.put(p);
TEST_UTIL.getHBaseAdmin().flush(tableName); TEST_UTIL.getHBaseAdmin().flush(tableName);
metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsRegionServer.getRegionServerWrapper().forceRecompute();
assertTrue(metricsHelper.getGaugeDouble("averageRegionSize", serverSource) > 0.0); assertTrue(metricsHelper.getGaugeDouble("averageRegionSize", serverSource) > 0.0);
t.close();
} }
} }