Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-22874 Define a public API for Canary checking and a non-public tool implementation 

Closes #580

* Canary is now an IA.Public interface
* CanaryTool is now the implementation

Signed-off-by: Sean Busbey <busbey@apache.org>
This commit is contained in:
Rushabh 2019-08-20 12:40:49 -07:00 committed by Sean Busbey
parent 61bc57f525
commit b5a396749d
5 changed files with 1910 additions and 1762 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/hbase
View File

@ -615,7 +615,7 @@ elif [ "$COMMAND" = "ltt" ] ; then
CLASS='org.apache.hadoop.hbase.util.LoadTestTool' CLASS='org.apache.hadoop.hbase.util.LoadTestTool'
HBASE_OPTS="$HBASE_OPTS $HBASE_LTT_OPTS" HBASE_OPTS="$HBASE_OPTS $HBASE_LTT_OPTS"
elif [ "$COMMAND" = "canary" ] ; then elif [ "$COMMAND" = "canary" ] ; then
CLASS='org.apache.hadoop.hbase.tool.Canary' CLASS='org.apache.hadoop.hbase.tool.CanaryTool'
HBASE_OPTS="$HBASE_OPTS $HBASE_CANARY_OPTS" HBASE_OPTS="$HBASE_OPTS $HBASE_CANARY_OPTS"
elif [ "$COMMAND" = "version" ] ; then elif [ "$COMMAND" = "version" ] ; then
CLASS='org.apache.hadoop.hbase.util.VersionInfo' CLASS='org.apache.hadoop.hbase.util.VersionInfo'

4
hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
View File

@ -55,7 +55,7 @@ org.apache.hadoop.hbase.quotas.QuotaUtil;
org.apache.hadoop.hbase.security.access.PermissionStorage; org.apache.hadoop.hbase.security.access.PermissionStorage;
org.apache.hadoop.hbase.security.visibility.VisibilityConstants; org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
org.apache.hadoop.hbase.tool.Canary; org.apache.hadoop.hbase.tool.CanaryTool;
org.apache.hadoop.hbase.util.Bytes; org.apache.hadoop.hbase.util.Bytes;
org.apache.hadoop.hbase.util.FSUtils; org.apache.hadoop.hbase.util.FSUtils;
org.apache.hadoop.hbase.util.JvmVersion; org.apache.hadoop.hbase.util.JvmVersion;
@ -513,7 +513,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
<%java>String description = null; <%java>String description = null;
if (tableName.equals(TableName.META_TABLE_NAME)){ if (tableName.equals(TableName.META_TABLE_NAME)){
description = "The hbase:meta table holds references to all User Table regions."; description = "The hbase:meta table holds references to all User Table regions.";
} else if (tableName.equals(Canary.DEFAULT_WRITE_TABLE_NAME)){ } else if (tableName.equals(CanaryTool.DEFAULT_WRITE_TABLE_NAME)){
description = "The hbase:canary table is used to sniff the write availbility of" description = "The hbase:canary table is used to sniff the write availbility of"
+ " each regionserver."; + " each regionserver.";
} else if (tableName.equals(PermissionStorage.ACL_TABLE_NAME)){ } else if (tableName.equals(PermissionStorage.ACL_TABLE_NAME)){

1759
hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
View File

File diff suppressed because it is too large Load Diff

1866
hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryTool.java Normal file
View File

File diff suppressed because it is too large Load Diff

41
hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
View File

@ -53,7 +53,6 @@ import org.apache.log4j.spi.LoggingEvent;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
@ -120,8 +119,8 @@ public class TestCanaryTool {
table.put(p); table.put(p);
} }
ExecutorService executor = new ScheduledThreadPoolExecutor(1); ExecutorService executor = new ScheduledThreadPoolExecutor(1);
Canary.RegionStdOutSink sink = spy(new Canary.RegionStdOutSink()); CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
Canary canary = new Canary(executor, sink); CanaryTool canary = new CanaryTool(executor, sink);
String[] args = { "-writeSniffing", "-t", "10000", tableName.getNameAsString() }; String[] args = { "-writeSniffing", "-t", "10000", tableName.getNameAsString() };
assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
assertEquals("verify no read error count", 0, canary.getReadFailures().size()); assertEquals("verify no read error count", 0, canary.getReadFailures().size());
@ -142,8 +141,8 @@ public class TestCanaryTool {
table.put(p); table.put(p);
} }
ExecutorService executor = new ScheduledThreadPoolExecutor(1); ExecutorService executor = new ScheduledThreadPoolExecutor(1);
Canary.RegionStdOutSink sink = spy(new Canary.RegionStdOutSink()); CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
Canary canary = new Canary(executor, sink); CanaryTool canary = new CanaryTool(executor, sink);
String[] args = { "-writeSniffing", "-t", "10000", "testCanaryRegionTaskResult" }; String[] args = { "-writeSniffing", "-t", "10000", "testCanaryRegionTaskResult" };
assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
@ -155,12 +154,12 @@ public class TestCanaryTool {
isA(ColumnFamilyDescriptor.class), anyLong()); isA(ColumnFamilyDescriptor.class), anyLong());
assertTrue("canary should expect to scan at least 1 region", assertTrue("canary should expect to scan at least 1 region",
sink.getTotalExpectedRegions() > 0); sink.getTotalExpectedRegions() > 0);
Map<String, Canary.RegionTaskResult> regionMap = sink.getRegionMap(); Map<String, CanaryTool.RegionTaskResult> regionMap = sink.getRegionMap();
assertFalse("verify region map has size > 0", regionMap.isEmpty()); assertFalse("verify region map has size > 0", regionMap.isEmpty());
for (String regionName : regionMap.keySet()) { for (String regionName : regionMap.keySet()) {
Canary.RegionTaskResult res = regionMap.get(regionName); CanaryTool.RegionTaskResult res = regionMap.get(regionName);
assertNotNull("verify each expected region has a RegionTaskResult object in the map", res); assertNotNull("verify each expected region has a RegionTaskResult object in the map", res);
assertNotNull("verify getRegionNameAsString()", regionName); assertNotNull("verify getRegionNameAsString()", regionName);
assertNotNull("verify getRegionInfo()", res.getRegionInfo()); assertNotNull("verify getRegionInfo()", res.getRegionInfo());
@ -169,7 +168,7 @@ public class TestCanaryTool {
assertNotNull("verify getServerName()", res.getServerName()); assertNotNull("verify getServerName()", res.getServerName());
assertNotNull("verify getServerNameAsString()", res.getServerNameAsString()); assertNotNull("verify getServerNameAsString()", res.getServerNameAsString());
if (regionName.contains(Canary.DEFAULT_WRITE_TABLE_NAME.getNameAsString())) { if (regionName.contains(CanaryTool.DEFAULT_WRITE_TABLE_NAME.getNameAsString())) {
assertTrue("write to region " + regionName + " succeeded", res.isWriteSuccess()); assertTrue("write to region " + regionName + " succeeded", res.isWriteSuccess());
assertTrue("write took some time", res.getWriteLatency() > -1); assertTrue("write took some time", res.getWriteLatency() > -1);
} else { } else {
@ -180,7 +179,6 @@ public class TestCanaryTool {
} }
@Test @Test
@Ignore("Intermittent argument matching failures, see HBASE-18813")
public void testReadTableTimeouts() throws Exception { public void testReadTableTimeouts() throws Exception {
final TableName [] tableNames = new TableName[2]; final TableName [] tableNames = new TableName[2];
tableNames[0] = TableName.valueOf(name.getMethodName() + "1"); tableNames[0] = TableName.valueOf(name.getMethodName() + "1");
@ -197,8 +195,8 @@ public class TestCanaryTool {
} }
} }
ExecutorService executor = new ScheduledThreadPoolExecutor(1); ExecutorService executor = new ScheduledThreadPoolExecutor(1);
Canary.RegionStdOutSink sink = spy(new Canary.RegionStdOutSink()); CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
Canary canary = new Canary(executor, sink); CanaryTool canary = new CanaryTool(executor, sink);
String configuredTimeoutStr = tableNames[0].getNameAsString() + "=" + Long.MAX_VALUE + "," + String configuredTimeoutStr = tableNames[0].getNameAsString() + "=" + Long.MAX_VALUE + "," +
tableNames[1].getNameAsString() + "=0"; tableNames[1].getNameAsString() + "=0";
String[] args = {"-readTableTimeouts", configuredTimeoutStr, name.getMethodName() + "1", String[] args = {"-readTableTimeouts", configuredTimeoutStr, name.getMethodName() + "1",
@ -219,17 +217,16 @@ public class TestCanaryTool {
verify(mockAppender, times(2)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() { verify(mockAppender, times(2)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
@Override @Override
public boolean matches(LoggingEvent argument) { public boolean matches(LoggingEvent argument) {
return argument.getRenderedMessage().contains("The configured read timeout was"); return argument.getRenderedMessage().contains("Configured read timeout");
} }
})); }));
} }
@Test @Test
@Ignore("Intermittent argument matching failures, see HBASE-18813")
public void testWriteTableTimeout() throws Exception { public void testWriteTableTimeout() throws Exception {
ExecutorService executor = new ScheduledThreadPoolExecutor(1); ExecutorService executor = new ScheduledThreadPoolExecutor(1);
Canary.RegionStdOutSink sink = spy(new Canary.RegionStdOutSink()); CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
Canary canary = new Canary(executor, sink); CanaryTool canary = new CanaryTool(executor, sink);
String[] args = { "-writeSniffing", "-writeTableTimeout", String.valueOf(Long.MAX_VALUE)}; String[] args = { "-writeSniffing", "-writeTableTimeout", String.valueOf(Long.MAX_VALUE)};
assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
assertNotEquals("verify non-null write latency", null, sink.getWriteLatency()); assertNotEquals("verify non-null write latency", null, sink.getWriteLatency());
@ -238,7 +235,7 @@ public class TestCanaryTool {
new ArgumentMatcher<LoggingEvent>() { new ArgumentMatcher<LoggingEvent>() {
@Override @Override
public boolean matches(LoggingEvent argument) { public boolean matches(LoggingEvent argument) {
return argument.getRenderedMessage().contains("The configured write timeout was"); return argument.getRenderedMessage().contains("Configured write timeout");
} }
})); }));
} }
@ -281,8 +278,8 @@ public class TestCanaryTool {
table.put(p); table.put(p);
} }
ExecutorService executor = new ScheduledThreadPoolExecutor(1); ExecutorService executor = new ScheduledThreadPoolExecutor(1);
Canary.RegionStdOutSink sink = spy(new Canary.RegionStdOutSink()); CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
Canary canary = new Canary(executor, sink); CanaryTool canary = new CanaryTool(executor, sink);
String[] args = { "-t", "10000", name.getMethodName() }; String[] args = { "-t", "10000", name.getMethodName() };
org.apache.hadoop.conf.Configuration conf = org.apache.hadoop.conf.Configuration conf =
new org.apache.hadoop.conf.Configuration(testingUtility.getConfiguration()); new org.apache.hadoop.conf.Configuration(testingUtility.getConfiguration());
@ -296,7 +293,7 @@ public class TestCanaryTool {
private void runRegionserverCanary() throws Exception { private void runRegionserverCanary() throws Exception {
ExecutorService executor = new ScheduledThreadPoolExecutor(1); ExecutorService executor = new ScheduledThreadPoolExecutor(1);
Canary canary = new Canary(executor, new Canary.RegionServerStdOutSink()); CanaryTool canary = new CanaryTool(executor, new CanaryTool.RegionServerStdOutSink());
String[] args = { "-t", "10000", "-regionserver"}; String[] args = { "-t", "10000", "-regionserver"};
assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
assertEquals("verify no read error count", 0, canary.getReadFailures().size()); assertEquals("verify no read error count", 0, canary.getReadFailures().size());
@ -308,8 +305,8 @@ public class TestCanaryTool {
testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM, testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM,
"localhost:" + port + "/hbase"); "localhost:" + port + "/hbase");
ExecutorService executor = new ScheduledThreadPoolExecutor(2); ExecutorService executor = new ScheduledThreadPoolExecutor(2);
Canary.ZookeeperStdOutSink sink = spy(new Canary.ZookeeperStdOutSink()); CanaryTool.ZookeeperStdOutSink sink = spy(new CanaryTool.ZookeeperStdOutSink());
Canary canary = new Canary(executor, sink); CanaryTool canary = new CanaryTool(executor, sink);
assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
String baseZnode = testingUtility.getConfiguration() String baseZnode = testingUtility.getConfiguration()