Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-22115 HBase RPC aspires to grow an infinite tree of trace scopes; some other places are also unsafe 

Signed-off-by: Andrew Purtell <apurtell@apache.org>, stack <stack@apache.org>
This commit is contained in:
Sergey Shelukhin 2019-03-29 13:38:11 -07:00 committed by Duo Zhang
parent 46d0e5a05d
commit 78c4e4996a
2 changed files with 70 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/CallRunner.java
View File

@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.security.User;
import org.apache.hbase.thirdparty.com.google.protobuf.Message; import org.apache.hbase.thirdparty.com.google.protobuf.Message;
import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
import org.apache.htrace.core.TraceScope;
/** /**
* The request processing logic, which is usually executed in thread pools provided by an * The request processing logic, which is usually executed in thread pools provided by an
@ -115,6 +116,7 @@ public class CallRunner {
String error = null; String error = null;
Pair<Message, CellScanner> resultPair = null; Pair<Message, CellScanner> resultPair = null;
RpcServer.CurCall.set(call); RpcServer.CurCall.set(call);
TraceScope traceScope = null;
try { try {
if (!this.rpcServer.isStarted()) { if (!this.rpcServer.isStarted()) {
InetSocketAddress address = rpcServer.getListenerAddress(); InetSocketAddress address = rpcServer.getListenerAddress();
@ -125,7 +127,7 @@ public class CallRunner {
call.getService() != null ? call.getService().getDescriptorForType().getName() : ""; call.getService() != null ? call.getService().getDescriptorForType().getName() : "";
String methodName = (call.getMethod() != null) ? call.getMethod().getName() : ""; String methodName = (call.getMethod() != null) ? call.getMethod().getName() : "";
String traceString = serviceName + "." + methodName; String traceString = serviceName + "." + methodName;
TraceUtil.createTrace(traceString); traceScope = TraceUtil.createTrace(traceString);
// make the call // make the call
resultPair = this.rpcServer.call(call, this.status); resultPair = this.rpcServer.call(call, this.status);
} catch (TimeoutIOException e){ } catch (TimeoutIOException e){
@ -147,6 +149,9 @@ public class CallRunner {
throw (Error)e; throw (Error)e;
} }
} finally { } finally {
if (traceScope != null) {
traceScope.close();
}
RpcServer.CurCall.set(null); RpcServer.CurCall.set(null);
if (resultPair != null) { if (resultPair != null) {
this.rpcServer.addCallSize(call.getSize() * -1); this.rpcServer.addCallSize(call.getSize() * -1);

12
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
View File

@ -693,11 +693,12 @@ class MemStoreFlusher implements FlushRequester {
* amount of memstore consumption. * amount of memstore consumption.
*/ */
public void reclaimMemStoreMemory() { public void reclaimMemStoreMemory() {
TraceScope scope = TraceUtil.createTrace("MemStoreFluser.reclaimMemStoreMemory"); try (TraceScope scope = TraceUtil.createTrace("MemStoreFluser.reclaimMemStoreMemory")) {
FlushType flushType = isAboveHighWaterMark(); FlushType flushType = isAboveHighWaterMark();
if (flushType != FlushType.NORMAL) { if (flushType != FlushType.NORMAL) {
TraceUtil.addTimelineAnnotation("Force Flush. We're above high water mark."); TraceUtil.addTimelineAnnotation("Force Flush. We're above high water mark.");
long start = EnvironmentEdgeManager.currentTime(); long start = EnvironmentEdgeManager.currentTime();
long nextLogTimeMs = start;
synchronized (this.blockSignal) { synchronized (this.blockSignal) {
boolean blocked = false; boolean blocked = false;
long startTime = 0; long startTime = 0;
@ -739,8 +740,11 @@ class MemStoreFlusher implements FlushRequester {
LOG.warn("Interrupted while waiting"); LOG.warn("Interrupted while waiting");
interrupted = true; interrupted = true;
} }
long took = EnvironmentEdgeManager.currentTime() - start; long nowMs = EnvironmentEdgeManager.currentTime();
LOG.warn("Memstore is above high water mark and block " + took + "ms"); if (nowMs >= nextLogTimeMs) {
LOG.warn("Memstore is above high water mark and block {} ms", nowMs - start);
nextLogTimeMs = nowMs + 1000;
}
flushType = isAboveHighWaterMark(); flushType = isAboveHighWaterMark();
} }
} finally { } finally {
@ -764,8 +768,6 @@ class MemStoreFlusher implements FlushRequester {
wakeupFlushThread(); wakeupFlushThread();
} }
} }
if(scope!= null) {
scope.close();
} }
} }