HDDS-1214. Enable tracing for the datanode read/write path. Contributed by Elek, Marton.
Closes #550.
This commit is contained in:
parent
c7307867f0
commit
d17e31e062
|
@ -22,6 +22,7 @@
|
||||||
import io.jaegertracing.Configuration;
|
import io.jaegertracing.Configuration;
|
||||||
import io.jaegertracing.internal.JaegerTracer;
|
import io.jaegertracing.internal.JaegerTracer;
|
||||||
import io.opentracing.Scope;
|
import io.opentracing.Scope;
|
||||||
|
import io.opentracing.Span;
|
||||||
import io.opentracing.SpanContext;
|
import io.opentracing.SpanContext;
|
||||||
import io.opentracing.Tracer;
|
import io.opentracing.Tracer;
|
||||||
import io.opentracing.util.GlobalTracer;
|
import io.opentracing.util.GlobalTracer;
|
||||||
|
@ -64,6 +65,19 @@ public static String exportCurrentSpan() {
|
||||||
return builder.toString();
|
return builder.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Export the specific span as a string.
|
||||||
|
*
|
||||||
|
* @return encoded tracing context.
|
||||||
|
*/
|
||||||
|
public static String exportSpan(Span span) {
|
||||||
|
StringBuilder builder = new StringBuilder();
|
||||||
|
if (span != null) {
|
||||||
|
GlobalTracer.get().inject(span.context(), StringCodec.FORMAT, builder);
|
||||||
|
}
|
||||||
|
return builder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a new scope and use the imported span as the parent.
|
* Create a new scope and use the imported span as the parent.
|
||||||
*
|
*
|
||||||
|
|
|
@ -33,6 +33,7 @@
|
||||||
.InvalidContainerStateException;
|
.InvalidContainerStateException;
|
||||||
import org.apache.hadoop.hdds.scm.container.common.helpers
|
import org.apache.hadoop.hdds.scm.container.common.helpers
|
||||||
.StorageContainerException;
|
.StorageContainerException;
|
||||||
|
import org.apache.hadoop.hdds.tracing.TracingUtil;
|
||||||
import org.apache.hadoop.ozone.audit.AuditAction;
|
import org.apache.hadoop.ozone.audit.AuditAction;
|
||||||
import org.apache.hadoop.ozone.audit.AuditEventStatus;
|
import org.apache.hadoop.ozone.audit.AuditEventStatus;
|
||||||
import org.apache.hadoop.ozone.audit.AuditLogger;
|
import org.apache.hadoop.ozone.audit.AuditLogger;
|
||||||
|
@ -61,6 +62,8 @@
|
||||||
ContainerDataProto.State;
|
ContainerDataProto.State;
|
||||||
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
|
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
|
||||||
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
|
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
|
||||||
|
|
||||||
|
import io.opentracing.Scope;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
@ -137,10 +140,19 @@ public void buildMissingContainerSet(Set<Long> createdContainerSet) {
|
||||||
containerSet.buildMissingContainerSet(createdContainerSet);
|
containerSet.buildMissingContainerSet(createdContainerSet);
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("methodlength")
|
|
||||||
@Override
|
@Override
|
||||||
public ContainerCommandResponseProto dispatch(
|
public ContainerCommandResponseProto dispatch(
|
||||||
ContainerCommandRequestProto msg, DispatcherContext dispatcherContext) {
|
ContainerCommandRequestProto msg, DispatcherContext dispatcherContext) {
|
||||||
|
String spanName = "HddsDispatcher." + msg.getCmdType().name();
|
||||||
|
try (Scope scope = TracingUtil
|
||||||
|
.importAndCreateScope(spanName, msg.getTraceID())) {
|
||||||
|
return dispatchRequest(msg, dispatcherContext);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("methodlength")
|
||||||
|
private ContainerCommandResponseProto dispatchRequest(
|
||||||
|
ContainerCommandRequestProto msg, DispatcherContext dispatcherContext) {
|
||||||
Preconditions.checkNotNull(msg);
|
Preconditions.checkNotNull(msg);
|
||||||
LOG.trace("Command {}, trace ID: {} ", msg.getCmdType().toString(),
|
LOG.trace("Command {}, trace ID: {} ", msg.getCmdType().toString(),
|
||||||
msg.getTraceID());
|
msg.getTraceID());
|
||||||
|
|
|
@ -24,6 +24,7 @@
|
||||||
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
|
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
|
||||||
import org.apache.hadoop.hdds.protocol.proto
|
import org.apache.hadoop.hdds.protocol.proto
|
||||||
.StorageContainerDatanodeProtocolProtos.CloseContainerCommandProto;
|
.StorageContainerDatanodeProtocolProtos.CloseContainerCommandProto;
|
||||||
|
import org.apache.hadoop.hdds.tracing.TracingUtil;
|
||||||
import org.apache.hadoop.ozone.container.common.interfaces.Container;
|
import org.apache.hadoop.ozone.container.common.interfaces.Container;
|
||||||
import org.apache.hadoop.ozone.container.common.statemachine
|
import org.apache.hadoop.ozone.container.common.statemachine
|
||||||
.SCMConnectionManager;
|
.SCMConnectionManager;
|
||||||
|
@ -133,6 +134,7 @@ private ContainerCommandRequestProto getContainerCommandRequestProto(
|
||||||
final ContainerCommandRequestProto.Builder command =
|
final ContainerCommandRequestProto.Builder command =
|
||||||
ContainerCommandRequestProto.newBuilder();
|
ContainerCommandRequestProto.newBuilder();
|
||||||
command.setCmdType(ContainerProtos.Type.CloseContainer);
|
command.setCmdType(ContainerProtos.Type.CloseContainer);
|
||||||
|
command.setTraceID(TracingUtil.exportCurrentSpan());
|
||||||
command.setContainerID(containerId);
|
command.setContainerID(containerId);
|
||||||
command.setCloseContainer(
|
command.setCloseContainer(
|
||||||
ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
|
ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
|
||||||
|
|
|
@ -31,10 +31,12 @@
|
||||||
import org.apache.hadoop.hdds.scm.container.common.helpers.
|
import org.apache.hadoop.hdds.scm.container.common.helpers.
|
||||||
StorageContainerException;
|
StorageContainerException;
|
||||||
import org.apache.hadoop.hdds.tracing.GrpcServerInterceptor;
|
import org.apache.hadoop.hdds.tracing.GrpcServerInterceptor;
|
||||||
|
import org.apache.hadoop.hdds.tracing.TracingUtil;
|
||||||
import org.apache.hadoop.ozone.OzoneConfigKeys;
|
import org.apache.hadoop.ozone.OzoneConfigKeys;
|
||||||
import org.apache.hadoop.ozone.OzoneConsts;
|
import org.apache.hadoop.ozone.OzoneConsts;
|
||||||
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
|
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
|
||||||
|
|
||||||
|
import io.opentracing.Scope;
|
||||||
import org.apache.ratis.thirdparty.io.grpc.BindableService;
|
import org.apache.ratis.thirdparty.io.grpc.BindableService;
|
||||||
import org.apache.ratis.thirdparty.io.grpc.Server;
|
import org.apache.ratis.thirdparty.io.grpc.Server;
|
||||||
import org.apache.ratis.thirdparty.io.grpc.ServerBuilder;
|
import org.apache.ratis.thirdparty.io.grpc.ServerBuilder;
|
||||||
|
@ -168,6 +170,11 @@ public void stop() {
|
||||||
@Override
|
@Override
|
||||||
public void submitRequest(ContainerCommandRequestProto request,
|
public void submitRequest(ContainerCommandRequestProto request,
|
||||||
HddsProtos.PipelineID pipelineID) throws IOException {
|
HddsProtos.PipelineID pipelineID) throws IOException {
|
||||||
|
try (Scope scope = TracingUtil
|
||||||
|
.importAndCreateScope(
|
||||||
|
"XceiverServerGrpc." + request.getCmdType().name(),
|
||||||
|
request.getTraceID())) {
|
||||||
|
|
||||||
super.submitRequest(request, pipelineID);
|
super.submitRequest(request, pipelineID);
|
||||||
ContainerProtos.ContainerCommandResponseProto response =
|
ContainerProtos.ContainerCommandResponseProto response =
|
||||||
storageContainer.dispatch(request, null);
|
storageContainer.dispatch(request, null);
|
||||||
|
@ -176,6 +183,7 @@ public void submitRequest(ContainerCommandRequestProto request,
|
||||||
response.getResult());
|
response.getResult());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean isExist(HddsProtos.PipelineID pipelineId) {
|
public boolean isExist(HddsProtos.PipelineID pipelineId) {
|
||||||
|
|
|
@ -25,7 +25,6 @@
|
||||||
import org.apache.hadoop.hdds.HddsUtils;
|
import org.apache.hadoop.hdds.HddsUtils;
|
||||||
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
|
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
|
||||||
|
|
||||||
import io.opentracing.Scope;
|
|
||||||
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
|
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
|
||||||
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
|
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
|
||||||
import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
|
import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
|
||||||
|
@ -51,7 +50,6 @@
|
||||||
.ReadChunkRequestProto;
|
.ReadChunkRequestProto;
|
||||||
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
||||||
.ReadChunkResponseProto;
|
.ReadChunkResponseProto;
|
||||||
import org.apache.hadoop.hdds.tracing.TracingUtil;
|
|
||||||
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
|
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
|
||||||
import org.apache.hadoop.hdds.security.token.TokenVerifier;
|
import org.apache.hadoop.hdds.security.token.TokenVerifier;
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
|
@ -271,8 +269,6 @@ public TransactionContext startTransaction(RaftClientRequest request)
|
||||||
final ContainerCommandRequestProto proto =
|
final ContainerCommandRequestProto proto =
|
||||||
getContainerCommandRequestProto(request.getMessage().getContent());
|
getContainerCommandRequestProto(request.getMessage().getContent());
|
||||||
Preconditions.checkArgument(request.getRaftGroupId().equals(gid));
|
Preconditions.checkArgument(request.getRaftGroupId().equals(gid));
|
||||||
try (Scope scope = TracingUtil
|
|
||||||
.importAndCreateScope(proto.getCmdType().name(), proto.getTraceID())) {
|
|
||||||
try {
|
try {
|
||||||
dispatcher.validateContainerCommand(proto);
|
dispatcher.validateContainerCommand(proto);
|
||||||
} catch (IOException ioe) {
|
} catch (IOException ioe) {
|
||||||
|
@ -298,6 +294,7 @@ public TransactionContext startTransaction(RaftClientRequest request)
|
||||||
ContainerCommandRequestProto
|
ContainerCommandRequestProto
|
||||||
.newBuilder(proto)
|
.newBuilder(proto)
|
||||||
.setWriteChunk(commitWriteChunkProto)
|
.setWriteChunk(commitWriteChunkProto)
|
||||||
|
.setTraceID(proto.getTraceID())
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
return TransactionContext.newBuilder()
|
return TransactionContext.newBuilder()
|
||||||
|
@ -315,7 +312,7 @@ public TransactionContext startTransaction(RaftClientRequest request)
|
||||||
.setLogData(request.getMessage().getContent())
|
.setLogData(request.getMessage().getContent())
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private ByteString getStateMachineData(StateMachineLogEntryProto entryProto) {
|
private ByteString getStateMachineData(StateMachineLogEntryProto entryProto) {
|
||||||
|
|
|
@ -486,7 +486,8 @@ public void submitRequest(ContainerCommandRequestProto request,
|
||||||
super.submitRequest(request, pipelineID);
|
super.submitRequest(request, pipelineID);
|
||||||
RaftClientReply reply;
|
RaftClientReply reply;
|
||||||
try (Scope scope = TracingUtil
|
try (Scope scope = TracingUtil
|
||||||
.importAndCreateScope(request.getCmdType().name(),
|
.importAndCreateScope(
|
||||||
|
"XceiverServerRatis." + request.getCmdType().name(),
|
||||||
request.getTraceID())) {
|
request.getTraceID())) {
|
||||||
|
|
||||||
RaftClientRequest raftClientRequest =
|
RaftClientRequest raftClientRequest =
|
||||||
|
|
Loading…
Reference in New Issue