Add Trace Logging of REST Requests (#51684) (#52015)

Being able to trace log all REST requests to a node would make debugging
a number of issues a lot easier.
This commit is contained in:
Armin Braun 2020-02-07 09:03:20 +01:00 committed by GitHub
parent 25daf5f1e1
commit 91e938ead8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 460 additions and 84 deletions

View File

@ -108,3 +108,33 @@ client HTTP responses, defaults to unbounded.
It also uses the common
<<modules-network,network settings>>.
[float]
=== Rest Request Tracer
The http module has a dedicated tracer logger which, when activated, logs incoming requests. The log can be dynamically activated
by setting the level of the `org.elasticsearch.http.HttpTracer` logger to `TRACE`:
[source,console]
--------------------------------------------------
PUT _cluster/settings
{
"transient" : {
"logger.org.elasticsearch.http.HttpTracer" : "TRACE"
}
}
--------------------------------------------------
You can also control which uris will be traced, using a set of include and exclude wildcard patterns. By default every request will be
traced.
[source,console]
--------------------------------------------------
PUT _cluster/settings
{
"transient" : {
"http.tracer.include" : "*",
"http.tracer.exclude" : ""
}
}
--------------------------------------------------

View File

@ -152,7 +152,7 @@ request was uncompressed--even when compression is enabled.
=== Transport Tracer
The transport module has a dedicated tracer logger which, when activated, logs incoming and out going requests. The log can be dynamically activated
by settings the level of the `org.elasticsearch.transport.TransportService.tracer` logger to `TRACE`:
by setting the level of the `org.elasticsearch.transport.TransportService.tracer` logger to `TRACE`:
[source,console]
--------------------------------------------------

View File

@ -44,6 +44,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
@ -148,8 +149,8 @@ public class Netty4HttpServerTransport extends AbstractHttpServerTransport {
private volatile ServerBootstrap serverBootstrap;
public Netty4HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool,
NamedXContentRegistry xContentRegistry, Dispatcher dispatcher) {
super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher);
NamedXContentRegistry xContentRegistry, Dispatcher dispatcher, ClusterSettings clusterSettings) {
super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings);
Netty4Utils.setAvailableProcessors(EsExecutors.NODE_PROCESSORS_SETTING.get(settings));
this.maxChunkSize = SETTING_HTTP_MAX_CHUNK_SIZE.get(settings);

View File

@ -23,6 +23,7 @@ import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
@ -85,8 +86,10 @@ public class Netty4Plugin extends Plugin implements NetworkPlugin {
CircuitBreakerService circuitBreakerService,
NamedXContentRegistry xContentRegistry,
NetworkService networkService,
HttpServerTransport.Dispatcher dispatcher) {
HttpServerTransport.Dispatcher dispatcher,
ClusterSettings clusterSettings) {
return Collections.singletonMap(NETTY_HTTP_TRANSPORT_NAME,
() -> new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher));
() -> new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher,
clusterSettings));
}
}

View File

@ -23,6 +23,7 @@ import io.netty.handler.codec.http.FullHttpResponse;
import io.netty.util.ReferenceCounted;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.util.MockBigArrays;
@ -85,8 +86,8 @@ public class Netty4BadRequestTests extends ESTestCase {
}
};
try (HttpServerTransport httpServerTransport =
new Netty4HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher)) {
try (HttpServerTransport httpServerTransport = new Netty4HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool,
xContentRegistry(), dispatcher, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {
httpServerTransport.start();
final TransportAddress transportAddress = randomFrom(httpServerTransport.boundAddress().boundAddresses());

View File

@ -32,6 +32,7 @@ import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.util.ReferenceCounted;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.util.MockBigArrays;
@ -118,7 +119,7 @@ public class Netty4HttpServerPipeliningTests extends ESTestCase {
Netty4HttpServerPipeliningTests.this.networkService,
Netty4HttpServerPipeliningTests.this.bigArrays,
Netty4HttpServerPipeliningTests.this.threadPool,
xContentRegistry(), new NullDispatcher());
xContentRegistry(), new NullDispatcher(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
}
@Override

View File

@ -43,6 +43,7 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
@ -92,12 +93,14 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
private NetworkService networkService;
private ThreadPool threadPool;
private MockBigArrays bigArrays;
private ClusterSettings clusterSettings;
@Before
public void setup() throws Exception {
networkService = new NetworkService(Collections.emptyList());
threadPool = new TestThreadPool("test");
bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
}
@After
@ -108,6 +111,7 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
threadPool = null;
networkService = null;
bigArrays = null;
clusterSettings = null;
}
/**
@ -160,7 +164,7 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
}
};
try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool,
xContentRegistry(), dispatcher)) {
xContentRegistry(), dispatcher, clusterSettings)) {
transport.start();
final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
try (Netty4HttpClient client = new Netty4HttpClient()) {
@ -193,7 +197,7 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
public void testBindUnavailableAddress() {
try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool,
xContentRegistry(), new NullDispatcher())) {
xContentRegistry(), new NullDispatcher(), clusterSettings)) {
transport.start();
TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
Settings settings = Settings.builder()
@ -201,7 +205,7 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
.put("network.host", remoteAddress.getAddress())
.build();
try (Netty4HttpServerTransport otherTransport = new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool,
xContentRegistry(), new NullDispatcher())) {
xContentRegistry(), new NullDispatcher(), clusterSettings)) {
BindHttpException bindHttpException = expectThrows(BindHttpException.class, otherTransport::start);
assertEquals(
"Failed to bind to " + NetworkAddress.format(remoteAddress.address()),
@ -244,8 +248,8 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
settings = Settings.builder().put(httpMaxInitialLineLengthSetting.getKey(), maxInitialLineLength + "b").build();
}
try (Netty4HttpServerTransport transport =
new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher)) {
try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(
settings, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher, clusterSettings)) {
transport.start();
final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
@ -290,8 +294,8 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
.put(SETTING_CORS_ENABLED.getKey(), true)
.put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "elastic.co").build();
try (Netty4HttpServerTransport transport =
new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher)) {
try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool,
xContentRegistry(), dispatcher, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {
transport.start();
final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
@ -348,10 +352,9 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
.put(HttpTransportSettings.SETTING_HTTP_READ_TIMEOUT.getKey(), new TimeValue(randomIntBetween(100, 300)))
.build();
NioEventLoopGroup group = new NioEventLoopGroup();
try (Netty4HttpServerTransport transport =
new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher)) {
try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool,
xContentRegistry(), dispatcher, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {
transport.start();
final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());

View File

@ -25,6 +25,7 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.util.BigArrays;
@ -85,8 +86,8 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport {
public NioHttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays,
PageCacheRecycler pageCacheRecycler, ThreadPool threadPool, NamedXContentRegistry xContentRegistry,
Dispatcher dispatcher, NioGroupFactory nioGroupFactory) {
super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher);
Dispatcher dispatcher, NioGroupFactory nioGroupFactory, ClusterSettings clusterSettings) {
super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings);
this.pageAllocator = new PageAllocator(pageCacheRecycler);
this.nioGroupFactory = nioGroupFactory;

View File

@ -25,6 +25,7 @@ import org.apache.lucene.util.SetOnce;
import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
@ -86,10 +87,11 @@ public class NioTransportPlugin extends Plugin implements NetworkPlugin {
CircuitBreakerService circuitBreakerService,
NamedXContentRegistry xContentRegistry,
NetworkService networkService,
HttpServerTransport.Dispatcher dispatcher) {
HttpServerTransport.Dispatcher dispatcher,
ClusterSettings clusterSettings) {
return Collections.singletonMap(NIO_HTTP_TRANSPORT_NAME,
() -> new NioHttpServerTransport(settings, networkService, bigArrays, pageCacheRecycler, threadPool, xContentRegistry,
dispatcher, getNioGroupFactory(settings)));
dispatcher, getNioGroupFactory(settings), clusterSettings));
}
private synchronized NioGroupFactory getNioGroupFactory(Settings settings) {

View File

@ -35,6 +35,7 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
@ -155,7 +156,8 @@ public class NioHttpServerTransportTests extends ESTestCase {
}
};
try (NioHttpServerTransport transport = new NioHttpServerTransport(settings, networkService, bigArrays, pageRecycler, threadPool,
xContentRegistry(), dispatcher, new NioGroupFactory(settings, logger))) {
xContentRegistry(), dispatcher, new NioGroupFactory(settings, logger),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {
transport.start();
final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
try (NioHttpClient client = new NioHttpClient()) {
@ -188,7 +190,8 @@ public class NioHttpServerTransportTests extends ESTestCase {
public void testBindUnavailableAddress() {
try (NioHttpServerTransport transport = new NioHttpServerTransport(Settings.EMPTY, networkService, bigArrays, pageRecycler,
threadPool, xContentRegistry(), new NullDispatcher(), new NioGroupFactory(Settings.EMPTY, logger))) {
threadPool, xContentRegistry(), new NullDispatcher(), new NioGroupFactory(Settings.EMPTY, logger),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {
transport.start();
TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
Settings settings = Settings.builder()
@ -196,7 +199,8 @@ public class NioHttpServerTransportTests extends ESTestCase {
.put("network.host", remoteAddress.getAddress())
.build();
try (NioHttpServerTransport otherTransport = new NioHttpServerTransport(settings, networkService, bigArrays, pageRecycler,
threadPool, xContentRegistry(), new NullDispatcher(), new NioGroupFactory(Settings.EMPTY, logger))) {
threadPool, xContentRegistry(), new NullDispatcher(), new NioGroupFactory(Settings.EMPTY, logger),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {
BindHttpException bindHttpException = expectThrows(BindHttpException.class, () -> otherTransport.start());
assertEquals(
"Failed to bind to " + NetworkAddress.format(remoteAddress.address()),
@ -228,7 +232,8 @@ public class NioHttpServerTransportTests extends ESTestCase {
.put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "elastic.co").build();
try (NioHttpServerTransport transport = new NioHttpServerTransport(settings, networkService, bigArrays, pageRecycler,
threadPool, xContentRegistry(), dispatcher, new NioGroupFactory(settings, logger))) {
threadPool, xContentRegistry(), dispatcher, new NioGroupFactory(settings, logger),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {
transport.start();
final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
@ -298,7 +303,8 @@ public class NioHttpServerTransportTests extends ESTestCase {
}
try (NioHttpServerTransport transport = new NioHttpServerTransport(settings, networkService, bigArrays, pageRecycler,
threadPool, xContentRegistry(), dispatcher, new NioGroupFactory(settings, logger))) {
threadPool, xContentRegistry(), dispatcher, new NioGroupFactory(settings, logger),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {
transport.start();
final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
@ -343,9 +349,9 @@ public class NioHttpServerTransportTests extends ESTestCase {
.put(HttpTransportSettings.SETTING_HTTP_READ_TIMEOUT.getKey(), new TimeValue(randomIntBetween(100, 300)))
.build();
try (NioHttpServerTransport transport = new NioHttpServerTransport(settings, networkService, bigArrays, pageRecycler,
threadPool, xContentRegistry(), dispatcher, new NioGroupFactory(settings, logger))) {
threadPool, xContentRegistry(), dispatcher, new NioGroupFactory(settings, logger),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {
transport.start();
final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());

View File

@ -37,6 +37,7 @@ import org.elasticsearch.common.inject.ModulesBuilder;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsModule;
@ -196,7 +197,8 @@ public abstract class TransportClient extends AbstractClient {
BigArrays bigArrays = new BigArrays(pageCacheRecycler, circuitBreakerService, CircuitBreaker.REQUEST);
modules.add(settingsModule);
NetworkModule networkModule = new NetworkModule(settings, true, pluginsService.filterPlugins(NetworkPlugin.class), threadPool,
bigArrays, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, null);
bigArrays, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, null,
new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
final Transport transport = networkModule.getTransportSupplier().get();
final TransportService transportService = new TransportService(settings, transport, threadPool,
networkModule.getTransportInterceptor(),

View File

@ -30,6 +30,7 @@ import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
@ -114,12 +115,13 @@ public final class NetworkModule {
CircuitBreakerService circuitBreakerService,
NamedWriteableRegistry namedWriteableRegistry,
NamedXContentRegistry xContentRegistry,
NetworkService networkService, HttpServerTransport.Dispatcher dispatcher) {
NetworkService networkService, HttpServerTransport.Dispatcher dispatcher,
ClusterSettings clusterSettings) {
this.settings = settings;
this.transportClient = transportClient;
for (NetworkPlugin plugin : plugins) {
Map<String, Supplier<HttpServerTransport>> httpTransportFactory = plugin.getHttpTransports(settings, threadPool, bigArrays,
pageCacheRecycler, circuitBreakerService, xContentRegistry, networkService, dispatcher);
pageCacheRecycler, circuitBreakerService, xContentRegistry, networkService, dispatcher, clusterSettings);
if (transportClient == false) {
for (Map.Entry<String, Supplier<HttpServerTransport>> entry : httpTransportFactory.entrySet()) {
registerHttpTransport(entry.getKey(), entry.getValue());

View File

@ -293,6 +293,8 @@ public final class ClusterSettings extends AbstractScopedSettings {
HttpTransportSettings.SETTING_HTTP_TCP_REUSE_ADDRESS,
HttpTransportSettings.SETTING_HTTP_TCP_SEND_BUFFER_SIZE,
HttpTransportSettings.SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE,
HttpTransportSettings.SETTING_HTTP_TRACE_LOG_INCLUDE,
HttpTransportSettings.SETTING_HTTP_TRACE_LOG_EXCLUDE,
HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING,
HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING,
HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING,

View File

@ -31,6 +31,7 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.network.CloseableChannel;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.BoundTransportAddress;
import org.elasticsearch.common.transport.NetworkExceptionHelper;
@ -86,8 +87,10 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo
private final Set<HttpChannel> httpChannels = Collections.newSetFromMap(new ConcurrentHashMap<>());
private final Set<HttpServerChannel> httpServerChannels = Collections.newSetFromMap(new ConcurrentHashMap<>());
private final HttpTracer tracer;
protected AbstractHttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool,
NamedXContentRegistry xContentRegistry, Dispatcher dispatcher) {
NamedXContentRegistry xContentRegistry, Dispatcher dispatcher, ClusterSettings clusterSettings) {
this.settings = settings;
this.networkService = networkService;
this.bigArrays = bigArrays;
@ -109,6 +112,7 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo
this.port = SETTING_HTTP_PORT.get(settings);
this.maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings);
this.tracer = new HttpTracer(settings, clusterSettings);
}
@Override
@ -352,6 +356,8 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo
restRequest = innerRestRequest;
}
final HttpTracer trace = tracer.maybeTraceRequest(restRequest, exception);
/*
* We now want to create a channel used to send the response on. However, creating this channel can fail if there are invalid
* parameter values for any of the filter_path, human, or pretty parameters. We detect these specific failures via an
@ -363,11 +369,13 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo
RestChannel innerChannel;
ThreadContext threadContext = threadPool.getThreadContext();
try {
innerChannel = new DefaultRestChannel(httpChannel, httpRequest, restRequest, bigArrays, handlingSettings, threadContext);
innerChannel =
new DefaultRestChannel(httpChannel, httpRequest, restRequest, bigArrays, handlingSettings, threadContext, trace);
} catch (final IllegalArgumentException e) {
badRequestCause = ExceptionsHelper.useOrSuppress(badRequestCause, e);
final RestRequest innerRequest = RestRequest.requestWithoutParameters(xContentRegistry, httpRequest, httpChannel);
innerChannel = new DefaultRestChannel(httpChannel, httpRequest, innerRequest, bigArrays, handlingSettings, threadContext);
innerChannel =
new DefaultRestChannel(httpChannel, httpRequest, innerRequest, bigArrays, handlingSettings, threadContext, trace);
}
channel = innerChannel;
}

View File

@ -20,6 +20,7 @@
package org.elasticsearch.http;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
@ -60,14 +61,18 @@ public class DefaultRestChannel extends AbstractRestChannel implements RestChann
private final ThreadContext threadContext;
private final HttpChannel httpChannel;
@Nullable
private final HttpTracer tracerLog;
DefaultRestChannel(HttpChannel httpChannel, HttpRequest httpRequest, RestRequest request, BigArrays bigArrays,
HttpHandlingSettings settings, ThreadContext threadContext) {
HttpHandlingSettings settings, ThreadContext threadContext, @Nullable HttpTracer tracerLog) {
super(request, settings.getDetailedErrorsEnabled());
this.httpChannel = httpChannel;
this.httpRequest = httpRequest;
this.bigArrays = bigArrays;
this.settings = settings;
this.threadContext = threadContext;
this.tracerLog = tracerLog;
}
@Override
@ -84,6 +89,8 @@ public class DefaultRestChannel extends AbstractRestChannel implements RestChann
}
boolean success = false;
String opaque = null;
String contentLength = null;
try {
final BytesReference content = restResponse.content();
if (content instanceof Releasable) {
@ -105,7 +112,7 @@ public class DefaultRestChannel extends AbstractRestChannel implements RestChann
// TODO: Ideally we should move the setting of Cors headers into :server
// NioCorsHandler.setCorsResponseHeaders(nettyRequest, resp, corsConfig);
String opaque = request.header(X_OPAQUE_ID);
opaque = request.header(X_OPAQUE_ID);
if (opaque != null) {
setHeaderField(httpResponse, X_OPAQUE_ID, opaque);
}
@ -117,7 +124,8 @@ public class DefaultRestChannel extends AbstractRestChannel implements RestChann
// If our response doesn't specify a content-type header, set one
setHeaderField(httpResponse, CONTENT_TYPE, restResponse.contentType(), false);
// If our response has no content-length, calculate and set one
setHeaderField(httpResponse, CONTENT_LENGTH, String.valueOf(restResponse.content().length()), false);
contentLength = String.valueOf(restResponse.content().length());
setHeaderField(httpResponse, CONTENT_LENGTH, contentLength, false);
addCookies(httpResponse);
@ -133,6 +141,9 @@ public class DefaultRestChannel extends AbstractRestChannel implements RestChann
if (success == false) {
Releasables.close(toClose);
}
if (tracerLog != null) {
tracerLog.traceResponse(restResponse, httpChannel, contentLength, opaque, request.getRequestId(), success);
}
}
}

View File

@ -0,0 +1,98 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.transport.TransportService;
import java.util.List;
/**
* Http request trace logger. See {@link #maybeTraceRequest(RestRequest, Exception)} for details.
*/
class HttpTracer {
private final Logger logger = LogManager.getLogger(HttpTracer.class);
private volatile String[] tracerLogInclude;
private volatile String[] tracerLogExclude;
HttpTracer(Settings settings, ClusterSettings clusterSettings) {
setTracerLogInclude(HttpTransportSettings.SETTING_HTTP_TRACE_LOG_INCLUDE.get(settings));
setTracerLogExclude(HttpTransportSettings.SETTING_HTTP_TRACE_LOG_EXCLUDE.get(settings));
clusterSettings.addSettingsUpdateConsumer(HttpTransportSettings.SETTING_HTTP_TRACE_LOG_INCLUDE, this::setTracerLogInclude);
clusterSettings.addSettingsUpdateConsumer(HttpTransportSettings.SETTING_HTTP_TRACE_LOG_EXCLUDE, this::setTracerLogExclude);
}
/**
* Logs the given request if request tracing is enabled and the request uri matches the current include and exclude patterns defined
* in {@link HttpTransportSettings#SETTING_HTTP_TRACE_LOG_INCLUDE} and {@link HttpTransportSettings#SETTING_HTTP_TRACE_LOG_EXCLUDE}.
* If the request was logged returns a logger to log sending the response with or {@code null} otherwise.
*
* @param restRequest Rest request to trace
* @param e Exception when handling the request or {@code null} if none
* @return This instance to use for logging the response via {@link #traceResponse} to this request if it was logged or
* {@code null} if the request wasn't logged
*/
@Nullable
HttpTracer maybeTraceRequest(RestRequest restRequest, @Nullable Exception e) {
if (logger.isTraceEnabled() && TransportService.shouldTraceAction(restRequest.uri(), tracerLogInclude, tracerLogExclude)) {
logger.trace(new ParameterizedMessage("[{}][{}][{}][{}] received request from [{}]", restRequest.getRequestId(),
restRequest.header(Task.X_OPAQUE_ID), restRequest.method(), restRequest.uri(), restRequest.getHttpChannel()), e);
return this;
}
return null;
}
/**
* Logs the response to a request that was logged by {@link #maybeTraceRequest(RestRequest, Exception)}.
*
* @param restResponse RestResponse
* @param httpChannel HttpChannel the response was sent on
* @param contentLength Value of the response content length header
* @param opaqueHeader Value of HTTP header {@link Task#X_OPAQUE_ID}
* @param requestId Request id as returned by {@link RestRequest#getRequestId()}
* @param success Whether the response was successfully sent
*/
void traceResponse(RestResponse restResponse, HttpChannel httpChannel, String contentLength, String opaqueHeader, long requestId,
boolean success) {
logger.trace(new ParameterizedMessage("[{}][{}][{}][{}][{}] sent response to [{}] success [{}]", requestId,
opaqueHeader, restResponse.status(), restResponse.contentType(), contentLength, httpChannel, success));
}
private void setTracerLogInclude(List<String> tracerLogInclude) {
this.tracerLogInclude = tracerLogInclude.toArray(Strings.EMPTY_ARRAY);
}
private void setTracerLogExclude(List<String> tracerLogExclude) {
this.tracerLogExclude = tracerLogExclude.toArray(Strings.EMPTY_ARRAY);
}
}

View File

@ -28,6 +28,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import java.util.Collections;
import java.util.List;
import java.util.function.Function;
@ -127,6 +128,13 @@ public final class HttpTransportSettings {
public static final Setting<ByteSizeValue> SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE =
Setting.byteSizeSetting("http.tcp.receive_buffer_size", NetworkService.TCP_RECEIVE_BUFFER_SIZE, Setting.Property.NodeScope);
public static final Setting<List<String>> SETTING_HTTP_TRACE_LOG_INCLUDE =
Setting.listSetting("http.tracer.include", Collections.emptyList(),
Function.identity(), Setting.Property.Dynamic, Setting.Property.NodeScope);
public static final Setting<List<String>> SETTING_HTTP_TRACE_LOG_EXCLUDE =
Setting.listSetting("http.tracer.exclude",
Collections.emptyList(), Function.identity(), Setting.Property.Dynamic, Setting.Property.NodeScope);
private HttpTransportSettings() {
}
}

View File

@ -481,7 +481,7 @@ public class Node implements Closeable {
final RestController restController = actionModule.getRestController();
final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class),
threadPool, bigArrays, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, xContentRegistry,
networkService, restController);
networkService, restController, clusterService.getClusterSettings());
Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders =
pluginsService.filterPlugins(Plugin.class).stream()
.map(Plugin::getIndexTemplateMetaDataUpgrader)

View File

@ -25,6 +25,7 @@ import java.util.function.Supplier;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.PageCacheRecycler;
@ -73,7 +74,8 @@ public interface NetworkPlugin {
CircuitBreakerService circuitBreakerService,
NamedXContentRegistry xContentRegistry,
NetworkService networkService,
HttpServerTransport.Dispatcher dispatcher) {
HttpServerTransport.Dispatcher dispatcher,
ClusterSettings clusterSettings) {
return Collections.emptyMap();
}
}

View File

@ -47,6 +47,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@ -58,6 +59,8 @@ public class RestRequest implements ToXContent.Params {
// tchar pattern as defined by RFC7230 section 3.2.6
private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-z0-9!#$%&'*+\\-.\\^_`|~]+");
private static final AtomicLong requestIdGenerator = new AtomicLong();
private final NamedXContentRegistry xContentRegistry;
private final Map<String, String> params;
private final Map<String, List<String>> headers;
@ -70,12 +73,19 @@ public class RestRequest implements ToXContent.Params {
private boolean contentConsumed = false;
private final long requestId;
public boolean isContentConsumed() {
return contentConsumed;
}
protected RestRequest(NamedXContentRegistry xContentRegistry, Map<String, String> params, String path,
Map<String, List<String>> headers, HttpRequest httpRequest, HttpChannel httpChannel) {
this(xContentRegistry, params, path, headers, httpRequest, httpChannel, requestIdGenerator.incrementAndGet());
}
private RestRequest(NamedXContentRegistry xContentRegistry, Map<String, String> params, String path,
Map<String, List<String>> headers, HttpRequest httpRequest, HttpChannel httpChannel, long requestId) {
final XContentType xContentType;
try {
xContentType = parseContentType(headers.get("Content-Type"));
@ -91,11 +101,12 @@ public class RestRequest implements ToXContent.Params {
this.params = params;
this.rawPath = path;
this.headers = Collections.unmodifiableMap(headers);
this.requestId = requestId;
}
protected RestRequest(RestRequest restRequest) {
this(restRequest.getXContentRegistry(), restRequest.params(), restRequest.path(), restRequest.getHeaders(),
restRequest.getHttpRequest(), restRequest.getHttpChannel());
restRequest.getHttpRequest(), restRequest.getHttpChannel(), restRequest.getRequestId());
}
/**
@ -120,7 +131,8 @@ public class RestRequest implements ToXContent.Params {
public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRequest httpRequest, HttpChannel httpChannel) {
Map<String, String> params = params(httpRequest.uri());
String path = path(httpRequest.uri());
return new RestRequest(xContentRegistry, params, path, httpRequest.getHeaders(), httpRequest, httpChannel);
return new RestRequest(xContentRegistry, params, path, httpRequest.getHeaders(), httpRequest, httpChannel,
requestIdGenerator.incrementAndGet());
}
private static Map<String, String> params(final String uri) {
@ -157,7 +169,8 @@ public class RestRequest implements ToXContent.Params {
public static RestRequest requestWithoutParameters(NamedXContentRegistry xContentRegistry, HttpRequest httpRequest,
HttpChannel httpChannel) {
Map<String, String> params = Collections.emptyMap();
return new RestRequest(xContentRegistry, params, httpRequest.uri(), httpRequest.getHeaders(), httpRequest, httpChannel);
return new RestRequest(xContentRegistry, params, httpRequest.uri(), httpRequest.getHeaders(), httpRequest, httpChannel,
requestIdGenerator.incrementAndGet());
}
public enum Method {
@ -250,6 +263,10 @@ public class RestRequest implements ToXContent.Params {
return headers;
}
public final long getRequestId() {
return requestId;
}
/**
* The {@link XContentType} that was parsed from the {@code Content-Type} header. This value will be {@code null} in the case of
* a request without a valid {@code Content-Type} header, a request without content ({@link #hasContent()}, or a plain text request

View File

@ -790,13 +790,17 @@ public class TransportService extends AbstractLifecycleComponent implements Tran
}
private boolean shouldTraceAction(String action) {
if (tracerLogInclude.length > 0) {
if (Regex.simpleMatch(tracerLogInclude, action) == false) {
return shouldTraceAction(action, tracerLogInclude, tracerLogExclude);
}
public static boolean shouldTraceAction(String action, String[] include, String[] exclude) {
if (include.length > 0) {
if (Regex.simpleMatch(include, action) == false) {
return false;
}
}
if (tracerLogExclude.length > 0) {
return !Regex.simpleMatch(tracerLogExclude, action);
if (exclude.length > 0) {
return !Regex.simpleMatch(exclude, action);
}
return true;
}

View File

@ -21,6 +21,7 @@ package org.elasticsearch.common.network;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.BoundTransportAddress;
import org.elasticsearch.common.util.BigArrays;
@ -123,7 +124,8 @@ public class NetworkModuleTests extends ESTestCase {
CircuitBreakerService circuitBreakerService,
NamedXContentRegistry xContentRegistry,
NetworkService networkService,
HttpServerTransport.Dispatcher requestDispatcher) {
HttpServerTransport.Dispatcher requestDispatcher,
ClusterSettings clusterSettings) {
return Collections.singletonMap("custom", custom);
}
});
@ -162,7 +164,8 @@ public class NetworkModuleTests extends ESTestCase {
CircuitBreakerService circuitBreakerService,
NamedXContentRegistry xContentRegistry,
NetworkService networkService,
HttpServerTransport.Dispatcher requestDispatcher) {
HttpServerTransport.Dispatcher requestDispatcher,
ClusterSettings clusterSettings) {
Map<String, Supplier<HttpServerTransport>> supplierMap = new HashMap<>();
supplierMap.put("custom", custom);
supplierMap.put("default_custom", def);
@ -197,7 +200,8 @@ public class NetworkModuleTests extends ESTestCase {
CircuitBreakerService circuitBreakerService,
NamedXContentRegistry xContentRegistry,
NetworkService networkService,
HttpServerTransport.Dispatcher requestDispatcher) {
HttpServerTransport.Dispatcher requestDispatcher,
ClusterSettings clusterSettings) {
Map<String, Supplier<HttpServerTransport>> supplierMap = new HashMap<>();
supplierMap.put("custom", custom);
supplierMap.put("default_custom", def);
@ -262,6 +266,7 @@ public class NetworkModuleTests extends ESTestCase {
private NetworkModule newNetworkModule(Settings settings, boolean transportClient, NetworkPlugin... plugins) {
return new NetworkModule(settings, transportClient, Arrays.asList(plugins), threadPool, null, null, null, null,
xContentRegistry(), null, new NullDispatcher());
xContentRegistry(), null, new NullDispatcher(),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
}
}

View File

@ -19,17 +19,31 @@
package org.elasticsearch.http;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.network.NetworkUtils;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.util.MockBigArrays;
import org.elasticsearch.common.util.MockPageCacheRecycler;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.rest.RestChannel;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.MockLogAppender;
import org.elasticsearch.test.junit.annotations.TestLogging;
import org.elasticsearch.test.rest.FakeRestRequest;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.junit.After;
@ -125,7 +139,8 @@ public class AbstractHttpServerTransportTests extends ESTestCase {
};
try (AbstractHttpServerTransport transport =
new AbstractHttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher) {
new AbstractHttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher,
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) {
@Override
protected HttpServerChannel bind(InetSocketAddress hostAddress) {
@ -158,6 +173,137 @@ public class AbstractHttpServerTransportTests extends ESTestCase {
}
}
@TestLogging(
value = "org.elasticsearch.http.HttpTracer:trace",
reason = "to ensure we log REST requests on TRACE level")
public void testTracerLog() throws Exception {
final String includeSettings;
final String excludeSettings;
if (randomBoolean()) {
includeSettings = randomBoolean() ? "*" : "";
} else {
includeSettings = "/internal/test";
}
excludeSettings = "/internal/testNotSeen";
final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
try (AbstractHttpServerTransport transport =
new AbstractHttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool, xContentRegistry(),
new HttpServerTransport.Dispatcher() {
@Override
public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
channel.sendResponse(emptyResponse(RestStatus.OK));
}
@Override
public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, Throwable cause) {
channel.sendResponse(emptyResponse(RestStatus.BAD_REQUEST));
}
}, clusterSettings) {
@Override
protected HttpServerChannel bind(InetSocketAddress hostAddress) {
return null;
}
@Override
protected void doStart() {
}
@Override
protected void stopInternal() {
}
@Override
public HttpStats stats() {
return null;
}
}) {
clusterSettings.applySettings(Settings.builder()
.put(HttpTransportSettings.SETTING_HTTP_TRACE_LOG_INCLUDE.getKey(), includeSettings)
.put(HttpTransportSettings.SETTING_HTTP_TRACE_LOG_EXCLUDE.getKey(), excludeSettings)
.build());
MockLogAppender appender = new MockLogAppender();
final String traceLoggerName = "org.elasticsearch.http.HttpTracer";
try {
appender.start();
Loggers.addAppender(LogManager.getLogger(traceLoggerName), appender);
final String opaqueId = UUIDs.randomBase64UUID(random());
appender.addExpectation(
new MockLogAppender.PatternSeenEventExpectation(
"received request", traceLoggerName, Level.TRACE,
"\\[\\d+\\]\\[" + opaqueId + "\\]\\[OPTIONS\\]\\[/internal/test\\] received request from \\[.*"));
final boolean badRequest = randomBoolean();
appender.addExpectation(
new MockLogAppender.PatternSeenEventExpectation(
"sent response", traceLoggerName, Level.TRACE,
"\\[\\d+\\]\\[" + opaqueId + "\\]\\[" +
(badRequest ? "BAD_REQUEST" : "OK")
+ "\\]\\[null\\]\\[0\\] sent response to \\[.*"));
appender.addExpectation(
new MockLogAppender.UnseenEventExpectation(
"received other request", traceLoggerName, Level.TRACE,
"\\[\\d+\\]\\[" + opaqueId + "\\]\\[OPTIONS\\]\\[/internal/testNotSeen\\] received request from \\[.*"));
final FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
.withMethod(RestRequest.Method.OPTIONS)
.withPath("/internal/test")
.withHeaders(Collections.singletonMap(Task.X_OPAQUE_ID, Collections.singletonList(opaqueId)))
.build();
if (badRequest) {
transport.incomingRequestError(fakeRestRequest.getHttpRequest(), fakeRestRequest.getHttpChannel(),
new RuntimeException());
} else {
transport.incomingRequest(fakeRestRequest.getHttpRequest(), fakeRestRequest.getHttpChannel());
}
final FakeRestRequest fakeRestRequestExcludedPath = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
.withMethod(RestRequest.Method.OPTIONS)
.withPath("/internal/testNotSeen")
.withHeaders(Collections.singletonMap(Task.X_OPAQUE_ID, Collections.singletonList(opaqueId)))
.build();
if (randomBoolean()) {
transport.incomingRequest(fakeRestRequestExcludedPath.getHttpRequest(), fakeRestRequestExcludedPath.getHttpChannel());
} else {
transport.incomingRequestError(
fakeRestRequestExcludedPath.getHttpRequest(), fakeRestRequestExcludedPath.getHttpChannel(),
new RuntimeException());
}
appender.assertAllExpectationsMatched();
} finally {
Loggers.removeAppender(LogManager.getLogger(traceLoggerName), appender);
appender.stop();
}
}
}
private static RestResponse emptyResponse(RestStatus status) {
return new RestResponse() {
@Override
public String contentType() {
return null;
}
@Override
public BytesReference content() {
return BytesArray.EMPTY;
}
@Override
public RestStatus status() {
return status;
}
};
}
private TransportAddress address(String host, int port) throws UnknownHostException {
return new TransportAddress(getByName(host), port);
}

View File

@ -193,7 +193,7 @@ public class DefaultRestChannelTests extends ESTestCase {
// send a response
DefaultRestChannel channel = new DefaultRestChannel(httpChannel, httpRequest, request, bigArrays, handlingSettings,
threadPool.getThreadContext());
threadPool.getThreadContext(), null);
TestRestResponse resp = new TestRestResponse();
final String customHeader = "custom-header";
final String customHeaderValue = "xyz";
@ -221,7 +221,7 @@ public class DefaultRestChannelTests extends ESTestCase {
// send a response
DefaultRestChannel channel = new DefaultRestChannel(httpChannel, httpRequest, request, bigArrays, handlingSettings,
threadPool.getThreadContext());
threadPool.getThreadContext(), null);
channel.sendResponse(new TestRestResponse());
// inspect what was written
@ -241,7 +241,7 @@ public class DefaultRestChannelTests extends ESTestCase {
HttpHandlingSettings handlingSettings = HttpHandlingSettings.fromSettings(settings);
DefaultRestChannel channel = new DefaultRestChannel(httpChannel, httpRequest, request, bigArrays, handlingSettings,
threadPool.getThreadContext());
threadPool.getThreadContext(), null);
final BytesRestResponse response = new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR,
JsonXContent.contentBuilder().startObject().endObject());
assertThat(response.content(), not(instanceOf(Releasable.class)));
@ -296,7 +296,7 @@ public class DefaultRestChannelTests extends ESTestCase {
HttpHandlingSettings handlingSettings = HttpHandlingSettings.fromSettings(settings);
DefaultRestChannel channel = new DefaultRestChannel(httpChannel, httpRequest, request, bigArrays, handlingSettings,
threadPool.getThreadContext());
threadPool.getThreadContext(), null);
channel.sendResponse(new TestRestResponse());
Class<ActionListener<Void>> listenerClass = (Class<ActionListener<Void>>) (Class) ActionListener.class;
ArgumentCaptor<ActionListener<Void>> listenerCaptor = ArgumentCaptor.forClass(listenerClass);
@ -327,7 +327,7 @@ public class DefaultRestChannelTests extends ESTestCase {
request.getHttpRequest().getHeaders().put(DefaultRestChannel.CONNECTION, Collections.singletonList(httpConnectionHeaderValue));
DefaultRestChannel channel = new DefaultRestChannel(httpChannel, request.getHttpRequest(), request, bigArrays,
HttpHandlingSettings.fromSettings(Settings.EMPTY), threadPool.getThreadContext());
HttpHandlingSettings.fromSettings(Settings.EMPTY), threadPool.getThreadContext(), null);
// ESTestCase#after will invoke ensureAllArraysAreReleased which will fail if the response content was not released
final BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
@ -364,7 +364,7 @@ public class DefaultRestChannelTests extends ESTestCase {
request.getHttpRequest().getHeaders().put(DefaultRestChannel.CONNECTION, Collections.singletonList(httpConnectionHeaderValue));
DefaultRestChannel channel = new DefaultRestChannel(httpChannel, request.getHttpRequest(), request, bigArrays,
HttpHandlingSettings.fromSettings(Settings.EMPTY), threadPool.getThreadContext());
HttpHandlingSettings.fromSettings(Settings.EMPTY), threadPool.getThreadContext(), null);
// ESTestCase#after will invoke ensureAllArraysAreReleased which will fail if the response content was not released
final BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
@ -395,7 +395,7 @@ public class DefaultRestChannelTests extends ESTestCase {
HttpHandlingSettings httpHandlingSettings = HttpHandlingSettings.fromSettings(settings);
RestChannel channel = new DefaultRestChannel(httpChannel, httpRequest, request, bigArrays, httpHandlingSettings,
threadPool.getThreadContext());
threadPool.getThreadContext(), null);
channel.sendResponse(new TestRestResponse());
// get the response

View File

@ -630,12 +630,12 @@ public class RestControllerTests extends ESTestCase {
}
}
private static final class AssertingChannel extends AbstractRestChannel {
public static final class AssertingChannel extends AbstractRestChannel {
private final RestStatus expectedStatus;
private final AtomicReference<RestResponse> responseReference = new AtomicReference<>();
protected AssertingChannel(RestRequest request, boolean detailedErrorsEnabled, RestStatus expectedStatus) {
public AssertingChannel(RestRequest request, boolean detailedErrorsEnabled, RestStatus expectedStatus) {
super(request, detailedErrorsEnabled);
this.expectedStatus = expectedStatus;
}

View File

@ -302,10 +302,11 @@ public class LocalStateCompositeXPackPlugin extends XPackPlugin implements Scrip
CircuitBreakerService circuitBreakerService,
NamedXContentRegistry xContentRegistry,
NetworkService networkService,
HttpServerTransport.Dispatcher dispatcher) {
HttpServerTransport.Dispatcher dispatcher,
ClusterSettings clusterSettings) {
Map<String, Supplier<HttpServerTransport>> transports = new HashMap<>();
filterPlugins(NetworkPlugin.class).stream().forEach(p -> transports.putAll(p.getHttpTransports(settings, threadPool, bigArrays,
pageCacheRecycler, circuitBreakerService, xContentRegistry, networkService, dispatcher)));
pageCacheRecycler, circuitBreakerService, xContentRegistry, networkService, dispatcher, clusterSettings)));
return transports;
}

View File

@ -969,16 +969,18 @@ public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin,
CircuitBreakerService circuitBreakerService,
NamedXContentRegistry xContentRegistry,
NetworkService networkService,
HttpServerTransport.Dispatcher dispatcher) {
HttpServerTransport.Dispatcher dispatcher,
ClusterSettings clusterSettings) {
if (enabled == false) { // don't register anything if we are not enabled
return Collections.emptyMap();
}
Map<String, Supplier<HttpServerTransport>> httpTransports = new HashMap<>();
httpTransports.put(SecurityField.NAME4, () -> new SecurityNetty4HttpServerTransport(settings, networkService, bigArrays,
ipFilter.get(), getSslService(), threadPool, xContentRegistry, dispatcher));
ipFilter.get(), getSslService(), threadPool, xContentRegistry, dispatcher, clusterSettings));
httpTransports.put(SecurityField.NIO, () -> new SecurityNioHttpServerTransport(settings, networkService, bigArrays,
pageCacheRecycler, threadPool, xContentRegistry, dispatcher, ipFilter.get(), getSslService(), getNioGroupFactory(settings)));
pageCacheRecycler, threadPool, xContentRegistry, dispatcher, ipFilter.get(), getSslService(), getNioGroupFactory(settings),
clusterSettings));
return httpTransports;
}

View File

@ -11,6 +11,7 @@ import io.netty.handler.ssl.SslHandler;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@ -36,8 +37,8 @@ public class SecurityNetty4HttpServerTransport extends Netty4HttpServerTransport
public SecurityNetty4HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, IPFilter ipFilter,
SSLService sslService, ThreadPool threadPool, NamedXContentRegistry xContentRegistry,
Dispatcher dispatcher) {
super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher);
Dispatcher dispatcher, ClusterSettings clusterSettings) {
super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings);
this.securityExceptionHandler = new SecurityHttpExceptionHandler(logger, lifecycle, (c, e) -> super.onException(c, e));
this.ipFilter = ipFilter;
final boolean ssl = HTTP_SSL_ENABLED.get(settings);

View File

@ -8,6 +8,7 @@ package org.elasticsearch.xpack.security.transport.nio;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.PageCacheRecycler;
@ -53,8 +54,10 @@ public class SecurityNioHttpServerTransport extends NioHttpServerTransport {
public SecurityNioHttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays,
PageCacheRecycler pageCacheRecycler, ThreadPool threadPool,
NamedXContentRegistry xContentRegistry, Dispatcher dispatcher, IPFilter ipFilter,
SSLService sslService, NioGroupFactory nioGroupFactory) {
super(settings, networkService, bigArrays, pageCacheRecycler, threadPool, xContentRegistry, dispatcher, nioGroupFactory);
SSLService sslService, NioGroupFactory nioGroupFactory,
ClusterSettings clusterSettings) {
super(settings, networkService, bigArrays, pageCacheRecycler, threadPool, xContentRegistry, dispatcher, nioGroupFactory,
clusterSettings);
this.securityExceptionHandler = new SecurityHttpExceptionHandler(logger, lifecycle, (c, e) -> super.onException(c, e));
this.ipFilter = ipFilter;
this.sslEnabled = HTTP_SSL_ENABLED.get(settings);

View File

@ -9,6 +9,7 @@ import io.netty.channel.ChannelHandler;
import io.netty.channel.embedded.EmbeddedChannel;
import io.netty.handler.ssl.SslHandler;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
@ -69,7 +70,8 @@ public class SecurityNetty4HttpServerTransportTests extends ESTestCase {
sslService = new SSLService(settings, env);
SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(IPFilter.class), sslService,
mock(ThreadPool.class), xContentRegistry(), new NullDispatcher());
mock(ThreadPool.class), xContentRegistry(), new NullDispatcher(),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
ChannelHandler handler = transport.configureServerChannelHandler();
final EmbeddedChannel ch = new EmbeddedChannel(handler);
assertThat(ch.pipeline().get(SslHandler.class).engine().getNeedClientAuth(), is(false));
@ -85,7 +87,8 @@ public class SecurityNetty4HttpServerTransportTests extends ESTestCase {
sslService = new SSLService(settings, env);
SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(IPFilter.class), sslService,
mock(ThreadPool.class), xContentRegistry(), new NullDispatcher());
mock(ThreadPool.class), xContentRegistry(), new NullDispatcher(),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
ChannelHandler handler = transport.configureServerChannelHandler();
final EmbeddedChannel ch = new EmbeddedChannel(handler);
assertThat(ch.pipeline().get(SslHandler.class).engine().getNeedClientAuth(), is(false));
@ -101,7 +104,8 @@ public class SecurityNetty4HttpServerTransportTests extends ESTestCase {
sslService = new SSLService(settings, env);
SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(IPFilter.class), sslService,
mock(ThreadPool.class), xContentRegistry(), new NullDispatcher());
mock(ThreadPool.class), xContentRegistry(), new NullDispatcher(),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
ChannelHandler handler = transport.configureServerChannelHandler();
final EmbeddedChannel ch = new EmbeddedChannel(handler);
assertThat(ch.pipeline().get(SslHandler.class).engine().getNeedClientAuth(), is(true));
@ -117,7 +121,8 @@ public class SecurityNetty4HttpServerTransportTests extends ESTestCase {
sslService = new SSLService(settings, env);
SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(IPFilter.class), sslService,
mock(ThreadPool.class), xContentRegistry(), new NullDispatcher());
mock(ThreadPool.class), xContentRegistry(), new NullDispatcher(),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
ChannelHandler handler = transport.configureServerChannelHandler();
final EmbeddedChannel ch = new EmbeddedChannel(handler);
assertThat(ch.pipeline().get(SslHandler.class).engine().getNeedClientAuth(), is(false));
@ -131,7 +136,8 @@ public class SecurityNetty4HttpServerTransportTests extends ESTestCase {
sslService = new SSLService(settings, env);
SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(IPFilter.class), sslService,
mock(ThreadPool.class), xContentRegistry(), new NullDispatcher());
mock(ThreadPool.class), xContentRegistry(), new NullDispatcher(),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
ChannelHandler handler = transport.configureServerChannelHandler();
EmbeddedChannel ch = new EmbeddedChannel(handler);
SSLEngine defaultEngine = ch.pipeline().get(SslHandler.class).engine();
@ -143,7 +149,8 @@ public class SecurityNetty4HttpServerTransportTests extends ESTestCase {
.build();
sslService = new SSLService(settings, TestEnvironment.newEnvironment(settings));
transport = new SecurityNetty4HttpServerTransport(settings, new NetworkService(Collections.emptyList()),
mock(BigArrays.class), mock(IPFilter.class), sslService, mock(ThreadPool.class), xContentRegistry(), new NullDispatcher());
mock(BigArrays.class), mock(IPFilter.class), sslService, mock(ThreadPool.class), xContentRegistry(), new NullDispatcher(),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
handler = transport.configureServerChannelHandler();
ch = new EmbeddedChannel(handler);
SSLEngine customEngine = ch.pipeline().get(SslHandler.class).engine();
@ -169,7 +176,8 @@ public class SecurityNetty4HttpServerTransportTests extends ESTestCase {
sslService = new SSLService(settings, env);
SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(IPFilter.class), sslService,
mock(ThreadPool.class), xContentRegistry(), new NullDispatcher());
mock(ThreadPool.class), xContentRegistry(), new NullDispatcher(),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
assertNotNull(transport.configureServerChannelHandler());
}
}

View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.security.transport.nio;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
@ -78,7 +79,8 @@ public class SecurityNioHttpServerTransportTests extends ESTestCase {
sslService = new SSLService(settings, env);
SecurityNioHttpServerTransport transport = new SecurityNioHttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory);
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory,
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
SecurityNioHttpServerTransport.SecurityHttpChannelFactory factory = transport.channelFactory();
SocketChannel socketChannel = mock(SocketChannel.class);
when(socketChannel.getRemoteAddress()).thenReturn(address);
@ -99,7 +101,8 @@ public class SecurityNioHttpServerTransportTests extends ESTestCase {
nioGroupFactory = new NioGroupFactory(settings, logger);
SecurityNioHttpServerTransport transport = new SecurityNioHttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory);
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory,
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
SecurityNioHttpServerTransport.SecurityHttpChannelFactory factory = transport.channelFactory();
SocketChannel socketChannel = mock(SocketChannel.class);
@ -120,7 +123,8 @@ public class SecurityNioHttpServerTransportTests extends ESTestCase {
sslService = new SSLService(settings, env);
SecurityNioHttpServerTransport transport = new SecurityNioHttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory);
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory,
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
SecurityNioHttpServerTransport.SecurityHttpChannelFactory factory = transport.channelFactory();
SocketChannel socketChannel = mock(SocketChannel.class);
@ -141,7 +145,8 @@ public class SecurityNioHttpServerTransportTests extends ESTestCase {
nioGroupFactory = new NioGroupFactory(settings, logger);
SecurityNioHttpServerTransport transport = new SecurityNioHttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory);
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory,
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
SecurityNioHttpServerTransport.SecurityHttpChannelFactory factory = transport.channelFactory();
SocketChannel socketChannel = mock(SocketChannel.class);
@ -160,7 +165,8 @@ public class SecurityNioHttpServerTransportTests extends ESTestCase {
nioGroupFactory = new NioGroupFactory(settings, logger);
SecurityNioHttpServerTransport transport = new SecurityNioHttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory);
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory,
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
SecurityNioHttpServerTransport.SecurityHttpChannelFactory factory = transport.channelFactory();
SocketChannel socketChannel = mock(SocketChannel.class);
when(socketChannel.getRemoteAddress()).thenReturn(address);
@ -176,7 +182,8 @@ public class SecurityNioHttpServerTransportTests extends ESTestCase {
nioGroupFactory = new NioGroupFactory(settings, logger);
transport = new SecurityNioHttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory);
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory,
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
factory = transport.channelFactory();
channel = factory.createChannel(mock(NioSelector.class), socketChannel, mock(Config.Socket.class));
SSLEngine customEngine = SSLEngineUtils.getSSLEngine(channel);
@ -203,6 +210,7 @@ public class SecurityNioHttpServerTransportTests extends ESTestCase {
nioGroupFactory = new NioGroupFactory(settings, logger);
SecurityNioHttpServerTransport transport = new SecurityNioHttpServerTransport(settings,
new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory);
xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService, nioGroupFactory,
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
}
}