honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'elastic/master' into ccr 

* elastic/master:
  [DOCS] Creates rest-api folder in docs
  [Rollup] Disallow index patterns that match the rollup index (#30491)
  Add cors support to NioHttpServerTransport (#30827)
  [DOCS] Fixes security example (#31082)
  Allow terms query in _rollup_search (#30973)
This commit is contained in:
Jason Tedor 2018-06-05 19:53:37 -04:00
parent 755a25a41e 7c05f69c39
commit 6e109e90c3
No known key found for this signature in database
GPG Key ID: FA89F05560F16BC5
23 changed files with 1526 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/reference/index.asciidoc
View File

@ -63,7 +63,7 @@ include::{xes-repo-dir}/monitoring/index.asciidoc[]
include::{xes-repo-dir}/rollup/index.asciidoc[]
include::{xes-repo-dir}/rest-api/index.asciidoc[]
include::rest-api/index.asciidoc[]
include::{xes-repo-dir}/commands/index.asciidoc[]

29
docs/reference/rest-api/index.asciidoc Normal file
View File

@ -0,0 +1,29 @@
[role="xpack"]
[[xpack-api]]
= {xpack} APIs
[partintro]
--
{xpack} exposes REST APIs that are used by the UI components and can be called
directly to configure and access {xpack} features.
* <<info-api,Info API>>
* <<graph-explore-api,Graph Explore API>>
* <<licensing-apis,Licensing APIs>>
* <<ml-apis,Machine Learning APIs>>
* <<security-api,Security APIs>>
* <<watcher-api,Watcher APIs>>
* <<rollup-apis,Rollup APIs>>
* <<migration-api,Migration APIs>>
--
include::{xes-repo-dir}/rest-api/info.asciidoc[]
include::{xes-repo-dir}/rest-api/graph/explore.asciidoc[]
include::{xes-repo-dir}/rest-api/licensing.asciidoc[]
include::{xes-repo-dir}/rest-api/migration.asciidoc[]
include::{xes-repo-dir}/rest-api/ml-api.asciidoc[]
include::{xes-repo-dir}/rest-api/rollup-api.asciidoc[]
include::{xes-repo-dir}/rest-api/security.asciidoc[]
include::{xes-repo-dir}/rest-api/watcher.asciidoc[]
include::{xes-repo-dir}/rest-api/defs.asciidoc[]

19
modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfig.java
View File

@ -76,7 +76,8 @@ public final class Netty4CorsConfig {
}
/**
* Determines whether a wildcard origin, '*', is supported.
* Determines whether a wildcard origin, '*', is supported. This also means that null origins are
* supported.
*
* @return {@code boolean} true if any origin is allowed.
*/
@ -121,21 +122,21 @@ public final class Netty4CorsConfig {
}
/**
* Determines if cookies are supported for CORS requests.
* Determines if credentials are supported for CORS requests.
*
* By default cookies are not included in CORS requests but if isCredentialsAllowed returns
* true cookies will be added to CORS requests. Setting this value to true will set the
* By default credentials are not included in CORS requests but if isCredentialsAllowed returns
* true credentials will be added to CORS requests. Setting this value to true will set the
* CORS 'Access-Control-Allow-Credentials' response header to true.
*
* Please note that cookie support needs to be enabled on the client side as well.
* The client needs to opt-in to send cookies by calling:
* Please note that credentials support needs to be enabled on the client side as well.
* The client needs to opt-in to send credentials by calling:
* <pre>
* xhr.withCredentials = true;
* </pre>
* The default value for 'withCredentials' is false in which case no cookies are sent.
* Setting this to true will included cookies in cross origin requests.
* The default value for 'withCredentials' is false in which case no credentials are sent.
* Setting this to true will included credentials in cross origin requests.
*
* @return {@code true} if cookies are supported.
* @return {@code true} if credentials are supported.
*/
public boolean isCredentialsAllowed() {
return allowCredentials;

15
plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/HttpReadWriteHandler.java
View File

@ -36,6 +36,8 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.http.HttpHandlingSettings;
import org.elasticsearch.http.HttpPipelinedRequest;
import org.elasticsearch.http.nio.cors.NioCorsConfig;
import org.elasticsearch.http.nio.cors.NioCorsHandler;
import org.elasticsearch.nio.FlushOperation;
import org.elasticsearch.nio.InboundChannelBuffer;
import org.elasticsearch.nio.NioSocketChannel;
@ -50,6 +52,8 @@ import java.util.Collections;
import java.util.List;
import java.util.function.BiConsumer;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
public class HttpReadWriteHandler implements ReadWriteHandler {
private final NettyAdaptor adaptor;
@ -57,14 +61,16 @@ public class HttpReadWriteHandler implements ReadWriteHandler {
private final NioHttpServerTransport transport;
private final HttpHandlingSettings settings;
private final NamedXContentRegistry xContentRegistry;
private final NioCorsConfig corsConfig;
private final ThreadContext threadContext;
HttpReadWriteHandler(NioSocketChannel nioChannel, NioHttpServerTransport transport, HttpHandlingSettings settings,
NamedXContentRegistry xContentRegistry, ThreadContext threadContext) {
NamedXContentRegistry xContentRegistry, NioCorsConfig corsConfig, ThreadContext threadContext) {
this.nioChannel = nioChannel;
this.transport = transport;
this.settings = settings;
this.xContentRegistry = xContentRegistry;
this.corsConfig = corsConfig;
this.threadContext = threadContext;
List<ChannelHandler> handlers = new ArrayList<>(5);
@ -78,6 +84,9 @@ public class HttpReadWriteHandler implements ReadWriteHandler {
if (settings.isCompression()) {
handlers.add(new HttpContentCompressor(settings.getCompressionLevel()));
}
if (settings.isCorsEnabled()) {
handlers.add(new NioCorsHandler(corsConfig));
}
handlers.add(new NioHttpPipeliningHandler(transport.getLogger(), settings.getPipeliningMaxEvents()));
adaptor = new NettyAdaptor(handlers.toArray(new ChannelHandler[0]));
@ -178,7 +187,7 @@ public class HttpReadWriteHandler implements ReadWriteHandler {
int sequence = pipelinedRequest.getSequence();
BigArrays bigArrays = transport.getBigArrays();
try {
innerChannel = new NioHttpChannel(nioChannel, bigArrays, httpRequest, sequence, settings, threadContext);
innerChannel = new NioHttpChannel(nioChannel, bigArrays, httpRequest, sequence, settings, corsConfig, threadContext);
} catch (final IllegalArgumentException e) {
if (badRequestCause == null) {
badRequestCause = e;
@ -191,7 +200,7 @@ public class HttpReadWriteHandler implements ReadWriteHandler {
Collections.emptyMap(), // we are going to dispatch the request as a bad request, drop all parameters
copiedRequest.uri(),
copiedRequest);
innerChannel = new NioHttpChannel(nioChannel, bigArrays, innerRequest, sequence, settings, threadContext);
innerChannel = new NioHttpChannel(nioChannel, bigArrays, innerRequest, sequence, settings, corsConfig, threadContext);
}
channel = innerChannel;
}

8
plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpChannel.java
View File

@ -41,6 +41,8 @@ import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.http.HttpHandlingSettings;
import org.elasticsearch.http.nio.cors.NioCorsConfig;
import org.elasticsearch.http.nio.cors.NioCorsHandler;
import org.elasticsearch.nio.NioSocketChannel;
import org.elasticsearch.rest.AbstractRestChannel;
import org.elasticsearch.rest.RestResponse;
@ -58,17 +60,19 @@ public class NioHttpChannel extends AbstractRestChannel {
private final BigArrays bigArrays;
private final int sequence;
private final NioCorsConfig corsConfig;
private final ThreadContext threadContext;
private final FullHttpRequest nettyRequest;
private final NioSocketChannel nioChannel;
private final boolean resetCookies;
NioHttpChannel(NioSocketChannel nioChannel, BigArrays bigArrays, NioHttpRequest request, int sequence,
HttpHandlingSettings settings, ThreadContext threadContext) {
HttpHandlingSettings settings, NioCorsConfig corsConfig, ThreadContext threadContext) {
super(request, settings.getDetailedErrorsEnabled());
this.nioChannel = nioChannel;
this.bigArrays = bigArrays;
this.sequence = sequence;
this.corsConfig = corsConfig;
this.threadContext = threadContext;
this.nettyRequest = request.getRequest();
this.resetCookies = settings.isResetCookies();
@ -87,6 +91,8 @@ public class NioHttpChannel extends AbstractRestChannel {
}
resp.setStatus(getStatus(response.status()));
NioCorsHandler.setCorsResponseHeaders(nettyRequest, resp, corsConfig);
String opaque = nettyRequest.headers().get("X-Opaque-Id");
if (opaque != null) {
setHeaderField(resp, "X-Opaque-Id", opaque);

52
plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.http.nio;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.timeout.ReadTimeoutException;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
@ -28,6 +29,7 @@ import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.Setting;
@ -38,11 +40,13 @@ import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.http.AbstractHttpServerTransport;
import org.elasticsearch.http.BindHttpException;
import org.elasticsearch.http.HttpHandlingSettings;
import org.elasticsearch.http.HttpServerTransport;
import org.elasticsearch.http.HttpStats;
import org.elasticsearch.http.AbstractHttpServerTransport;
import org.elasticsearch.http.nio.cors.NioCorsConfig;
import org.elasticsearch.http.nio.cors.NioCorsConfigBuilder;
import org.elasticsearch.nio.AcceptingSelector;
import org.elasticsearch.nio.AcceptorEventHandler;
import org.elasticsearch.nio.BytesChannelContext;
@ -56,6 +60,7 @@ import org.elasticsearch.nio.ServerChannelContext;
import org.elasticsearch.nio.SocketChannelContext;
import org.elasticsearch.nio.SocketEventHandler;
import org.elasticsearch.nio.SocketSelector;
import org.elasticsearch.rest.RestUtils;
import org.elasticsearch.threadpool.ThreadPool;
import java.io.IOException;
@ -64,15 +69,23 @@ import java.net.InetSocketAddress;
import java.nio.channels.ServerSocketChannel;
import java.nio.channels.SocketChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.regex.Pattern;
import static org.elasticsearch.common.settings.Setting.intSetting;
import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION_LEVEL;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED;
@ -86,6 +99,7 @@ import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_TCP_RECE
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_TCP_REUSE_ADDRESS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_TCP_SEND_BUFFER_SIZE;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS;
import static org.elasticsearch.http.nio.cors.NioCorsHandler.ANY_ORIGIN;
public class NioHttpServerTransport extends AbstractHttpServerTransport {
@ -115,6 +129,7 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport {
private final Set<NioSocketChannel> socketChannels = Collections.newSetFromMap(new ConcurrentHashMap<>());
private NioGroup nioGroup;
private HttpChannelFactory channelFactory;
private final NioCorsConfig corsConfig;
public NioHttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool,
NamedXContentRegistry xContentRegistry, HttpServerTransport.Dispatcher dispatcher) {
@ -136,6 +151,7 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport {
SETTING_HTTP_COMPRESSION_LEVEL.get(settings),
SETTING_HTTP_DETAILED_ERRORS_ENABLED.get(settings),
pipeliningMaxEvents);
this.corsConfig = buildCorsConfig(settings);
this.tcpNoDelay = SETTING_HTTP_TCP_NO_DELAY.get(settings);
this.tcpKeepAlive = SETTING_HTTP_TCP_KEEP_ALIVE.get(settings);
@ -279,6 +295,38 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport {
logger.warn(new ParameterizedMessage("exception caught on transport layer [thread={}]", Thread.currentThread().getName()), ex);
}
static NioCorsConfig buildCorsConfig(Settings settings) {
if (SETTING_CORS_ENABLED.get(settings) == false) {
return NioCorsConfigBuilder.forOrigins().disable().build();
}
String origin = SETTING_CORS_ALLOW_ORIGIN.get(settings);
final NioCorsConfigBuilder builder;
if (Strings.isNullOrEmpty(origin)) {
builder = NioCorsConfigBuilder.forOrigins();
} else if (origin.equals(ANY_ORIGIN)) {
builder = NioCorsConfigBuilder.forAnyOrigin();
} else {
Pattern p = RestUtils.checkCorsSettingForRegex(origin);
if (p == null) {
builder = NioCorsConfigBuilder.forOrigins(RestUtils.corsSettingAsArray(origin));
} else {
builder = NioCorsConfigBuilder.forPattern(p);
}
}
if (SETTING_CORS_ALLOW_CREDENTIALS.get(settings)) {
builder.allowCredentials();
}
String[] strMethods = Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_METHODS.get(settings), ",");
HttpMethod[] methods = Arrays.stream(strMethods)
.map(HttpMethod::valueOf)
.toArray(HttpMethod[]::new);
return builder.allowedRequestMethods(methods)
.maxAge(SETTING_CORS_MAX_AGE.get(settings))
.allowedRequestHeaders(Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_HEADERS.get(settings), ","))
.shortCircuit()
.build();
}
private void closeChannels(List<NioChannel> channels) {
List<ActionFuture<Void>> futures = new ArrayList<>(channels.size());
@ -315,7 +363,7 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport {
public NioSocketChannel createChannel(SocketSelector selector, SocketChannel channel) throws IOException {
NioSocketChannel nioChannel = new NioSocketChannel(channel);
HttpReadWriteHandler httpReadWritePipeline = new HttpReadWriteHandler(nioChannel,NioHttpServerTransport.this,
httpHandlingSettings, xContentRegistry, threadPool.getThreadContext());
httpHandlingSettings, xContentRegistry, corsConfig, threadPool.getThreadContext());
Consumer<Exception> exceptionHandler = (e) -> exceptionCaught(nioChannel, e);
SocketChannelContext context = new BytesChannelContext(nioChannel, selector, exceptionHandler, httpReadWritePipeline,
InboundChannelBuffer.allocatingInstance());

236
plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsConfig.java Normal file
View File

@ -0,0 +1,236 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.nio.cors;
import io.netty.handler.codec.http.DefaultHttpHeaders;
import io.netty.handler.codec.http.EmptyHttpHeaders;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.regex.Pattern;
/**
* Configuration for Cross-Origin Resource Sharing (CORS).
*
* This class was lifted from the Netty project:
* https://github.com/netty/netty
*/
public final class NioCorsConfig {
private final Optional<Set<String>> origins;
private final Optional<Pattern> pattern;
private final boolean anyOrigin;
private final boolean enabled;
private final boolean allowCredentials;
private final long maxAge;
private final Set<HttpMethod> allowedRequestMethods;
private final Set<String> allowedRequestHeaders;
private final boolean allowNullOrigin;
private final Map<CharSequence, Callable<?>> preflightHeaders;
private final boolean shortCircuit;
NioCorsConfig(final NioCorsConfigBuilder builder) {
origins = builder.origins.map(s -> new LinkedHashSet<>(s));
pattern = builder.pattern;
anyOrigin = builder.anyOrigin;
enabled = builder.enabled;
allowCredentials = builder.allowCredentials;
maxAge = builder.maxAge;
allowedRequestMethods = builder.requestMethods;
allowedRequestHeaders = builder.requestHeaders;
allowNullOrigin = builder.allowNullOrigin;
preflightHeaders = builder.preflightHeaders;
shortCircuit = builder.shortCircuit;
}
/**
* Determines if support for CORS is enabled.
*
* @return {@code true} if support for CORS is enabled, false otherwise.
*/
public boolean isCorsSupportEnabled() {
return enabled;
}
/**
* Determines whether a wildcard origin, '*', is supported. This also means that null origins are
* supported.
*
* @return {@code boolean} true if any origin is allowed.
*/
public boolean isAnyOriginSupported() {
return anyOrigin;
}
/**
* Returns the set of allowed origins.
*
* @return {@code Set} the allowed origins.
*/
public Optional<Set<String>> origins() {
return origins;
}
/**
* Returns whether the input origin is allowed by this configuration.
*
* @return {@code true} if the origin is allowed, otherwise {@code false}
*/
public boolean isOriginAllowed(final String origin) {
if (origins.isPresent()) {
return origins.get().contains(origin);
} else if (pattern.isPresent()) {
return pattern.get().matcher(origin).matches();
}
return false;
}
/**
* Web browsers may set the 'Origin' request header to 'null' if a resource is loaded
* from the local file system.
*
* If isNullOriginAllowed is true then the server will response with the wildcard for the
* the CORS response header 'Access-Control-Allow-Origin'.
*
* @return {@code true} if a 'null' origin should be supported.
*/
public boolean isNullOriginAllowed() {
return allowNullOrigin;
}
/**
* Determines if credentials are supported for CORS requests.
*
* By default credentials are not included in CORS requests but if isCredentialsAllowed returns
* true credentials will be added to CORS requests. Setting this value to true will set the
* CORS 'Access-Control-Allow-Credentials' response header to true.
*
* Please note that credentials support needs to be enabled on the client side as well.
* The client needs to opt-in to send credentials by calling:
* <pre>
* xhr.withCredentials = true;
* </pre>
* The default value for 'withCredentials' is false in which case no credentials are sent.
* Setting this to true will included cookies in cross origin requests.
*
* @return {@code true} if credentials are supported.
*/
public boolean isCredentialsAllowed() {
return allowCredentials;
}
/**
* Gets the maxAge setting.
*
* When making a preflight request the client has to perform two request with can be inefficient.
* This setting will set the CORS 'Access-Control-Max-Age' response header and enables the
* caching of the preflight response for the specified time. During this time no preflight
* request will be made.
*
* @return {@code long} the time in seconds that a preflight request may be cached.
*/
public long maxAge() {
return maxAge;
}
/**
* Returns the allowed set of Request Methods. The Http methods that should be returned in the
* CORS 'Access-Control-Request-Method' response header.
*
* @return {@code Set} of {@link HttpMethod}s that represent the allowed Request Methods.
*/
public Set<HttpMethod> allowedRequestMethods() {
return Collections.unmodifiableSet(allowedRequestMethods);
}
/**
* Returns the allowed set of Request Headers.
*
* The header names returned from this method will be used to set the CORS
* 'Access-Control-Allow-Headers' response header.
*
* @return {@code Set<String>} of strings that represent the allowed Request Headers.
*/
public Set<String> allowedRequestHeaders() {
return Collections.unmodifiableSet(allowedRequestHeaders);
}
/**
* Returns HTTP response headers that should be added to a CORS preflight response.
*
* @return {@link HttpHeaders} the HTTP response headers to be added.
*/
public HttpHeaders preflightResponseHeaders() {
if (preflightHeaders.isEmpty()) {
return EmptyHttpHeaders.INSTANCE;
}
final HttpHeaders preflightHeaders = new DefaultHttpHeaders();
for (Map.Entry<CharSequence, Callable<?>> entry : this.preflightHeaders.entrySet()) {
final Object value = getValue(entry.getValue());
if (value instanceof Iterable) {
preflightHeaders.add(entry.getKey().toString(), (Iterable<?>) value);
} else {
preflightHeaders.add(entry.getKey().toString(), value);
}
}
return preflightHeaders;
}
/**
* Determines whether a CORS request should be rejected if it's invalid before being
* further processing.
*
* CORS headers are set after a request is processed. This may not always be desired
* and this setting will check that the Origin is valid and if it is not valid no
* further processing will take place, and a error will be returned to the calling client.
*
* @return {@code true} if a CORS request should short-circuit upon receiving an invalid Origin header.
*/
public boolean isShortCircuit() {
return shortCircuit;
}
private static <T> T getValue(final Callable<T> callable) {
try {
return callable.call();
} catch (final Exception e) {
throw new IllegalStateException("Could not generate value for callable [" + callable + ']', e);
}
}
@Override
public String toString() {
return "CorsConfig[enabled=" + enabled +
", origins=" + origins +
", anyOrigin=" + anyOrigin +
", isCredentialsAllowed=" + allowCredentials +
", maxAge=" + maxAge +
", allowedRequestMethods=" + allowedRequestMethods +
", allowedRequestHeaders=" + allowedRequestHeaders +
", preflightHeaders=" + preflightHeaders + ']';
}
}

357
plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsConfigBuilder.java Normal file
View File

@ -0,0 +1,357 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.nio.cors;
import io.netty.handler.codec.http.HttpMethod;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.regex.Pattern;
/**
* Builder used to configure and build a {@link NioCorsConfig} instance.
*
* This class was lifted from the Netty project:
* https://github.com/netty/netty
*/
public final class NioCorsConfigBuilder {
/**
* Creates a Builder instance with it's origin set to '*'.
*
* @return Builder to support method chaining.
*/
public static NioCorsConfigBuilder forAnyOrigin() {
return new NioCorsConfigBuilder();
}
/**
* Creates a {@link NioCorsConfigBuilder} instance with the specified origin.
*
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
public static NioCorsConfigBuilder forOrigin(final String origin) {
if ("*".equals(origin)) {
return new NioCorsConfigBuilder();
}
return new NioCorsConfigBuilder(origin);
}
/**
* Create a {@link NioCorsConfigBuilder} instance with the specified pattern origin.
*
* @param pattern the regular expression pattern to match incoming origins on.
* @return {@link NioCorsConfigBuilder} with the configured origin pattern.
*/
public static NioCorsConfigBuilder forPattern(final Pattern pattern) {
if (pattern == null) {
throw new IllegalArgumentException("CORS pattern cannot be null");
}
return new NioCorsConfigBuilder(pattern);
}
/**
* Creates a {@link NioCorsConfigBuilder} instance with the specified origins.
*
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
public static NioCorsConfigBuilder forOrigins(final String... origins) {
return new NioCorsConfigBuilder(origins);
}
Optional<Set<String>> origins;
Optional<Pattern> pattern;
final boolean anyOrigin;
boolean allowNullOrigin;
boolean enabled = true;
boolean allowCredentials;
long maxAge;
final Set<HttpMethod> requestMethods = new HashSet<>();
final Set<String> requestHeaders = new HashSet<>();
final Map<CharSequence, Callable<?>> preflightHeaders = new HashMap<>();
private boolean noPreflightHeaders;
boolean shortCircuit;
/**
* Creates a new Builder instance with the origin passed in.
*
* @param origins the origin to be used for this builder.
*/
NioCorsConfigBuilder(final String... origins) {
this.origins = Optional.of(new LinkedHashSet<>(Arrays.asList(origins)));
pattern = Optional.empty();
anyOrigin = false;
}
/**
* Creates a new Builder instance allowing any origin, "*" which is the
* wildcard origin.
*
*/
NioCorsConfigBuilder() {
anyOrigin = true;
origins = Optional.empty();
pattern = Optional.empty();
}
/**
* Creates a new Builder instance allowing any origin that matches the pattern.
*
* @param pattern the pattern to match against for incoming origins.
*/
NioCorsConfigBuilder(final Pattern pattern) {
this.pattern = Optional.of(pattern);
origins = Optional.empty();
anyOrigin = false;
}
/**
* Web browsers may set the 'Origin' request header to 'null' if a resource is loaded
* from the local file system. Calling this method will enable a successful CORS response
* with a wildcard for the CORS response header 'Access-Control-Allow-Origin'.
*
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
NioCorsConfigBuilder allowNullOrigin() {
allowNullOrigin = true;
return this;
}
/**
* Disables CORS support.
*
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
public NioCorsConfigBuilder disable() {
enabled = false;
return this;
}
/**
* By default cookies are not included in CORS requests, but this method will enable cookies to
* be added to CORS requests. Calling this method will set the CORS 'Access-Control-Allow-Credentials'
* response header to true.
*
* Please note, that cookie support needs to be enabled on the client side as well.
* The client needs to opt-in to send cookies by calling:
* <pre>
* xhr.withCredentials = true;
* </pre>
* The default value for 'withCredentials' is false in which case no cookies are sent.
* Setting this to true will included cookies in cross origin requests.
*
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
public NioCorsConfigBuilder allowCredentials() {
allowCredentials = true;
return this;
}
/**
* When making a preflight request the client has to perform two request with can be inefficient.
* This setting will set the CORS 'Access-Control-Max-Age' response header and enables the
* caching of the preflight response for the specified time. During this time no preflight
* request will be made.
*
* @param max the maximum time, in seconds, that the preflight response may be cached.
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
public NioCorsConfigBuilder maxAge(final long max) {
maxAge = max;
return this;
}
/**
* Specifies the allowed set of HTTP Request Methods that should be returned in the
* CORS 'Access-Control-Request-Method' response header.
*
* @param methods the {@link HttpMethod}s that should be allowed.
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
public NioCorsConfigBuilder allowedRequestMethods(final HttpMethod... methods) {
requestMethods.addAll(Arrays.asList(methods));
return this;
}
/**
* Specifies the if headers that should be returned in the CORS 'Access-Control-Allow-Headers'
* response header.
*
* If a client specifies headers on the request, for example by calling:
* <pre>
* xhr.setRequestHeader('My-Custom-Header', "SomeValue");
* </pre>
* the server will receive the above header name in the 'Access-Control-Request-Headers' of the
* preflight request. The server will then decide if it allows this header to be sent for the
* real request (remember that a preflight is not the real request but a request asking the server
* if it allow a request).
*
* @param headers the headers to be added to the preflight 'Access-Control-Allow-Headers' response header.
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
public NioCorsConfigBuilder allowedRequestHeaders(final String... headers) {
requestHeaders.addAll(Arrays.asList(headers));
return this;
}
/**
* Returns HTTP response headers that should be added to a CORS preflight response.
*
* An intermediary like a load balancer might require that a CORS preflight request
* have certain headers set. This enables such headers to be added.
*
* @param name the name of the HTTP header.
* @param values the values for the HTTP header.
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
public NioCorsConfigBuilder preflightResponseHeader(final CharSequence name, final Object... values) {
if (values.length == 1) {
preflightHeaders.put(name, new ConstantValueGenerator(values[0]));
} else {
preflightResponseHeader(name, Arrays.asList(values));
}
return this;
}
/**
* Returns HTTP response headers that should be added to a CORS preflight response.
*
* An intermediary like a load balancer might require that a CORS preflight request
* have certain headers set. This enables such headers to be added.
*
* @param name the name of the HTTP header.
* @param value the values for the HTTP header.
* @param <T> the type of values that the Iterable contains.
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
public <T> NioCorsConfigBuilder preflightResponseHeader(final CharSequence name, final Iterable<T> value) {
preflightHeaders.put(name, new ConstantValueGenerator(value));
return this;
}
/**
* Returns HTTP response headers that should be added to a CORS preflight response.
*
* An intermediary like a load balancer might require that a CORS preflight request
* have certain headers set. This enables such headers to be added.
*
* Some values must be dynamically created when the HTTP response is created, for
* example the 'Date' response header. This can be accomplished by using a Callable
* which will have its 'call' method invoked when the HTTP response is created.
*
* @param name the name of the HTTP header.
* @param valueGenerator a Callable which will be invoked at HTTP response creation.
* @param <T> the type of the value that the Callable can return.
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
public <T> NioCorsConfigBuilder preflightResponseHeader(final CharSequence name, final Callable<T> valueGenerator) {
preflightHeaders.put(name, valueGenerator);
return this;
}
/**
* Specifies that no preflight response headers should be added to a preflight response.
*
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
public NioCorsConfigBuilder noPreflightResponseHeaders() {
noPreflightHeaders = true;
return this;
}
/**
* Specifies that a CORS request should be rejected if it's invalid before being
* further processing.
*
* CORS headers are set after a request is processed. This may not always be desired
* and this setting will check that the Origin is valid and if it is not valid no
* further processing will take place, and a error will be returned to the calling client.
*
* @return {@link NioCorsConfigBuilder} to support method chaining.
*/
public NioCorsConfigBuilder shortCircuit() {
shortCircuit = true;
return this;
}
/**
* Builds a {@link NioCorsConfig} with settings specified by previous method calls.
*
* @return {@link NioCorsConfig} the configured CorsConfig instance.
*/
public NioCorsConfig build() {
if (preflightHeaders.isEmpty() && !noPreflightHeaders) {
preflightHeaders.put("date", DateValueGenerator.INSTANCE);
preflightHeaders.put("content-length", new ConstantValueGenerator("0"));
}
return new NioCorsConfig(this);
}
/**
* This class is used for preflight HTTP response values that do not need to be
* generated, but instead the value is "static" in that the same value will be returned
* for each call.
*/
private static final class ConstantValueGenerator implements Callable<Object> {
private final Object value;
/**
* Sole constructor.
*
* @param value the value that will be returned when the call method is invoked.
*/
private ConstantValueGenerator(final Object value) {
if (value == null) {
throw new IllegalArgumentException("value must not be null");
}
this.value = value;
}
@Override
public Object call() {
return value;
}
}
/**
* This callable is used for the DATE preflight HTTP response HTTP header.
* It's value must be generated when the response is generated, hence will be
* different for every call.
*/
private static final class DateValueGenerator implements Callable<Date> {
static final DateValueGenerator INSTANCE = new DateValueGenerator();
@Override
public Date call() throws Exception {
return new Date();
}
}
}

235
plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsHandler.java Normal file
View File

@ -0,0 +1,235 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.nio.cors;
import io.netty.channel.ChannelDuplexHandler;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.http.DefaultFullHttpResponse;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.HttpResponse;
import io.netty.handler.codec.http.HttpResponseStatus;
import org.elasticsearch.common.Strings;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* Handles <a href="http://www.w3.org/TR/cors/">Cross Origin Resource Sharing</a> (CORS) requests.
* <p>
* This handler can be configured using a {@link NioCorsConfig}, please
* refer to this class for details about the configuration options available.
*
* This code was borrowed from Netty 4 and refactored to work for Elasticsearch's Netty 3 setup.
*/
public class NioCorsHandler extends ChannelDuplexHandler {
public static final String ANY_ORIGIN = "*";
private static Pattern SCHEME_PATTERN = Pattern.compile("^https?://");
private final NioCorsConfig config;
private HttpRequest request;
/**
* Creates a new instance with the specified {@link NioCorsConfig}.
*/
public NioCorsHandler(final NioCorsConfig config) {
if (config == null) {
throw new NullPointerException();
}
this.config = config;
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
if (config.isCorsSupportEnabled() && msg instanceof HttpRequest) {
request = (HttpRequest) msg;
if (isPreflightRequest(request)) {
handlePreflight(ctx, request);
return;
}
if (config.isShortCircuit() && !validateOrigin()) {
forbidden(ctx, request);
return;
}
}
ctx.fireChannelRead(msg);
}
public static void setCorsResponseHeaders(HttpRequest request, HttpResponse resp, NioCorsConfig config) {
if (!config.isCorsSupportEnabled()) {
return;
}
String originHeader = request.headers().get(HttpHeaderNames.ORIGIN);
if (!Strings.isNullOrEmpty(originHeader)) {
final String originHeaderVal;
if (config.isAnyOriginSupported()) {
originHeaderVal = ANY_ORIGIN;
} else if (config.isOriginAllowed(originHeader) || isSameOrigin(originHeader, request.headers().get(HttpHeaderNames.HOST))) {
originHeaderVal = originHeader;
} else {
originHeaderVal = null;
}
if (originHeaderVal != null) {
resp.headers().add(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN, originHeaderVal);
}
}
if (config.isCredentialsAllowed()) {
resp.headers().add(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS, "true");
}
}
private void handlePreflight(final ChannelHandlerContext ctx, final HttpRequest request) {
final HttpResponse response = new DefaultFullHttpResponse(request.protocolVersion(), HttpResponseStatus.OK, true, true);
if (setOrigin(response)) {
setAllowMethods(response);
setAllowHeaders(response);
setAllowCredentials(response);
setMaxAge(response);
setPreflightHeaders(response);
ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE);
} else {
forbidden(ctx, request);
}
}
private static void forbidden(final ChannelHandlerContext ctx, final HttpRequest request) {
ctx.writeAndFlush(new DefaultFullHttpResponse(request.protocolVersion(), HttpResponseStatus.FORBIDDEN))
.addListener(ChannelFutureListener.CLOSE);
}
private static boolean isSameOrigin(final String origin, final String host) {
if (Strings.isNullOrEmpty(host) == false) {
// strip protocol from origin
final String originDomain = SCHEME_PATTERN.matcher(origin).replaceFirst("");
if (host.equals(originDomain)) {
return true;
}
}
return false;
}
/**
* This is a non CORS specification feature which enables the setting of preflight
* response headers that might be required by intermediaries.
*
* @param response the HttpResponse to which the preflight response headers should be added.
*/
private void setPreflightHeaders(final HttpResponse response) {
response.headers().add(config.preflightResponseHeaders());
}
private boolean setOrigin(final HttpResponse response) {
final String origin = request.headers().get(HttpHeaderNames.ORIGIN);
if (!Strings.isNullOrEmpty(origin)) {
if ("null".equals(origin) && config.isNullOriginAllowed()) {
setAnyOrigin(response);
return true;
}
if (config.isAnyOriginSupported()) {
if (config.isCredentialsAllowed()) {
echoRequestOrigin(response);
setVaryHeader(response);
} else {
setAnyOrigin(response);
}
return true;
}
if (config.isOriginAllowed(origin)) {
setOrigin(response, origin);
setVaryHeader(response);
return true;
}
}
return false;
}
private boolean validateOrigin() {
if (config.isAnyOriginSupported()) {
return true;
}
final String origin = request.headers().get(HttpHeaderNames.ORIGIN);
if (Strings.isNullOrEmpty(origin)) {
// Not a CORS request so we cannot validate it. It may be a non CORS request.
return true;
}
if ("null".equals(origin) && config.isNullOriginAllowed()) {
return true;
}
// if the origin is the same as the host of the request, then allow
if (isSameOrigin(origin, request.headers().get(HttpHeaderNames.HOST))) {
return true;
}
return config.isOriginAllowed(origin);
}
private void echoRequestOrigin(final HttpResponse response) {
setOrigin(response, request.headers().get(HttpHeaderNames.ORIGIN));
}
private static void setVaryHeader(final HttpResponse response) {
response.headers().set(HttpHeaderNames.VARY, HttpHeaderNames.ORIGIN);
}
private static void setAnyOrigin(final HttpResponse response) {
setOrigin(response, ANY_ORIGIN);
}
private static void setOrigin(final HttpResponse response, final String origin) {
response.headers().set(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN, origin);
}
private void setAllowCredentials(final HttpResponse response) {
if (config.isCredentialsAllowed()
&& !response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN).equals(ANY_ORIGIN)) {
response.headers().set(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS, "true");
}
}
private static boolean isPreflightRequest(final HttpRequest request) {
final HttpHeaders headers = request.headers();
return request.method().equals(HttpMethod.OPTIONS) &&
headers.contains(HttpHeaderNames.ORIGIN) &&
headers.contains(HttpHeaderNames.ACCESS_CONTROL_REQUEST_METHOD);
}
private void setAllowMethods(final HttpResponse response) {
response.headers().set(HttpHeaderNames.ACCESS_CONTROL_ALLOW_METHODS, config.allowedRequestMethods().stream()
.map(m -> m.name().trim())
.collect(Collectors.toList()));
}
private void setAllowHeaders(final HttpResponse response) {
response.headers().set(HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS, config.allowedRequestHeaders());
}
private void setMaxAge(final HttpResponse response) {
response.headers().set(HttpHeaderNames.ACCESS_CONTROL_MAX_AGE, config.maxAge());
}
}

5
plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java
View File

@ -39,6 +39,8 @@ import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.http.HttpHandlingSettings;
import org.elasticsearch.http.nio.cors.NioCorsConfig;
import org.elasticsearch.http.nio.cors.NioCorsConfigBuilder;
import org.elasticsearch.nio.FlushOperation;
import org.elasticsearch.nio.InboundChannelBuffer;
import org.elasticsearch.nio.NioSocketChannel;
@ -95,7 +97,8 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
SETTING_PIPELINING_MAX_EVENTS.getDefault(settings));
ThreadContext threadContext = new ThreadContext(settings);
nioSocketChannel = mock(NioSocketChannel.class);
handler = new HttpReadWriteHandler(nioSocketChannel, transport, httpHandlingSettings, NamedXContentRegistry.EMPTY, threadContext);
handler = new HttpReadWriteHandler(nioSocketChannel, transport, httpHandlingSettings, NamedXContentRegistry.EMPTY,
NioCorsConfigBuilder.forAnyOrigin().build(), threadContext);
}
public void testSuccessfulDecodeHttpRequest() throws IOException {

349
plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpChannelTests.java Normal file
View File

@ -0,0 +1,349 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.nio;
import io.netty.buffer.Unpooled;
import io.netty.handler.codec.http.DefaultFullHttpRequest;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.FullHttpResponse;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.HttpHeaderValues;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpResponse;
import io.netty.handler.codec.http.HttpVersion;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.MockBigArrays;
import org.elasticsearch.common.util.MockPageCacheRecycler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.http.HttpHandlingSettings;
import org.elasticsearch.http.HttpTransportSettings;
import org.elasticsearch.http.nio.cors.NioCorsConfig;
import org.elasticsearch.http.nio.cors.NioCorsHandler;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.nio.NioSocketChannel;
import org.elasticsearch.nio.SocketChannelContext;
import org.elasticsearch.rest.BytesRestResponse;
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.junit.After;
import org.junit.Before;
import org.mockito.ArgumentCaptor;
import java.io.IOException;
import java.nio.channels.ClosedChannelException;
import java.nio.charset.StandardCharsets;
import java.util.function.BiConsumer;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class NioHttpChannelTests extends ESTestCase {
private ThreadPool threadPool;
private MockBigArrays bigArrays;
private NioSocketChannel nioChannel;
private SocketChannelContext channelContext;
@Before
public void setup() throws Exception {
nioChannel = mock(NioSocketChannel.class);
channelContext = mock(SocketChannelContext.class);
when(nioChannel.getContext()).thenReturn(channelContext);
threadPool = new TestThreadPool("test");
bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
}
@After
public void shutdown() throws Exception {
if (threadPool != null) {
threadPool.shutdownNow();
}
}
public void testResponse() {
final FullHttpResponse response = executeRequest(Settings.EMPTY, "request-host");
assertThat(response.content(), equalTo(ByteBufUtils.toByteBuf(new TestResponse().content())));
}
public void testCorsEnabledWithoutAllowOrigins() {
// Set up a HTTP transport with only the CORS enabled setting
Settings settings = Settings.builder()
.put(HttpTransportSettings.SETTING_CORS_ENABLED.getKey(), true)
.build();
HttpResponse response = executeRequest(settings, "remote-host", "request-host");
// inspect response and validate
assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), nullValue());
}
public void testCorsEnabledWithAllowOrigins() {
final String originValue = "remote-host";
// create a http transport with CORS enabled and allow origin configured
Settings settings = Settings.builder()
.put(SETTING_CORS_ENABLED.getKey(), true)
.put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue)
.build();
HttpResponse response = executeRequest(settings, originValue, "request-host");
// inspect response and validate
assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
}
public void testCorsAllowOriginWithSameHost() {
String originValue = "remote-host";
String host = "remote-host";
// create a http transport with CORS enabled
Settings settings = Settings.builder()
.put(SETTING_CORS_ENABLED.getKey(), true)
.build();
HttpResponse response = executeRequest(settings, originValue, host);
// inspect response and validate
assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
originValue = "http://" + originValue;
response = executeRequest(settings, originValue, host);
assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
originValue = originValue + ":5555";
host = host + ":5555";
response = executeRequest(settings, originValue, host);
assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
originValue = originValue.replace("http", "https");
response = executeRequest(settings, originValue, host);
assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
}
public void testThatStringLiteralWorksOnMatch() {
final String originValue = "remote-host";
Settings settings = Settings.builder()
.put(SETTING_CORS_ENABLED.getKey(), true)
.put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue)
.put(SETTING_CORS_ALLOW_METHODS.getKey(), "get, options, post")
.put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
.build();
HttpResponse response = executeRequest(settings, originValue, "request-host");
// inspect response and validate
assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS), equalTo("true"));
}
public void testThatAnyOriginWorks() {
final String originValue = NioCorsHandler.ANY_ORIGIN;
Settings settings = Settings.builder()
.put(SETTING_CORS_ENABLED.getKey(), true)
.put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue)
.build();
HttpResponse response = executeRequest(settings, originValue, "request-host");
// inspect response and validate
assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS), nullValue());
}
public void testHeadersSet() {
Settings settings = Settings.builder().build();
final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
httpRequest.headers().add(HttpHeaderNames.ORIGIN, "remote");
final NioHttpRequest request = new NioHttpRequest(xContentRegistry(), httpRequest);
HttpHandlingSettings handlingSettings = HttpHandlingSettings.fromSettings(settings);
NioCorsConfig corsConfig = NioHttpServerTransport.buildCorsConfig(settings);
// send a response
NioHttpChannel channel = new NioHttpChannel(nioChannel, bigArrays, request, 1, handlingSettings, corsConfig,
threadPool.getThreadContext());
TestResponse resp = new TestResponse();
final String customHeader = "custom-header";
final String customHeaderValue = "xyz";
resp.addHeader(customHeader, customHeaderValue);
channel.sendResponse(resp);
// inspect what was written
ArgumentCaptor<Object> responseCaptor = ArgumentCaptor.forClass(Object.class);
verify(channelContext).sendMessage(responseCaptor.capture(), any());
Object nioResponse = responseCaptor.getValue();
HttpResponse response = ((NioHttpResponse) nioResponse).getResponse();
assertThat(response.headers().get("non-existent-header"), nullValue());
assertThat(response.headers().get(customHeader), equalTo(customHeaderValue));
assertThat(response.headers().get(HttpHeaderNames.CONTENT_LENGTH), equalTo(Integer.toString(resp.content().length())));
assertThat(response.headers().get(HttpHeaderNames.CONTENT_TYPE), equalTo(resp.contentType()));
}
@SuppressWarnings("unchecked")
public void testReleaseInListener() throws IOException {
final Settings settings = Settings.builder().build();
final NamedXContentRegistry registry = xContentRegistry();
final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
final NioHttpRequest request = new NioHttpRequest(registry, httpRequest);
HttpHandlingSettings handlingSettings = HttpHandlingSettings.fromSettings(settings);
NioCorsConfig corsConfig = NioHttpServerTransport.buildCorsConfig(settings);
NioHttpChannel channel = new NioHttpChannel(nioChannel, bigArrays, request, 1, handlingSettings,
corsConfig, threadPool.getThreadContext());
final BytesRestResponse response = new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR,
JsonXContent.contentBuilder().startObject().endObject());
assertThat(response.content(), not(instanceOf(Releasable.class)));
// ensure we have reserved bytes
if (randomBoolean()) {
BytesStreamOutput out = channel.bytesOutput();
assertThat(out, instanceOf(ReleasableBytesStreamOutput.class));
} else {
try (XContentBuilder builder = channel.newBuilder()) {
// do something builder
builder.startObject().endObject();
}
}
channel.sendResponse(response);
Class<BiConsumer<Void, Exception>> listenerClass = (Class<BiConsumer<Void, Exception>>) (Class) BiConsumer.class;
ArgumentCaptor<BiConsumer<Void, Exception>> listenerCaptor = ArgumentCaptor.forClass(listenerClass);
verify(channelContext).sendMessage(any(), listenerCaptor.capture());
BiConsumer<Void, Exception> listener = listenerCaptor.getValue();
if (randomBoolean()) {
listener.accept(null, null);
} else {
listener.accept(null, new ClosedChannelException());
}
// ESTestCase#after will invoke ensureAllArraysAreReleased which will fail if the response content was not released
}
@SuppressWarnings("unchecked")
public void testConnectionClose() throws Exception {
final Settings settings = Settings.builder().build();
final FullHttpRequest httpRequest;
final boolean close = randomBoolean();
if (randomBoolean()) {
httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
if (close) {
httpRequest.headers().add(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE);
}
} else {
httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_0, HttpMethod.GET, "/");
if (!close) {
httpRequest.headers().add(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE);
}
}
final NioHttpRequest request = new NioHttpRequest(xContentRegistry(), httpRequest);
HttpHandlingSettings handlingSettings = HttpHandlingSettings.fromSettings(settings);
NioCorsConfig corsConfig = NioHttpServerTransport.buildCorsConfig(settings);
NioHttpChannel channel = new NioHttpChannel(nioChannel, bigArrays, request, 1, handlingSettings,
corsConfig, threadPool.getThreadContext());
final TestResponse resp = new TestResponse();
channel.sendResponse(resp);
Class<BiConsumer<Void, Exception>> listenerClass = (Class<BiConsumer<Void, Exception>>) (Class) BiConsumer.class;
ArgumentCaptor<BiConsumer<Void, Exception>> listenerCaptor = ArgumentCaptor.forClass(listenerClass);
verify(channelContext).sendMessage(any(), listenerCaptor.capture());
BiConsumer<Void, Exception> listener = listenerCaptor.getValue();
listener.accept(null, null);
if (close) {
verify(nioChannel, times(1)).close();
} else {
verify(nioChannel, times(0)).close();
}
}
private FullHttpResponse executeRequest(final Settings settings, final String host) {
return executeRequest(settings, null, host);
}
private FullHttpResponse executeRequest(final Settings settings, final String originValue, final String host) {
// construct request and send it over the transport layer
final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
if (originValue != null) {
httpRequest.headers().add(HttpHeaderNames.ORIGIN, originValue);
}
httpRequest.headers().add(HttpHeaderNames.HOST, host);
final NioHttpRequest request = new NioHttpRequest(xContentRegistry(), httpRequest);
HttpHandlingSettings httpHandlingSettings = HttpHandlingSettings.fromSettings(settings);
NioCorsConfig corsConfig = NioHttpServerTransport.buildCorsConfig(settings);
NioHttpChannel channel = new NioHttpChannel(nioChannel, bigArrays, request, 1, httpHandlingSettings, corsConfig,
threadPool.getThreadContext());
channel.sendResponse(new TestResponse());
// get the response
ArgumentCaptor<Object> responseCaptor = ArgumentCaptor.forClass(Object.class);
verify(channelContext, atLeastOnce()).sendMessage(responseCaptor.capture(), any());
return ((NioHttpResponse) responseCaptor.getValue()).getResponse();
}
private static class TestResponse extends RestResponse {
private final BytesReference reference;
TestResponse() {
reference = ByteBufUtils.toBytesReference(Unpooled.copiedBuffer("content", StandardCharsets.UTF_8));
}
@Override
public String contentType() {
return "text";
}
@Override
public BytesReference content() {
return reference;
}
@Override
public RestStatus status() {
return RestStatus.OK;
}
}
}

70
plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java
View File

@ -32,6 +32,7 @@ import io.netty.handler.codec.http.HttpResponseStatus;
import io.netty.handler.codec.http.HttpUtil;
import io.netty.handler.codec.http.HttpVersion;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.Setting;
@ -45,6 +46,7 @@ import org.elasticsearch.http.BindHttpException;
import org.elasticsearch.http.HttpServerTransport;
import org.elasticsearch.http.HttpTransportSettings;
import org.elasticsearch.http.NullDispatcher;
import org.elasticsearch.http.nio.cors.NioCorsConfig;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.rest.BytesRestResponse;
import org.elasticsearch.rest.RestChannel;
@ -58,9 +60,19 @@ import org.junit.Before;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE;
import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
import static org.elasticsearch.rest.RestStatus.OK;
import static org.hamcrest.Matchers.containsString;
@ -94,36 +106,36 @@ public class NioHttpServerTransportTests extends ESTestCase {
bigArrays = null;
}
// public void testCorsConfig() {
// final Set<String> methods = new HashSet<>(Arrays.asList("get", "options", "post"));
// final Set<String> headers = new HashSet<>(Arrays.asList("Content-Type", "Content-Length"));
// final String prefix = randomBoolean() ? " " : ""; // sometimes have a leading whitespace between comma delimited elements
// final Settings settings = Settings.builder()
// .put(SETTING_CORS_ENABLED.getKey(), true)
// .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*")
// .put(SETTING_CORS_ALLOW_METHODS.getKey(), collectionToDelimitedString(methods, ",", prefix, ""))
// .put(SETTING_CORS_ALLOW_HEADERS.getKey(), collectionToDelimitedString(headers, ",", prefix, ""))
// .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
// .build();
// final Netty4CorsConfig corsConfig = Netty4HttpServerTransport.buildCorsConfig(settings);
// assertTrue(corsConfig.isAnyOriginSupported());
// assertEquals(headers, corsConfig.allowedRequestHeaders());
// assertEquals(methods, corsConfig.allowedRequestMethods().stream().map(HttpMethod::name).collect(Collectors.toSet()));
// }
public void testCorsConfig() {
final Set<String> methods = new HashSet<>(Arrays.asList("get", "options", "post"));
final Set<String> headers = new HashSet<>(Arrays.asList("Content-Type", "Content-Length"));
final String prefix = randomBoolean() ? " " : ""; // sometimes have a leading whitespace between comma delimited elements
final Settings settings = Settings.builder()
.put(SETTING_CORS_ENABLED.getKey(), true)
.put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*")
.put(SETTING_CORS_ALLOW_METHODS.getKey(), Strings.collectionToDelimitedString(methods, ",", prefix, ""))
.put(SETTING_CORS_ALLOW_HEADERS.getKey(), Strings.collectionToDelimitedString(headers, ",", prefix, ""))
.put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
.build();
final NioCorsConfig corsConfig = NioHttpServerTransport.buildCorsConfig(settings);
assertTrue(corsConfig.isAnyOriginSupported());
assertEquals(headers, corsConfig.allowedRequestHeaders());
assertEquals(methods, corsConfig.allowedRequestMethods().stream().map(HttpMethod::name).collect(Collectors.toSet()));
}
// public void testCorsConfigWithDefaults() {
// final Set<String> methods = Strings.commaDelimitedListToSet(SETTING_CORS_ALLOW_METHODS.getDefault(Settings.EMPTY));
// final Set<String> headers = Strings.commaDelimitedListToSet(SETTING_CORS_ALLOW_HEADERS.getDefault(Settings.EMPTY));
// final long maxAge = SETTING_CORS_MAX_AGE.getDefault(Settings.EMPTY);
// final Settings settings = Settings.builder().put(SETTING_CORS_ENABLED.getKey(), true).build();
// final Netty4CorsConfig corsConfig = Netty4HttpServerTransport.buildCorsConfig(settings);
// assertFalse(corsConfig.isAnyOriginSupported());
// assertEquals(Collections.emptySet(), corsConfig.origins().get());
// assertEquals(headers, corsConfig.allowedRequestHeaders());
// assertEquals(methods, corsConfig.allowedRequestMethods().stream().map(HttpMethod::name).collect(Collectors.toSet()));
// assertEquals(maxAge, corsConfig.maxAge());
// assertFalse(corsConfig.isCredentialsAllowed());
// }
public void testCorsConfigWithDefaults() {
final Set<String> methods = Strings.commaDelimitedListToSet(SETTING_CORS_ALLOW_METHODS.getDefault(Settings.EMPTY));
final Set<String> headers = Strings.commaDelimitedListToSet(SETTING_CORS_ALLOW_HEADERS.getDefault(Settings.EMPTY));
final long maxAge = SETTING_CORS_MAX_AGE.getDefault(Settings.EMPTY);
final Settings settings = Settings.builder().put(SETTING_CORS_ENABLED.getKey(), true).build();
final NioCorsConfig corsConfig = NioHttpServerTransport.buildCorsConfig(settings);
assertFalse(corsConfig.isAnyOriginSupported());
assertEquals(Collections.emptySet(), corsConfig.origins().get());
assertEquals(headers, corsConfig.allowedRequestHeaders());
assertEquals(methods, corsConfig.allowedRequestMethods().stream().map(HttpMethod::name).collect(Collectors.toSet()));
assertEquals(maxAge, corsConfig.maxAge());
assertFalse(corsConfig.isCredentialsAllowed());
}
/**
* Test that {@link NioHttpServerTransport} supports the "Expect: 100-continue" HTTP header

30
server/src/main/java/org/elasticsearch/http/HttpHandlingSettings.java
View File

@ -19,6 +19,19 @@
package org.elasticsearch.http;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION_LEVEL;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_RESET_COOKIES;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS;
public class HttpHandlingSettings {
private final int maxContentLength;
@ -30,6 +43,7 @@ public class HttpHandlingSettings {
private final int compressionLevel;
private final boolean detailedErrorsEnabled;
private final int pipeliningMaxEvents;
private boolean corsEnabled;
public HttpHandlingSettings(int maxContentLength, int maxChunkSize, int maxHeaderSize, int maxInitialLineLength,
boolean resetCookies, boolean compression, int compressionLevel, boolean detailedErrorsEnabled,
@ -45,6 +59,18 @@ public class HttpHandlingSettings {
this.pipeliningMaxEvents = pipeliningMaxEvents;
}
public static HttpHandlingSettings fromSettings(Settings settings) {
return new HttpHandlingSettings(Math.toIntExact(SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings).getBytes()),
Math.toIntExact(SETTING_HTTP_MAX_CHUNK_SIZE.get(settings).getBytes()),
Math.toIntExact(SETTING_HTTP_MAX_HEADER_SIZE.get(settings).getBytes()),
Math.toIntExact(SETTING_HTTP_MAX_INITIAL_LINE_LENGTH.get(settings).getBytes()),
SETTING_HTTP_RESET_COOKIES.get(settings),
SETTING_HTTP_COMPRESSION.get(settings),
SETTING_HTTP_COMPRESSION_LEVEL.get(settings),
SETTING_HTTP_DETAILED_ERRORS_ENABLED.get(settings),
SETTING_PIPELINING_MAX_EVENTS.get(settings));
}
public int getMaxContentLength() {
return maxContentLength;
}
@ -80,4 +106,8 @@ public class HttpHandlingSettings {
public int getPipeliningMaxEvents() {
return pipeliningMaxEvents;
}
public boolean isCorsEnabled() {
return corsEnabled;
}
}

1
x-pack/docs/build.gradle
View File

@ -81,7 +81,6 @@ buildRestTests.expectedUnconvertedCandidates = [
'en/rest-api/ml/validate-job.asciidoc',
'en/rest-api/security/authenticate.asciidoc',
'en/rest-api/watcher/stats.asciidoc',
'en/security/authorization/managing-roles.asciidoc',
'en/watcher/example-watches/watching-time-series-data.asciidoc',
]

29
x-pack/docs/en/rest-api/index.asciidoc
View File

@ -1,29 +0,0 @@
[role="xpack"]
[[xpack-api]]
= {xpack} APIs
[partintro]
--
{xpack} exposes REST APIs that are used by the UI components and can be called
directly to configure and access {xpack} features.
* <<info-api,Info API>>
* <<graph-explore-api,Graph Explore API>>
* <<licensing-apis,Licensing APIs>>
* <<ml-apis,Machine Learning APIs>>
* <<security-api,Security APIs>>
* <<watcher-api,Watcher APIs>>
* <<rollup-apis,Rollup APIs>>
* <<migration-api,Migration APIs>>
--
include::info.asciidoc[]
include::graph/explore.asciidoc[]
include::licensing.asciidoc[]
include::migration.asciidoc[]
include::ml-api.asciidoc[]
include::rollup-api.asciidoc[]
include::security.asciidoc[]
include::watcher.asciidoc[]
include::defs.asciidoc[]

8
x-pack/docs/en/security/authorization/managing-roles.asciidoc
View File

@ -12,6 +12,8 @@ A role is defined by the following JSON structure:
"indices": [ ... ] <3>
}
-----
// NOTCONSOLE
<1> A list of usernames the owners of this role can <<run-as-privilege, impersonate>>.
<2> A list of cluster privileges. These privileges define the
cluster level actions users with this role are able to execute. This field
@ -37,6 +39,8 @@ The following describes the structure of an indices permissions entry:
"query": "..." <4>
}
-------
// NOTCONSOLE
<1> A list of indices (or index name patterns) to which the permissions in this
entry apply.
<2> The index level privileges the owners of the role have on the associated
@ -77,8 +81,9 @@ The following snippet shows an example definition of a `clicks_admin` role:
[source,js]
-----------
POST /_xpack/security/role/clicks_admin
{
"run_as": [ "clicks_watcher_1" ]
"run_as": [ "clicks_watcher_1" ],
"cluster": [ "monitor" ],
"indices": [
{
@ -92,6 +97,7 @@ The following snippet shows an example definition of a `clicks_admin` role:
]
}
-----------
// CONSOLE
Based on the above definition, users owning the `clicks_admin` role can:

14
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java
View File

@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContent;
@ -173,7 +174,7 @@ public class RollupJobConfig implements NamedWriteable, ToXContentObject {
builder.endObject();
return builder;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(id);
@ -336,6 +337,17 @@ public class RollupJobConfig implements NamedWriteable, ToXContentObject {
if (indexPattern == null || indexPattern.isEmpty()) {
throw new IllegalArgumentException("An index pattern is mandatory.");
}
if (Regex.isMatchAllPattern(indexPattern)) {
throw new IllegalArgumentException("Index pattern must not match all indices (as it would match it's own rollup index");
}
if (Regex.isSimpleMatchPattern(indexPattern)) {
if (Regex.simpleMatch(indexPattern, rollupIndex)) {
throw new IllegalArgumentException("Index pattern would match rollup index name which is not allowed.");
}
}
if (indexPattern.equals(rollupIndex)) {
throw new IllegalArgumentException("Rollup index may not be the same as the index pattern.");
}
if (rollupIndex == null || rollupIndex.isEmpty()) {
throw new IllegalArgumentException("A rollup index name is mandatory.");
}

5
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java
View File

@ -27,8 +27,9 @@ public class ConfigTestHelpers {
builder.setId(jobId);
builder.setCron(getCronString());
builder.setTimeout(new TimeValue(ESTestCase.randomIntBetween(1,100)));
builder.setIndexPattern(ESTestCase.randomAlphaOfLengthBetween(1,10));
builder.setRollupIndex(ESTestCase.randomAlphaOfLengthBetween(1,10));
String indexPattern = ESTestCase.randomAlphaOfLengthBetween(1,10);
builder.setIndexPattern(indexPattern);
builder.setRollupIndex("rollup_" + indexPattern); // to ensure the index pattern != rollup index
builder.setGroupConfig(ConfigTestHelpers.getGroupConfig().build());
builder.setPageSize(ESTestCase.randomIntBetween(1,10));
if (ESTestCase.randomBoolean()) {

4
x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
View File

@ -95,8 +95,8 @@ public class TransportPutRollupJobAction extends TransportMasterNodeAction<PutRo
XPackPlugin.checkReadyForXPackCustomMetadata(clusterState);
FieldCapabilitiesRequest fieldCapsRequest = new FieldCapabilitiesRequest()
.indices(request.getConfig().getIndexPattern())
.fields(request.getConfig().getAllFields().toArray(new String[0]));
.indices(request.getConfig().getIndexPattern())
.fields(request.getConfig().getAllFields().toArray(new String[0]));
client.fieldCaps(fieldCapsRequest, new ActionListener<FieldCapabilitiesResponse>() {
@Override

169
x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
View File

@ -39,6 +39,7 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.index.query.TermsQueryBuilder;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregatorFactories;
@ -66,6 +67,7 @@ import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
@ -271,91 +273,38 @@ public class TransportRollupSearchAction extends TransportAction<SearchRequest,
rewriteQuery(((BoostingQueryBuilder)builder).positiveQuery(), jobCaps));
} else if (builder.getWriteableName().equals(DisMaxQueryBuilder.NAME)) {
DisMaxQueryBuilder rewritten = new DisMaxQueryBuilder();
((DisMaxQueryBuilder)builder).innerQueries().forEach(query -> rewritten.add(rewriteQuery(query, jobCaps)));
((DisMaxQueryBuilder) builder).innerQueries().forEach(query -> rewritten.add(rewriteQuery(query, jobCaps)));
return rewritten;
} else if (builder.getWriteableName().equals(RangeQueryBuilder.NAME) || builder.getWriteableName().equals(TermQueryBuilder.NAME)) {
} else if (builder.getWriteableName().equals(RangeQueryBuilder.NAME)) {
RangeQueryBuilder range = (RangeQueryBuilder) builder;
String fieldName = range.fieldName();
// Many range queries don't include the timezone because the default is UTC, but the query
// builder will return null so we need to set it here
String timeZone = range.timeZone() == null ? DateTimeZone.UTC.toString() : range.timeZone();
String fieldName = builder.getWriteableName().equals(RangeQueryBuilder.NAME)
? ((RangeQueryBuilder)builder).fieldName()
: ((TermQueryBuilder)builder).fieldName();
List<String> incorrectTimeZones = new ArrayList<>();
List<String> rewrittenFieldName = jobCaps.stream()
// We only care about job caps that have the query's target field
.filter(caps -> caps.getFieldCaps().keySet().contains(fieldName))
.map(caps -> {
RollupJobCaps.RollupFieldCaps fieldCaps = caps.getFieldCaps().get(fieldName);
return fieldCaps.getAggs().stream()
// For now, we only allow filtering on grouping fields
.filter(agg -> {
String type = (String)agg.get(RollupField.AGG);
// If the cap is for a date_histo, and the query is a range, the timezones need to match
if (type.equals(DateHistogramAggregationBuilder.NAME) && builder instanceof RangeQueryBuilder) {
String timeZone = ((RangeQueryBuilder)builder).timeZone();
// Many range queries don't include the timezone because the default is UTC, but the query
// builder will return null so we need to set it here
if (timeZone == null) {
timeZone = DateTimeZone.UTC.toString();
}
boolean matchingTZ = ((String)agg.get(DateHistoGroupConfig.TIME_ZONE.getPreferredName()))
.equalsIgnoreCase(timeZone);
if (matchingTZ == false) {
incorrectTimeZones.add((String)agg.get(DateHistoGroupConfig.TIME_ZONE.getPreferredName()));
}
return matchingTZ;
}
// Otherwise just make sure it's one of the three groups
return type.equals(TermsAggregationBuilder.NAME)
|| type.equals(DateHistogramAggregationBuilder.NAME)
|| type.equals(HistogramAggregationBuilder.NAME);
})
// Rewrite the field name to our convention (e.g. "foo" -> "date_histogram.foo.timestamp")
.map(agg -> {
if (agg.get(RollupField.AGG).equals(DateHistogramAggregationBuilder.NAME)) {
return RollupField.formatFieldName(fieldName, (String)agg.get(RollupField.AGG), RollupField.TIMESTAMP);
} else {
return RollupField.formatFieldName(fieldName, (String)agg.get(RollupField.AGG), RollupField.VALUE);
}
})
.collect(Collectors.toList());
})
.distinct()
.collect(ArrayList::new, List::addAll, List::addAll);
if (rewrittenFieldName.isEmpty()) {
if (incorrectTimeZones.isEmpty()) {
throw new IllegalArgumentException("Field [" + fieldName + "] in [" + builder.getWriteableName()
+ "] query is not available in selected rollup indices, cannot query.");
} else {
throw new IllegalArgumentException("Field [" + fieldName + "] in [" + builder.getWriteableName()
+ "] query was found in rollup indices, but requested timezone is not compatible. Options include: "
+ incorrectTimeZones);
}
String rewrittenFieldName = rewriteFieldName(jobCaps, RangeQueryBuilder.NAME, fieldName, timeZone);
RangeQueryBuilder rewritten = new RangeQueryBuilder(rewrittenFieldName)
.from(range.from())
.to(range.to())
.includeLower(range.includeLower())
.includeUpper(range.includeUpper());
if (range.timeZone() != null) {
rewritten.timeZone(range.timeZone());
}
if (rewrittenFieldName.size() > 1) {
throw new IllegalArgumentException("Ambiguous field name resolution when mapping to rolled fields. Field name [" +
fieldName + "] was mapped to: [" + Strings.collectionToDelimitedString(rewrittenFieldName, ",") + "].");
if (range.format() != null) {
rewritten.format(range.format());
}
//Note: instanceof here to make casting checks happier
if (builder instanceof RangeQueryBuilder) {
RangeQueryBuilder rewritten = new RangeQueryBuilder(rewrittenFieldName.get(0));
RangeQueryBuilder original = (RangeQueryBuilder)builder;
rewritten.from(original.from());
rewritten.to(original.to());
if (original.timeZone() != null) {
rewritten.timeZone(original.timeZone());
}
rewritten.includeLower(original.includeLower());
rewritten.includeUpper(original.includeUpper());
return rewritten;
} else {
return new TermQueryBuilder(rewrittenFieldName.get(0), ((TermQueryBuilder)builder).value());
}
return rewritten;
} else if (builder.getWriteableName().equals(TermQueryBuilder.NAME)) {
TermQueryBuilder term = (TermQueryBuilder) builder;
String fieldName = term.fieldName();
String rewrittenFieldName = rewriteFieldName(jobCaps, TermQueryBuilder.NAME, fieldName, null);
return new TermQueryBuilder(rewrittenFieldName, term.value());
} else if (builder.getWriteableName().equals(TermsQueryBuilder.NAME)) {
TermsQueryBuilder terms = (TermsQueryBuilder) builder;
String fieldName = terms.fieldName();
String rewrittenFieldName = rewriteFieldName(jobCaps, TermQueryBuilder.NAME, fieldName, null);
return new TermsQueryBuilder(rewrittenFieldName, terms.values());
} else if (builder.getWriteableName().equals(MatchAllQueryBuilder.NAME)) {
// no-op
return builder;
@ -364,6 +313,64 @@ public class TransportRollupSearchAction extends TransportAction<SearchRequest,
}
}
private static String rewriteFieldName(Set<RollupJobCaps> jobCaps,
String builderName,
String fieldName,
String timeZone) {
List<String> incompatibleTimeZones = timeZone == null ? Collections.emptyList() : new ArrayList<>();
List<String> rewrittenFieldNames = jobCaps.stream()
// We only care about job caps that have the query's target field
.filter(caps -> caps.getFieldCaps().keySet().contains(fieldName))
.map(caps -> {
RollupJobCaps.RollupFieldCaps fieldCaps = caps.getFieldCaps().get(fieldName);
return fieldCaps.getAggs().stream()
// For now, we only allow filtering on grouping fields
.filter(agg -> {
String type = (String)agg.get(RollupField.AGG);
// If the cap is for a date_histo, and the query is a range, the timezones need to match
if (type.equals(DateHistogramAggregationBuilder.NAME) && timeZone != null) {
boolean matchingTZ = ((String)agg.get(DateHistoGroupConfig.TIME_ZONE.getPreferredName()))
.equalsIgnoreCase(timeZone);
if (matchingTZ == false) {
incompatibleTimeZones.add((String)agg.get(DateHistoGroupConfig.TIME_ZONE.getPreferredName()));
}
return matchingTZ;
}
// Otherwise just make sure it's one of the three groups
return type.equals(TermsAggregationBuilder.NAME)
|| type.equals(DateHistogramAggregationBuilder.NAME)
|| type.equals(HistogramAggregationBuilder.NAME);
})
// Rewrite the field name to our convention (e.g. "foo" -> "date_histogram.foo.timestamp")
.map(agg -> {
if (agg.get(RollupField.AGG).equals(DateHistogramAggregationBuilder.NAME)) {
return RollupField.formatFieldName(fieldName, (String)agg.get(RollupField.AGG), RollupField.TIMESTAMP);
} else {
return RollupField.formatFieldName(fieldName, (String)agg.get(RollupField.AGG), RollupField.VALUE);
}
})
.collect(Collectors.toList());
})
.distinct()
.collect(ArrayList::new, List::addAll, List::addAll);
if (rewrittenFieldNames.isEmpty()) {
if (incompatibleTimeZones.isEmpty()) {
throw new IllegalArgumentException("Field [" + fieldName + "] in [" + builderName
+ "] query is not available in selected rollup indices, cannot query.");
} else {
throw new IllegalArgumentException("Field [" + fieldName + "] in [" + builderName
+ "] query was found in rollup indices, but requested timezone is not compatible. Options include: "
+ incompatibleTimeZones);
}
} else if (rewrittenFieldNames.size() > 1) {
throw new IllegalArgumentException("Ambiguous field name resolution when mapping to rolled fields. Field name [" +
fieldName + "] was mapped to: [" + Strings.collectionToDelimitedString(rewrittenFieldNames, ",") + "].");
} else {
return rewrittenFieldNames.get(0);
}
}
static RollupSearchContext separateIndices(String[] indices, ImmutableOpenMap<String, IndexMetaData> indexMetaData) {
if (indices.length == 0) {

22
x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
View File

@ -25,9 +25,11 @@ import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
import org.elasticsearch.index.query.DisMaxQueryBuilder;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.index.query.TermsQueryBuilder;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.script.ScriptService;
@ -61,6 +63,7 @@ import org.mockito.Mockito;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@ -153,7 +156,7 @@ public class SearchActionTests extends ESTestCase {
"compatible. Options include: [UTC]"));
}
public void testTerms() {
public void testTermQuery() {
RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
group.setTerms(ConfigTestHelpers.getTerms().setFields(Collections.singletonList("foo")).build());
@ -166,6 +169,23 @@ public class SearchActionTests extends ESTestCase {
assertThat(((TermQueryBuilder)rewritten).fieldName(), equalTo("foo.terms.value"));
}
public void testTermsQuery() {
RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
group.setTerms(ConfigTestHelpers.getTerms().setFields(Collections.singletonList("foo")).build());
job.setGroupConfig(group.build());
RollupJobCaps cap = new RollupJobCaps(job.build());
Set<RollupJobCaps> caps = new HashSet<>();
caps.add(cap);
QueryBuilder original = new TermsQueryBuilder("foo", Arrays.asList("bar", "baz"));
QueryBuilder rewritten =
TransportRollupSearchAction.rewriteQuery(original, caps);
assertThat(rewritten, instanceOf(TermsQueryBuilder.class));
assertNotSame(rewritten, original);
assertThat(((TermsQueryBuilder)rewritten).fieldName(), equalTo("foo.terms.value"));
assertThat(((TermsQueryBuilder)rewritten).values(), equalTo(Arrays.asList("bar", "baz")));
}
public void testCompounds() {
RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();

31
x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java
View File

@ -122,6 +122,37 @@ public class ConfigTests extends ESTestCase {
assertThat(e.getMessage(), equalTo("An index pattern is mandatory."));
}
public void testMatchAllIndexPattern() {
RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
job.setIndexPattern("*");
Exception e = expectThrows(IllegalArgumentException.class, job::build);
assertThat(e.getMessage(), equalTo("Index pattern must not match all indices (as it would match it's own rollup index"));
}
public void testMatchOwnRollupPatternPrefix() {
RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
job.setIndexPattern("foo-*");
job.setRollupIndex("foo-rollup");
Exception e = expectThrows(IllegalArgumentException.class, job::build);
assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed."));
}
public void testMatchOwnRollupPatternSuffix() {
RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
job.setIndexPattern("*-rollup");
job.setRollupIndex("foo-rollup");
Exception e = expectThrows(IllegalArgumentException.class, job::build);
assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed."));
}
public void testIndexPatternIdenticalToRollup() {
RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
job.setIndexPattern("foo");
job.setRollupIndex("foo");
Exception e = expectThrows(IllegalArgumentException.class, job::build);
assertThat(e.getMessage(), equalTo("Rollup index may not be the same as the index pattern."));
}
public void testEmptyRollupIndex() {
RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
job.setRollupIndex("");

1
x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
View File

@ -188,4 +188,3 @@ setup:
]
}