diff --git a/build.gradle b/build.gradle index 807e3cfd222..c2bf6c9309f 100644 --- a/build.gradle +++ b/build.gradle @@ -175,6 +175,7 @@ subprojects { "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage', // for transport client "org.elasticsearch.plugin:transport-netty3-client:${version}": ':modules:transport-netty3', + "org.elasticsearch.plugin:transport-netty4-client:${version}": ':modules:transport-netty4', "org.elasticsearch.plugin:reindex-client:${version}": ':modules:reindex', "org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache', "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator', diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index f825819cb3e..16f3535d9f3 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -297,6 +297,10 @@ class BuildPlugin implements Plugin { url "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" } } + repos.maven { + name 'netty-snapshots' + url "http://s3.amazonaws.com/download.elasticsearch.org/nettysnapshots/20160722" + } } /** Returns a closure which can be used with a MavenPom for removing transitive dependencies. */ diff --git a/client/transport/build.gradle b/client/transport/build.gradle index 3c4f967f766..3573467b0ef 100644 --- a/client/transport/build.gradle +++ b/client/transport/build.gradle @@ -26,6 +26,7 @@ group = 'org.elasticsearch.client' dependencies { compile "org.elasticsearch:elasticsearch:${version}" compile project(path: ':modules:transport-netty3', configuration: 'runtime') + compile project(path: ':modules:transport-netty4', configuration: 'runtime') compile project(path: ':modules:reindex', configuration: 'runtime') compile project(path: ':modules:lang-mustache', configuration: 'runtime') compile project(path: ':modules:percolator', configuration: 'runtime') diff --git a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java index 30d4beddae2..aebcf5fa3a3 100644 --- a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java +++ b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java @@ -16,30 +16,47 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.transport.client; import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.reindex.ReindexPlugin; import org.elasticsearch.percolator.PercolatorPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.mustache.MustachePlugin; import org.elasticsearch.transport.Netty3Plugin; +import org.elasticsearch.transport.Netty4Plugin; import java.util.Arrays; import java.util.Collection; import java.util.Collections; - +import java.util.List; /** * A builder to create an instance of {@link TransportClient} - * This class pre-installs the {@link Netty3Plugin}, {@link ReindexPlugin}, {@link PercolatorPlugin}, and {@link MustachePlugin} + * This class pre-installs the + * {@link Netty3Plugin}, + * {@link Netty4Plugin}, + * {@link ReindexPlugin}, + * {@link PercolatorPlugin}, + * and {@link MustachePlugin} * for the client. These plugins are all elasticsearch core modules required. */ @SuppressWarnings({"unchecked","varargs"}) public class PreBuiltTransportClient extends TransportClient { - private static final Collection> PRE_INSTALLED_PLUGINS = Collections.unmodifiableList(Arrays.asList( - TransportPlugin.class, ReindexPlugin.class, PercolatorPlugin.class, MustachePlugin.class)); + + private static final Collection> PRE_INSTALLED_PLUGINS = + Collections.unmodifiableList( + Arrays.asList( + Netty3Plugin.class, + Netty4Plugin.class, + TransportPlugin.class, + ReindexPlugin.class, + PercolatorPlugin.class, + MustachePlugin.class)); @SafeVarargs public PreBuiltTransportClient(Settings settings, Class... plugins) { @@ -50,14 +67,25 @@ public class PreBuiltTransportClient extends TransportClient { super(settings, Settings.EMPTY, addPlugins(plugins, PRE_INSTALLED_PLUGINS)); } - /** - * The default transport implementation for the transport client. - */ - public static final class TransportPlugin extends Netty3Plugin { - // disable assertions for permissions since we might not have the permissions here - // compared to if we are loaded as a real module to the es server - public TransportPlugin(Settings settings) { - super(Settings.builder().put("netty.assert.buglevel", false).put(settings).build()); + public static final class TransportPlugin extends Plugin { + + private static final Setting ASSERT_NETTY_BUGLEVEL = + Setting.boolSetting("netty.assert.buglevel", true, Setting.Property.NodeScope); + + @Override + public List> getSettings() { + return Collections.singletonList(ASSERT_NETTY_BUGLEVEL); } + + @Override + public Settings additionalSettings() { + return Settings.builder() + .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME) + .put(NetworkModule.HTTP_TYPE_KEY, Netty3Plugin.NETTY_HTTP_TRANSPORT_NAME) + .put("netty.assert.buglevel", true) + .build(); + } + } + } diff --git a/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java b/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java index ed300b70021..5b72006f5f0 100644 --- a/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java +++ b/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.transport.client; import com.carrotsearch.randomizedtesting.RandomizedTest; @@ -57,4 +58,5 @@ public class PreBuiltTransportClientTests extends RandomizedTest { } } } + } diff --git a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java index ef1102326d6..b336acfba20 100644 --- a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java +++ b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java @@ -21,7 +21,6 @@ package org.elasticsearch.common.bytes; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ByteArray; @@ -36,24 +35,24 @@ public class PagedBytesReference extends BytesReference { private static final int PAGE_SIZE = BigArrays.BYTE_PAGE_SIZE; private final BigArrays bigarrays; - protected final ByteArray bytearray; + protected final ByteArray byteArray; private final int offset; private final int length; - public PagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int length) { - this(bigarrays, bytearray, 0, length); + public PagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int length) { + this(bigarrays, byteArray, 0, length); } - public PagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int from, int length) { + public PagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int from, int length) { this.bigarrays = bigarrays; - this.bytearray = bytearray; + this.byteArray = byteArray; this.offset = from; this.length = length; } @Override public byte get(int index) { - return bytearray.get(offset + index); + return byteArray.get(offset + index); } @Override @@ -66,14 +65,14 @@ public class PagedBytesReference extends BytesReference { if (from < 0 || (from + length) > length()) { throw new IllegalArgumentException("can't slice a buffer with length [" + length() + "], with slice parameters from [" + from + "], length [" + length + "]"); } - return new PagedBytesReference(bigarrays, bytearray, offset + from, length); + return new PagedBytesReference(bigarrays, byteArray, offset + from, length); } @Override public BytesRef toBytesRef() { BytesRef bref = new BytesRef(); // if length <= pagesize this will dereference the page, or materialize the byte[] - bytearray.get(offset, length, bref); + byteArray.get(offset, length, bref); return bref; } @@ -95,7 +94,7 @@ public class PagedBytesReference extends BytesReference { @Override public BytesRef next() throws IOException { if (nextFragmentSize != 0) { - final boolean materialized = bytearray.get(offset + position, nextFragmentSize, slice); + final boolean materialized = byteArray.get(offset + position, nextFragmentSize, slice); assert materialized == false : "iteration should be page aligned but array got materialized"; position += nextFragmentSize; final int remaining = length - position; @@ -111,6 +110,6 @@ public class PagedBytesReference extends BytesReference { @Override public long ramBytesUsed() { - return bytearray.ramBytesUsed(); + return byteArray.ramBytesUsed(); } } diff --git a/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java index 2152aa226a8..2700ea4dc13 100644 --- a/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java +++ b/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java @@ -30,12 +30,13 @@ import org.elasticsearch.common.util.ByteArray; */ public final class ReleasablePagedBytesReference extends PagedBytesReference implements Releasable { - public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int length) { - super(bigarrays, bytearray, length); + public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int length) { + super(bigarrays, byteArray, length); } @Override public void close() { - Releasables.close(bytearray); + Releasables.close(byteArray); } + } diff --git a/core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java b/core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java index 15bc324a346..e31f206bcad 100644 --- a/core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java +++ b/core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java @@ -28,4 +28,5 @@ public interface ReleasableBytesStream extends BytesStream { @Override ReleasablePagedBytesReference bytes(); + } diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java index 2d2719a113e..3de5c757ae1 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java @@ -33,7 +33,7 @@ import java.io.IOException; */ public class BytesStreamOutput extends StreamOutput implements BytesStream { - protected final BigArrays bigarrays; + protected final BigArrays bigArrays; protected ByteArray bytes; protected int count; @@ -57,9 +57,9 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream { this(expectedSize, BigArrays.NON_RECYCLING_INSTANCE); } - protected BytesStreamOutput(int expectedSize, BigArrays bigarrays) { - this.bigarrays = bigarrays; - this.bytes = bigarrays.newByteArray(expectedSize); + protected BytesStreamOutput(int expectedSize, BigArrays bigArrays) { + this.bigArrays = bigArrays; + this.bytes = bigArrays.newByteArray(expectedSize); } @Override @@ -100,7 +100,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream { public void reset() { // shrink list of pages if (bytes.size() > BigArrays.PAGE_SIZE_IN_BYTES) { - bytes = bigarrays.resize(bytes, BigArrays.PAGE_SIZE_IN_BYTES); + bytes = bigArrays.resize(bytes, BigArrays.PAGE_SIZE_IN_BYTES); } // go back to start @@ -145,7 +145,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream { @Override public BytesReference bytes() { - return new PagedBytesReference(bigarrays, bytes, count); + return new PagedBytesReference(bigArrays, bytes, count); } /** @@ -157,7 +157,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream { } private void ensureCapacity(int offset) { - bytes = bigarrays.grow(bytes, offset); + bytes = bigArrays.grow(bytes, offset); } } diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java index 4e5380e66d5..674ff18f0fc 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java @@ -36,12 +36,13 @@ public class ReleasableBytesStreamOutput extends BytesStreamOutput implements Re super(BigArrays.PAGE_SIZE_IN_BYTES, bigarrays); } - public ReleasableBytesStreamOutput(int expectedSize, BigArrays bigarrays) { - super(expectedSize, bigarrays); + public ReleasableBytesStreamOutput(int expectedSize, BigArrays bigArrays) { + super(expectedSize, bigArrays); } @Override public ReleasablePagedBytesReference bytes() { - return new ReleasablePagedBytesReference(bigarrays, bytes, count); + return new ReleasablePagedBytesReference(bigArrays, bytes, count); } + } diff --git a/core/src/main/java/org/elasticsearch/common/lease/Releasable.java b/core/src/main/java/org/elasticsearch/common/lease/Releasable.java index a86d812016e..61b1e85c749 100644 --- a/core/src/main/java/org/elasticsearch/common/lease/Releasable.java +++ b/core/src/main/java/org/elasticsearch/common/lease/Releasable.java @@ -30,4 +30,5 @@ public interface Releasable extends Closeable { @Override void close(); + } diff --git a/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java b/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java index 7fe5562f17a..16b0bd00443 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java +++ b/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java @@ -29,4 +29,5 @@ import org.elasticsearch.rest.RestRequest; public interface HttpServerAdapter { void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext context); + } diff --git a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java index ab8b2bd1797..0ec57e2bfc2 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java +++ b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java @@ -22,9 +22,6 @@ package org.elasticsearch.http; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.transport.BoundTransportAddress; -/** - * - */ public interface HttpServerTransport extends LifecycleComponent { BoundTransportAddress boundAddress(); @@ -34,4 +31,5 @@ public interface HttpServerTransport extends LifecycleComponent { HttpStats stats(); void httpServerAdapter(HttpServerAdapter httpServerAdapter); + } diff --git a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index aa00943ed08..f5d4f4eb695 100644 --- a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -101,4 +101,5 @@ public abstract class AbstractRestChannel implements RestChannel { public boolean detailedErrorsEnabled() { return detailedErrorsEnabled; } + } diff --git a/core/src/main/java/org/elasticsearch/rest/RestChannel.java b/core/src/main/java/org/elasticsearch/rest/RestChannel.java index d895285ca8c..2a56313fd8a 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/RestChannel.java @@ -30,6 +30,7 @@ import java.io.IOException; * A channel used to construct bytes / builder based outputs, and send responses. */ public interface RestChannel { + XContentBuilder newBuilder() throws IOException; XContentBuilder newErrorBuilder() throws IOException; @@ -46,4 +47,5 @@ public interface RestChannel { boolean detailedErrorsEnabled(); void sendResponse(RestResponse response); + } diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java index 7072c4d59de..cf7c811a84c 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestController.java +++ b/core/src/main/java/org/elasticsearch/rest/RestController.java @@ -193,7 +193,7 @@ public class RestController extends AbstractLifecycleComponent { if (!checkRequestParameters(request, channel)) { return; } - try (ThreadContext.StoredContext t = threadContext.stashContext()) { + try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { for (String key : headersToCopy) { String httpHeader = request.header(key); if (httpHeader != null) { diff --git a/core/src/main/java/org/elasticsearch/rest/RestRequest.java b/core/src/main/java/org/elasticsearch/rest/RestRequest.java index 8872484d589..897ba294039 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/core/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -29,16 +29,35 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.rest.support.RestUtils; import java.net.SocketAddress; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import static org.elasticsearch.common.unit.ByteSizeValue.parseBytesSizeValue; import static org.elasticsearch.common.unit.TimeValue.parseTimeValue; -/** - * - */ public abstract class RestRequest implements ToXContent.Params { + private final Map params; + private final String rawPath; + + public RestRequest(String uri) { + final Map params = new HashMap<>(); + int pathEndPos = uri.indexOf('?'); + if (pathEndPos < 0) { + this.rawPath = uri; + } else { + this.rawPath = uri.substring(0, pathEndPos); + RestUtils.decodeQueryString(uri, pathEndPos + 1, params); + } + this.params = params; + } + + public RestRequest(Map params, String path) { + this.params = params; + this.rawPath = path; + } + public enum Method { GET, POST, PUT, DELETE, OPTIONS, HEAD } @@ -53,7 +72,9 @@ public abstract class RestRequest implements ToXContent.Params { /** * The non decoded, raw path provided. */ - public abstract String rawPath(); + public String rawPath() { + return rawPath; + } /** * The path part of the URI (without the query string), decoded. @@ -80,12 +101,27 @@ public abstract class RestRequest implements ToXContent.Params { return null; } - public abstract boolean hasParam(String key); + public final boolean hasParam(String key) { + return params.containsKey(key); + } @Override - public abstract String param(String key); + public final String param(String key) { + return params.get(key); + } - public abstract Map params(); + @Override + public final String param(String key, String defaultValue) { + String value = params.get(key); + if (value == null) { + return defaultValue; + } + return value; + } + + public Map params() { + return params; + } public float paramAsFloat(String key, float defaultValue) { String sValue = param(key); diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java index 0e601ecb5b5..7145777aad7 100644 --- a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -93,6 +93,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -1076,23 +1077,23 @@ public abstract class TcpTransport extends AbstractLifecycleComponent i if (buffer.get(offset) != 'E' || buffer.get(offset + 1) != 'S') { // special handling for what is probably HTTP if (bufferStartsWith(buffer, offset, "GET ") || - bufferStartsWith(buffer, offset, "POST ") || - bufferStartsWith(buffer, offset, "PUT ") || - bufferStartsWith(buffer, offset, "HEAD ") || - bufferStartsWith(buffer, offset, "DELETE ") || - bufferStartsWith(buffer, offset, "OPTIONS ") || - bufferStartsWith(buffer, offset, "PATCH ") || - bufferStartsWith(buffer, offset, "TRACE ")) { + bufferStartsWith(buffer, offset, "POST ") || + bufferStartsWith(buffer, offset, "PUT ") || + bufferStartsWith(buffer, offset, "HEAD ") || + bufferStartsWith(buffer, offset, "DELETE ") || + bufferStartsWith(buffer, offset, "OPTIONS ") || + bufferStartsWith(buffer, offset, "PATCH ") || + bufferStartsWith(buffer, offset, "TRACE ")) { throw new HttpOnTransportException("This is not a HTTP port"); } // we have 6 readable bytes, show 4 (should be enough) throw new StreamCorruptedException("invalid internal transport message format, got (" - + Integer.toHexString(buffer.get(offset) & 0xFF) + "," - + Integer.toHexString(buffer.get(offset + 1) & 0xFF) + "," - + Integer.toHexString(buffer.get(offset + 2) & 0xFF) + "," - + Integer.toHexString(buffer.get(offset + 3) & 0xFF) + ")"); + + Integer.toHexString(buffer.get(offset) & 0xFF) + "," + + Integer.toHexString(buffer.get(offset + 1) & 0xFF) + "," + + Integer.toHexString(buffer.get(offset + 2) & 0xFF) + "," + + Integer.toHexString(buffer.get(offset + 3) & 0xFF) + ")"); } final int dataLen; @@ -1111,7 +1112,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent i // safety against too large frames being sent if (dataLen > NINETY_PER_HEAP_SIZE) { throw new IllegalArgumentException("transport content length received [" + new ByteSizeValue(dataLen) + "] exceeded [" - + new ByteSizeValue(NINETY_PER_HEAP_SIZE) + "]"); + + new ByteSizeValue(NINETY_PER_HEAP_SIZE) + "]"); } if (buffer.length() < dataLen + sizeHeaderLength) { @@ -1159,7 +1160,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent i public final void messageReceived(BytesReference reference, Channel channel, String profileName, InetSocketAddress remoteAddress, int messageLengthBytes) throws IOException { final int totalMessageSize = messageLengthBytes + TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE; - transportServiceAdapter.received(totalMessageSize); + transportServiceAdapter.addBytesReceived(totalMessageSize); // we have additional bytes to read, outside of the header boolean hasMessageBytesToRead = (totalMessageSize - TcpHeader.HEADER_SIZE) > 0; StreamInput streamIn = reference.streamInput(); diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java index 16c9ede851b..1fceb5aa1a3 100644 --- a/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java +++ b/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java @@ -23,9 +23,6 @@ import org.elasticsearch.Version; import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; -/** - * - */ public final class TcpTransportChannel implements TransportChannel { private final TcpTransport transport; protected final Version version; diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java index 765bb988777..15164a5d201 100644 --- a/core/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java @@ -625,12 +625,12 @@ public class TransportService extends AbstractLifecycleComponent { final MeanMetric txMetric = new MeanMetric(); @Override - public void received(long size) { + public void addBytesReceived(long size) { rxMetric.inc(size); } @Override - public void sent(long size) { + public void addBytesSent(long size) { txMetric.inc(size); } diff --git a/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java b/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java index 5a1b9e9b349..34910532a02 100644 --- a/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java +++ b/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java @@ -21,14 +21,11 @@ package org.elasticsearch.transport; import org.elasticsearch.cluster.node.DiscoveryNode; -/** - * - */ public interface TransportServiceAdapter { - void received(long size); + void addBytesReceived(long size); - void sent(long size); + void addBytesSent(long size); /** called by the {@link Transport} implementation once a request has been sent */ void onRequestSent(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options); @@ -57,4 +54,5 @@ public interface TransportServiceAdapter { void raiseNodeConnected(DiscoveryNode node); void raiseNodeDisconnected(DiscoveryNode node); + } diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java index c94e62ea422..61559442ff3 100644 --- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java +++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java @@ -229,7 +229,7 @@ public class LocalTransport extends AbstractLifecycleComponent implements Transp } final byte[] data = BytesReference.toBytes(stream.bytes()); - transportServiceAdapter.sent(data.length); + transportServiceAdapter.addBytesSent(data.length); transportServiceAdapter.onRequestSent(node, requestId, action, request, options); targetTransport.receiveMessage(version, data, action, requestId, this); } @@ -272,7 +272,7 @@ public class LocalTransport extends AbstractLifecycleComponent implements Transp @Nullable final Long sendRequestId) { Transports.assertTransportThread(); try { - transportServiceAdapter.received(data.length); + transportServiceAdapter.addBytesReceived(data.length); StreamInput stream = StreamInput.wrap(data); stream.setVersion(version); diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java index f9de466e978..87167cdb733 100644 --- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java +++ b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java @@ -189,11 +189,11 @@ public class HttpServerTests extends ESTestCase { } private static final class TestRestRequest extends RestRequest { - private final String path; + private final BytesReference content; private TestRestRequest(String path, String content) { - this.path = path; + super(Collections.emptyMap(), path); this.content = new BytesArray(content); } @@ -207,11 +207,6 @@ public class HttpServerTests extends ESTestCase { return null; } - @Override - public String rawPath() { - return path; - } - @Override public boolean hasContent() { return true; @@ -232,24 +227,5 @@ public class HttpServerTests extends ESTestCase { return null; } - @Override - public boolean hasParam(String key) { - return false; - } - - @Override - public String param(String key) { - return null; - } - - @Override - public String param(String key, String defaultValue) { - return null; - } - - @Override - public Map params() { - return null; - } } } diff --git a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java index 051159b448b..642b3a53a34 100644 --- a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java +++ b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.Index; import org.elasticsearch.rest.support.RestUtils; import org.elasticsearch.search.SearchShardTarget; @@ -33,11 +34,15 @@ import org.elasticsearch.transport.RemoteTransportException; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.Collections; +import java.util.Map; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -157,8 +162,37 @@ public class BytesRestResponseTests extends ESTestCase { public void testResponseWhenPathContainsEncodingError() throws IOException { final String path = "%a"; - final RestRequest request = mock(RestRequest.class); - when(request.rawPath()).thenReturn(path); + final RestRequest request = new RestRequest(Collections.emptyMap(), path) { + @Override + public Method method() { + return null; + } + + @Override + public String uri() { + return null; + } + + @Override + public boolean hasContent() { + return false; + } + + @Override + public BytesReference content() { + return null; + } + + @Override + public String header(String name) { + return null; + } + + @Override + public Iterable> headers() { + return null; + } + }; final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestUtils.decodeComponent(request.rawPath())); final RestChannel channel = new DetailedExceptionRestChannel(request); // if we try to decode the path, this will throw an IllegalArgumentException again diff --git a/distribution/build.gradle b/distribution/build.gradle index dcd5f170fc5..bfbf96b5d2a 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -72,7 +72,8 @@ project.rootProject.subprojects.findAll { it.path.startsWith(':modules:') }.each // See https://discuss.gradle.org/t/cross-project-task-dependencies-ordering-screws-up-finalizers/13190 project.configure(project.subprojects.findAll { it.name != 'integ-test-zip' }) { Project distribution -> distribution.afterEvaluate({ - distribution.integTest.mustRunAfter("${module.path}:integTest#stop") + // some integTest tasks will have multiple finalizers + distribution.integTest.mustRunAfter module.tasks.find { t -> t.name.matches(".*integTest\$") }.getFinalizedBy() }) } // also want to make sure the module's integration tests run after the integ-test-zip (ie rest tests) diff --git a/distribution/integ-test-zip/build.gradle b/distribution/integ-test-zip/build.gradle index 67f99aa884a..f1f3a2c26c5 100644 --- a/distribution/integ-test-zip/build.gradle +++ b/distribution/integ-test-zip/build.gradle @@ -36,4 +36,3 @@ publishing { } integTest.dependsOn buildZip - diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 0d02d1d9474..e159b3af8c5 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -35,7 +35,6 @@ run { setting 'reindex.remote.whitelist', 'myself' } - dependencies { compile "org.elasticsearch.client:rest:${version}" // dependencies of the rest client @@ -45,6 +44,7 @@ dependencies { compile "commons-logging:commons-logging:${versions.commonslogging}" // for http - testing reindex from remote testCompile project(path: ':modules:transport-netty3', configuration: 'runtime') + testCompile project(path: ':modules:transport-netty4', configuration: 'runtime') } dependencyLicenses { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java index fd251d96a1f..2be27a1a1ad 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Netty3Plugin; +import org.elasticsearch.transport.Netty4Plugin; import org.junit.After; import org.junit.Before; @@ -56,17 +57,47 @@ import static org.hamcrest.Matchers.hasSize; * tests won't verify that. */ public class RetryTests extends ESSingleNodeTestCase { + private static final int DOC_COUNT = 20; private List blockedExecutors = new ArrayList<>(); + private boolean useNetty4; + + @Before + public void setUp() throws Exception { + super.setUp(); + useNetty4 = randomBoolean(); + createIndex("source"); + // Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools. + BulkRequestBuilder bulk = client().prepareBulk(); + for (int i = 0; i < DOC_COUNT; i++) { + bulk.add(client().prepareIndex("source", "test").setSource("foo", "bar " + i)); + } + Retry retry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.exponentialBackoff()); + BulkResponse response = retry.withSyncBackoff(client(), bulk.request()); + assertFalse(response.buildFailureMessage(), response.hasFailures()); + client().admin().indices().prepareRefresh("source").get(); + } + + @After + public void forceUnblockAllExecutors() { + for (CyclicBarrier barrier: blockedExecutors) { + barrier.reset(); + } + } + @Override protected Collection> getPlugins() { - return pluginList(ReindexPlugin.class, Netty3Plugin.class, BogusPlugin.class); // we need netty here to http communication + return pluginList( + ReindexPlugin.class, + Netty3Plugin.class, + Netty4Plugin.class, + BogusPlugin.class); } public static final class BogusPlugin extends Plugin { - // se Netty3Plugin.... this runs without the permission from the netty3 module so it will fail since reindex can't set the property + // this runs without the permission from the netty module so it will fail since reindex can't set the property // to make it still work we disable that check but need to register the setting first private static final Setting ASSERT_NETTY_BUGLEVEL = Setting.boolSetting("netty.assert.buglevel", true, Setting.Property.NodeScope); @@ -94,30 +125,13 @@ public class RetryTests extends ESSingleNodeTestCase { settings.put(NetworkModule.HTTP_ENABLED.getKey(), true); // Whitelist reindexing from the http host we're going to use settings.put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "myself"); + if (useNetty4) { + settings.put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME); + settings.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME); + } return settings.build(); } - @Before - public void setupSourceIndex() throws Exception { - createIndex("source"); - // Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools. - BulkRequestBuilder bulk = client().prepareBulk(); - for (int i = 0; i < DOC_COUNT; i++) { - bulk.add(client().prepareIndex("source", "test").setSource("foo", "bar " + i)); - } - Retry retry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.exponentialBackoff()); - BulkResponse response = retry.withSyncBackoff(client(), bulk.request()); - assertFalse(response.buildFailureMessage(), response.hasFailures()); - client().admin().indices().prepareRefresh("source").get(); - } - - @After - public void forceUnblockAllExecutors() { - for (CyclicBarrier barrier: blockedExecutors) { - barrier.reset(); - } - } - public void testReindex() throws Exception { testCase(ReindexAction.NAME, ReindexAction.INSTANCE.newRequestBuilder(client()).source("source").destination("dest"), matcher().created(DOC_COUNT)); @@ -222,4 +236,5 @@ public class RetryTests extends ESSingleNodeTestCase { assertThat(response.getTasks(), hasSize(1)); return (BulkByScrollTask.Status) response.getTasks().get(0).getStatus(); } + } diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java index 39a5c94b239..abf23563b8a 100644 --- a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java +++ b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java @@ -26,40 +26,27 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.support.RestUtils; import org.jboss.netty.channel.Channel; import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpRequest; import java.net.SocketAddress; import java.util.HashMap; import java.util.Map; -/** - * - */ public class Netty3HttpRequest extends RestRequest { - private final org.jboss.netty.handler.codec.http.HttpRequest request; + private final HttpRequest request; private final Channel channel; - private final Map params; - private final String rawPath; private final BytesReference content; - public Netty3HttpRequest(org.jboss.netty.handler.codec.http.HttpRequest request, Channel channel) { + public Netty3HttpRequest(HttpRequest request, Channel channel) { + super(request.getUri()); this.request = request; this.channel = channel; - this.params = new HashMap<>(); if (request.getContent().readable()) { this.content = Netty3Utils.toBytesReference(request.getContent()); } else { this.content = BytesArray.EMPTY; } - - String uri = request.getUri(); - int pathEndPos = uri.indexOf('?'); - if (pathEndPos < 0) { - this.rawPath = uri; - } else { - this.rawPath = uri.substring(0, pathEndPos); - RestUtils.decodeQueryString(uri, pathEndPos + 1, params); - } } public org.jboss.netty.handler.codec.http.HttpRequest request() { @@ -97,16 +84,6 @@ public class Netty3HttpRequest extends RestRequest { return request.getUri(); } - @Override - public String rawPath() { - return rawPath; - } - - @Override - public Map params() { - return params; - } - @Override public boolean hasContent() { return content.length() > 0; @@ -153,22 +130,4 @@ public class Netty3HttpRequest extends RestRequest { return request.headers().entries(); } - @Override - public boolean hasParam(String key) { - return params.containsKey(key); - } - - @Override - public String param(String key) { - return params.get(key); - } - - @Override - public String param(String key, String defaultValue) { - String value = params.get(key); - if (value == null) { - return defaultValue; - } - return value; - } } diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java index 968eb6a24ab..829a2d9aa06 100644 --- a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java +++ b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java @@ -54,7 +54,6 @@ public class Netty3HttpRequestHandler extends SimpleChannelUpstreamHandler { request = (HttpRequest) e.getMessage(); } - threadContext.copyHeaders(request.headers()); // the netty HTTP handling always copy over the buffer to its own buffer, either in NioWorker internally // when reading, or using a cumalation buffer Netty3HttpRequest httpRequest = new Netty3HttpRequest(request, e.getChannel()); diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java index edbcc74e646..25884d14b70 100644 --- a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java +++ b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java @@ -120,29 +120,29 @@ public class Netty3HttpServerTransport extends AbstractLifecycleComponent implem public static Setting SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY = Setting.byteSizeSetting("http.netty.max_cumulation_buffer_capacity", new ByteSizeValue(-1), - Property.NodeScope); + Property.NodeScope, Property.Shared); public static Setting SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = - Setting.intSetting("http.netty.max_composite_buffer_components", -1, Property.NodeScope); + Setting.intSetting("http.netty.max_composite_buffer_components", -1, Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_WORKER_COUNT = new Setting<>("http.netty.worker_count", (s) -> Integer.toString(EsExecutors.boundedNumberOfProcessors(s) * 2), - (s) -> Setting.parseInt(s, 1, "http.netty.worker_count"), Property.NodeScope); + (s) -> Setting.parseInt(s, 1, "http.netty.worker_count"), Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_TCP_NO_DELAY = - boolSetting("http.tcp_no_delay", NetworkService.TcpSettings.TCP_NO_DELAY, Property.NodeScope); + boolSetting("http.tcp_no_delay", NetworkService.TcpSettings.TCP_NO_DELAY, Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_TCP_KEEP_ALIVE = - boolSetting("http.tcp.keep_alive", NetworkService.TcpSettings.TCP_KEEP_ALIVE, Property.NodeScope); + boolSetting("http.tcp.keep_alive", NetworkService.TcpSettings.TCP_KEEP_ALIVE, Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_TCP_BLOCKING_SERVER = - boolSetting("http.tcp.blocking_server", NetworkService.TcpSettings.TCP_BLOCKING_SERVER, Property.NodeScope); + boolSetting("http.tcp.blocking_server", NetworkService.TcpSettings.TCP_BLOCKING_SERVER, Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_TCP_REUSE_ADDRESS = - boolSetting("http.tcp.reuse_address", NetworkService.TcpSettings.TCP_REUSE_ADDRESS, Property.NodeScope); + boolSetting("http.tcp.reuse_address", NetworkService.TcpSettings.TCP_REUSE_ADDRESS, Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_TCP_SEND_BUFFER_SIZE = Setting.byteSizeSetting("http.tcp.send_buffer_size", NetworkService.TcpSettings.TCP_SEND_BUFFER_SIZE, - Property.NodeScope); + Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE = Setting.byteSizeSetting("http.tcp.receive_buffer_size", NetworkService.TcpSettings.TCP_RECEIVE_BUFFER_SIZE, - Property.NodeScope); + Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE = Setting.byteSizeSetting("transport.netty.receive_predictor_size", settings -> { diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java index ff4dc27e2c8..e2ac10873cd 100644 --- a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java +++ b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java @@ -50,7 +50,7 @@ class Netty3MessageChannelHandler extends SimpleChannelUpstreamHandler { @Override public void writeComplete(ChannelHandlerContext ctx, WriteCompletionEvent e) throws Exception { - transportServiceAdapter.sent(e.getWrittenAmount()); + transportServiceAdapter.addBytesSent(e.getWrittenAmount()); super.writeComplete(ctx, e); } diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java index 5ca2252cf2c..f5f32044eb7 100644 --- a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java +++ b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.Booleans; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.network.NetworkService.TcpSettings; @@ -45,6 +46,7 @@ import org.elasticsearch.transport.TransportServiceAdapter; import org.elasticsearch.transport.TransportSettings; import org.jboss.netty.bootstrap.ClientBootstrap; import org.jboss.netty.bootstrap.ServerBootstrap; +import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.channel.AdaptiveReceiveBufferSizePredictorFactory; import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.ChannelFuture; @@ -93,31 +95,36 @@ public class Netty3Transport extends TcpTransport { public static final Setting WORKER_COUNT = new Setting<>("transport.netty.worker_count", (s) -> Integer.toString(EsExecutors.boundedNumberOfProcessors(s) * 2), - (s) -> Setting.parseInt(s, 1, "transport.netty.worker_count"), Property.NodeScope); + (s) -> Setting.parseInt(s, 1, "transport.netty.worker_count"), Property.NodeScope, Property.Shared); public static final Setting NETTY_MAX_CUMULATION_BUFFER_CAPACITY = - Setting.byteSizeSetting("transport.netty.max_cumulation_buffer_capacity", new ByteSizeValue(-1), Property.NodeScope); + Setting.byteSizeSetting( + "transport.netty.max_cumulation_buffer_capacity", + new ByteSizeValue(-1), + Property.NodeScope, + Property.Shared); public static final Setting NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = - Setting.intSetting("transport.netty.max_composite_buffer_components", -1, -1, Property.NodeScope); + Setting.intSetting("transport.netty.max_composite_buffer_components", -1, -1, Property.NodeScope, Property.Shared); // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one public static final Setting NETTY_RECEIVE_PREDICTOR_SIZE = Setting.byteSizeSetting( - "transport.netty.receive_predictor_size", - settings -> { - long defaultReceiverPredictor = 512 * 1024; - if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) { - // we can guess a better default... - long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / WORKER_COUNT.get(settings)); - defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024)); - } - return new ByteSizeValue(defaultReceiverPredictor).toString(); - }, Property.NodeScope); + "transport.netty.receive_predictor_size", + settings -> { + long defaultReceiverPredictor = 512 * 1024; + if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) { + // we can guess a better default... + long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / WORKER_COUNT.get(settings)); + defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024)); + } + return new ByteSizeValue(defaultReceiverPredictor).toString(); + }, Property.NodeScope, + Property.Shared); public static final Setting NETTY_RECEIVE_PREDICTOR_MIN = - byteSizeSetting("transport.netty.receive_predictor_min", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope); + byteSizeSetting("transport.netty.receive_predictor_min", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared); public static final Setting NETTY_RECEIVE_PREDICTOR_MAX = - byteSizeSetting("transport.netty.receive_predictor_max", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope); + byteSizeSetting("transport.netty.receive_predictor_max", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared); public static final Setting NETTY_BOSS_COUNT = - intSetting("transport.netty.boss_count", 1, 1, Property.NodeScope); + intSetting("transport.netty.boss_count", 1, 1, Property.NodeScope, Property.Shared); protected final ByteSizeValue maxCumulationBufferCapacity; @@ -557,4 +564,5 @@ public class Netty3Transport extends TcpTransport { } }); } + } diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/ESNetty3IntegTestCase.java b/modules/transport-netty3/src/test/java/org/elasticsearch/ESNetty3IntegTestCase.java index a7f8f254fcc..3c90bc4ccea 100644 --- a/modules/transport-netty3/src/test/java/org/elasticsearch/ESNetty3IntegTestCase.java +++ b/modules/transport-netty3/src/test/java/org/elasticsearch/ESNetty3IntegTestCase.java @@ -47,15 +47,15 @@ public abstract class ESNetty3IntegTestCase extends ESIntegTestCase { if (randomBoolean()) { builder.put(Netty3Transport.WORKER_COUNT.getKey(), random().nextInt(3) + 1); } - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, "netty3"); - builder.put(NetworkModule.HTTP_TYPE_KEY, "netty3"); + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME); + builder.put(NetworkModule.HTTP_TYPE_KEY, Netty3Plugin.NETTY_HTTP_TRANSPORT_NAME); return builder.build(); } @Override protected Settings transportClientSettings() { Settings.Builder builder = Settings.builder().put(super.transportClientSettings()); - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, "netty3"); + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME); return builder.build(); } diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java index 8b9b2d397db..d25951e254c 100644 --- a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java +++ b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java @@ -73,7 +73,7 @@ public class Netty3TransportMultiPortIntegrationIT extends ESNetty3IntegTestCase public void testThatTransportClientCanConnect() throws Exception { Settings settings = Settings.builder() .put("cluster.name", internalCluster().getClusterName()) - .put(NetworkModule.TRANSPORT_TYPE_KEY, "netty3") + .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME) .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .build(); try (TransportClient transportClient = new MockTransportClient(settings, Netty3Plugin.class)) { diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle new file mode 100644 index 00000000000..82d98a981ab --- /dev/null +++ b/modules/transport-netty4/build.gradle @@ -0,0 +1,170 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + TODOs: + * fix permissions such that only netty4 can open sockets etc? + * fix the hack in the build framework that copies transport-netty4 into the integ test cluster + * maybe figure out a way to run all tests from core with netty4/network? + */ +esplugin { + description 'Netty 4 based transport implementation' + classname 'org.elasticsearch.transport.Netty4Plugin' + hasClientJar = true +} + +compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" + +dependencies { + // network stack + compile "io.netty:netty-buffer:4.1.4.Final-elastic-SNAPSHOT" + compile "io.netty:netty-codec:4.1.4.Final-elastic-SNAPSHOT" + compile "io.netty:netty-codec-http:4.1.4.Final-elastic-SNAPSHOT" + compile "io.netty:netty-common:4.1.4.Final-elastic-SNAPSHOT" + compile "io.netty:netty-handler:4.1.4.Final-elastic-SNAPSHOT" + compile "io.netty:netty-resolver:4.1.4.Final-elastic-SNAPSHOT" + compile "io.netty:netty-transport:4.1.4.Final-elastic-SNAPSHOT" +} + +integTest { + includePackaged = true + cluster { + setting 'http.type', 'netty4' + setting 'transport.type', 'netty4' + numNodes = 2 + } +} + +thirdPartyAudit.excludes = [ + // classes are missing + + // from io.netty.handler.codec.protobuf.ProtobufDecoder (netty) + 'com.google.protobuf.ExtensionRegistry', + 'com.google.protobuf.MessageLite$Builder', + 'com.google.protobuf.MessageLite', + 'com.google.protobuf.Parser', + + // from io.netty.logging.CommonsLoggerFactory (netty) + 'org.apache.commons.logging.Log', + 'org.apache.commons.logging.LogFactory', + + // from io.netty.handler.ssl.OpenSslEngine (netty) + 'org.apache.tomcat.jni.Buffer', + 'org.apache.tomcat.jni.Library', + 'org.apache.tomcat.jni.Pool', + 'org.apache.tomcat.jni.SSL', + 'org.apache.tomcat.jni.SSLContext', + + // from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty) + 'org.bouncycastle.asn1.x500.X500Name', + 'org.bouncycastle.cert.X509v3CertificateBuilder', + 'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter', + 'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder', + 'org.bouncycastle.jce.provider.BouncyCastleProvider', + 'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder', + + // from io.netty.handler.ssl.JettyNpnSslEngine (netty) + 'org.eclipse.jetty.npn.NextProtoNego$ClientProvider', + 'org.eclipse.jetty.npn.NextProtoNego$ServerProvider', + 'org.eclipse.jetty.npn.NextProtoNego', + + // from io.netty.handler.codec.marshalling.ChannelBufferByteInput (netty) + 'org.jboss.marshalling.ByteInput', + + // from io.netty.handler.codec.marshalling.ChannelBufferByteOutput (netty) + 'org.jboss.marshalling.ByteOutput', + + // from io.netty.handler.codec.marshalling.CompatibleMarshallingEncoder (netty) + 'org.jboss.marshalling.Marshaller', + + // from io.netty.handler.codec.marshalling.ContextBoundUnmarshallerProvider (netty) + 'org.jboss.marshalling.MarshallerFactory', + 'org.jboss.marshalling.MarshallingConfiguration', + 'org.jboss.marshalling.Unmarshaller', + + // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional + 'org.slf4j.Logger', + 'org.slf4j.LoggerFactory', + + 'com.google.protobuf.ExtensionRegistryLite', + 'com.google.protobuf.MessageLiteOrBuilder', + 'com.google.protobuf.nano.CodedOutputByteBufferNano', + 'com.google.protobuf.nano.MessageNano', + 'com.jcraft.jzlib.Deflater', + 'com.jcraft.jzlib.Inflater', + 'com.jcraft.jzlib.JZlib$WrapperType', + 'com.jcraft.jzlib.JZlib', + 'com.ning.compress.BufferRecycler', + 'com.ning.compress.lzf.ChunkDecoder', + 'com.ning.compress.lzf.ChunkEncoder', + 'com.ning.compress.lzf.LZFEncoder', + 'com.ning.compress.lzf.util.ChunkDecoderFactory', + 'com.ning.compress.lzf.util.ChunkEncoderFactory', + 'javassist/ClassClassPath', + 'javassist/ClassPath', + 'javassist/ClassPool', + 'javassist.CtClass', + 'javassist.CtMethod', + 'lzma.sdk.lzma.Encoder', + 'net.jpountz.lz4.LZ4Compressor', + 'net.jpountz.lz4.LZ4Factory', + 'net.jpountz.lz4.LZ4FastDecompressor', + 'net.jpountz.xxhash.StreamingXXHash32', + 'net.jpountz.xxhash.XXHashFactory', + 'org.apache.logging.log4j.LogManager', + 'org.apache.logging.log4j.Logger', + 'org.apache.tomcat.jni.CertificateRequestedCallback', + 'org.apache.tomcat.jni.CertificateVerifier', + 'org.apache.tomcat.jni.SessionTicketKey', + 'org.eclipse.jetty.alpn.ALPN$ClientProvider', + 'org.eclipse.jetty.alpn.ALPN$ServerProvider', + 'org.eclipse.jetty.alpn.ALPN', + + 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator', + 'io.netty.util.internal.PlatformDependent0', + 'io.netty.util.internal.UnsafeAtomicIntegerFieldUpdater', + 'io.netty.util.internal.UnsafeAtomicLongFieldUpdater', + 'io.netty.util.internal.UnsafeAtomicReferenceFieldUpdater', + 'io.netty.util.internal.chmv8.ConcurrentHashMapV8', + 'io.netty.util.internal.chmv8.ConcurrentHashMapV8$1', + 'io.netty.util.internal.chmv8.ConcurrentHashMapV8$TreeBin', + 'io.netty.util.internal.chmv8.CountedCompleter', + 'io.netty.util.internal.chmv8.CountedCompleter$1', + 'io.netty.util.internal.chmv8.ForkJoinPool', + 'io.netty.util.internal.chmv8.ForkJoinPool$2', + 'io.netty.util.internal.chmv8.ForkJoinPool$WorkQueue', + 'io.netty.util.internal.chmv8.ForkJoinTask', + 'io.netty.util.internal.chmv8.ForkJoinTask$1', + 'io.netty.util.internal.chmv8.Striped64', + 'io.netty.util.internal.chmv8.Striped64$1', + 'io.netty.util.internal.chmv8.Striped64$Cell', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef', + 'io.netty.util.internal.shaded.org.jctools.queues.ConcurrentSequencedCircularArrayQueue', + 'io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode', + 'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueHeadLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueTailField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscChunkedArrayQueue', + 'io.netty.util.internal.shaded.org.jctools.util.JvmInfo', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', +] diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..e45d72cb77e --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +bae159f72e78d211733db0f13b0d8436a65af5ac \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-LICENSE.txt b/modules/transport-netty4/licenses/netty-buffer-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/transport-netty4/licenses/netty-buffer-NOTICE.txt b/modules/transport-netty4/licenses/netty-buffer-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..a1872082d0f --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +a8fb268e5756768ed9ddc46806cf724481c14298 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-LICENSE.txt b/modules/transport-netty4/licenses/netty-codec-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/transport-netty4/licenses/netty-codec-NOTICE.txt b/modules/transport-netty4/licenses/netty-codec-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..7c96920d161 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +8219258943eaa26115a8f39b16163b4d0e53770e \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-LICENSE.txt b/modules/transport-netty4/licenses/netty-codec-http-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/transport-netty4/licenses/netty-codec-http-NOTICE.txt b/modules/transport-netty4/licenses/netty-codec-http-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/licenses/netty-common-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..0e775de8650 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +8f2a9aafddf7a5db56e6c7dbbc94834e5bdeb186 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-LICENSE.txt b/modules/transport-netty4/licenses/netty-common-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/transport-netty4/licenses/netty-common-NOTICE.txt b/modules/transport-netty4/licenses/netty-common-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..e91cb888a44 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +99a2d3a7285195c7f42d25f236212b984d0dfcb7 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-LICENSE.txt b/modules/transport-netty4/licenses/netty-handler-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/transport-netty4/licenses/netty-handler-NOTICE.txt b/modules/transport-netty4/licenses/netty-handler-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..951fcbf0f80 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +1f832049b94980a9f8d01ff87a83170c5a03f7eb \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-LICENSE.txt b/modules/transport-netty4/licenses/netty-resolver-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/transport-netty4/licenses/netty-resolver-NOTICE.txt b/modules/transport-netty4/licenses/netty-resolver-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..4b7743836ca --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +16a2ffc9a952a8c1deccca462e7f9965608db2b2 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-LICENSE.txt b/modules/transport-netty4/licenses/netty-transport-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/transport-netty4/licenses/netty-transport-NOTICE.txt b/modules/transport-netty4/licenses/netty-transport-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java new file mode 100644 index 00000000000..62443dc541e --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java @@ -0,0 +1,268 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaderValues; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.cookie.ServerCookieDecoder; +import io.netty.handler.codec.http.cookie.ServerCookieEncoder; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.http.netty4.cors.Netty4CorsHandler; +import org.elasticsearch.http.netty4.pipelining.HttpPipelinedRequest; +import org.elasticsearch.rest.AbstractRestChannel; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.transport.netty4.Netty4Utils; + +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +final class Netty4HttpChannel extends AbstractRestChannel { + + private final Netty4HttpServerTransport transport; + private final Channel channel; + private final FullHttpRequest nettyRequest; + private final HttpPipelinedRequest pipelinedRequest; + private final ThreadContext threadContext; + + /** + * @param transport The corresponding NettyHttpServerTransport where this channel belongs to. + * @param request The request that is handled by this channel. + * @param pipelinedRequest If HTTP pipelining is enabled provide the corresponding pipelined request. May be null if + * HTTP pipelining is disabled. + * @param detailedErrorsEnabled true iff error messages should include stack traces. + * @param threadContext the thread context for the channel + */ + Netty4HttpChannel( + final Netty4HttpServerTransport transport, + final Netty4HttpRequest request, + final HttpPipelinedRequest pipelinedRequest, + final boolean detailedErrorsEnabled, + final ThreadContext threadContext) { + super(request, detailedErrorsEnabled); + this.transport = transport; + this.channel = request.getChannel(); + this.nettyRequest = request.request(); + this.pipelinedRequest = pipelinedRequest; + this.threadContext = threadContext; + } + + @Override + public BytesStreamOutput newBytesOutput() { + return new ReleasableBytesStreamOutput(transport.bigArrays); + } + + + @Override + public void sendResponse(RestResponse response) { + // if the response object was created upstream, then use it; + // otherwise, create a new one + ByteBuf buffer = Netty4Utils.toByteBuf(response.content()); + FullHttpResponse resp = newResponse(buffer); + resp.setStatus(getStatus(response.status())); + + Netty4CorsHandler.setCorsResponseHeaders(nettyRequest, resp, transport.getCorsConfig()); + + String opaque = nettyRequest.headers().get("X-Opaque-Id"); + if (opaque != null) { + setHeaderField(resp, "X-Opaque-Id", opaque); + } + + // Add all custom headers + addCustomHeaders(resp, response.getHeaders()); + addCustomHeaders(resp, threadContext.getResponseHeaders()); + + BytesReference content = response.content(); + boolean release = content instanceof Releasable; + try { + // If our response doesn't specify a content-type header, set one + setHeaderField(resp, HttpHeaderNames.CONTENT_TYPE.toString(), response.contentType(), false); + // If our response has no content-length, calculate and set one + setHeaderField(resp, HttpHeaderNames.CONTENT_LENGTH.toString(), String.valueOf(buffer.readableBytes()), false); + + addCookies(resp); + + final ChannelPromise promise = channel.newPromise(); + + if (release) { + promise.addListener(f -> ((Releasable)content).close()); + release = false; + } + + if (isCloseConnection()) { + promise.addListener(ChannelFutureListener.CLOSE); + } + + if (pipelinedRequest != null) { + channel.writeAndFlush(pipelinedRequest.createHttpResponse(resp, promise)); + } else { + channel.writeAndFlush(resp, promise); + } + + } finally { + if (release) { + ((Releasable) content).close(); + } + if (pipelinedRequest != null) { + pipelinedRequest.release(); + } + } + } + + private void setHeaderField(HttpResponse resp, String headerField, String value) { + setHeaderField(resp, headerField, value, true); + } + + private void setHeaderField(HttpResponse resp, String headerField, String value, boolean override) { + if (override || !resp.headers().contains(headerField)) { + resp.headers().add(headerField, value); + } + } + + private void addCookies(HttpResponse resp) { + if (transport.resetCookies) { + String cookieString = nettyRequest.headers().get(HttpHeaders.Names.COOKIE); + if (cookieString != null) { + Set cookies = ServerCookieDecoder.STRICT.decode(cookieString); + if (!cookies.isEmpty()) { + // Reset the cookies if necessary. + resp.headers().set(HttpHeaderNames.SET_COOKIE, ServerCookieEncoder.STRICT.encode(cookies)); + } + } + } + } + + private void addCustomHeaders(HttpResponse response, Map> customHeaders) { + if (customHeaders != null) { + for (Map.Entry> headerEntry : customHeaders.entrySet()) { + for (String headerValue : headerEntry.getValue()) { + setHeaderField(response, headerEntry.getKey(), headerValue); + } + } + } + } + + // Determine if the request protocol version is HTTP 1.0 + private boolean isHttp10() { + return nettyRequest.protocolVersion().equals(HttpVersion.HTTP_1_0); + } + + // Determine if the request connection should be closed on completion. + private boolean isCloseConnection() { + final boolean http10 = isHttp10(); + return HttpHeaderValues.CLOSE.equals(nettyRequest.headers().get(HttpHeaderNames.CONNECTION)) || + (http10 && HttpHeaderValues.KEEP_ALIVE.equals(nettyRequest.headers().get(HttpHeaderNames.CONNECTION)) == false); + } + + // Create a new {@link HttpResponse} to transmit the response for the netty request. + private FullHttpResponse newResponse(ByteBuf buffer) { + final boolean http10 = isHttp10(); + final boolean close = isCloseConnection(); + // Build the response object. + final HttpResponseStatus status = HttpResponseStatus.OK; // default to initialize + final FullHttpResponse response; + if (http10) { + response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_0, status, buffer); + if (!close) { + response.headers().add(HttpHeaderNames.CONNECTION, "Keep-Alive"); + } + } else { + response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status, buffer); + } + return response; + } + + private static final HttpResponseStatus TOO_MANY_REQUESTS = new HttpResponseStatus(429, "Too Many Requests"); + + private static Map MAP; + + static { + EnumMap map = new EnumMap<>(RestStatus.class); + map.put(RestStatus.CONTINUE, HttpResponseStatus.CONTINUE); + map.put(RestStatus.SWITCHING_PROTOCOLS, HttpResponseStatus.SWITCHING_PROTOCOLS); + map.put(RestStatus.OK, HttpResponseStatus.OK); + map.put(RestStatus.CREATED, HttpResponseStatus.CREATED); + map.put(RestStatus.ACCEPTED, HttpResponseStatus.ACCEPTED); + map.put(RestStatus.NON_AUTHORITATIVE_INFORMATION, HttpResponseStatus.NON_AUTHORITATIVE_INFORMATION); + map.put(RestStatus.NO_CONTENT, HttpResponseStatus.NO_CONTENT); + map.put(RestStatus.RESET_CONTENT, HttpResponseStatus.RESET_CONTENT); + map.put(RestStatus.PARTIAL_CONTENT, HttpResponseStatus.PARTIAL_CONTENT); + map.put(RestStatus.MULTI_STATUS, HttpResponseStatus.INTERNAL_SERVER_ERROR); // no status for this?? + map.put(RestStatus.MULTIPLE_CHOICES, HttpResponseStatus.MULTIPLE_CHOICES); + map.put(RestStatus.MOVED_PERMANENTLY, HttpResponseStatus.MOVED_PERMANENTLY); + map.put(RestStatus.FOUND, HttpResponseStatus.FOUND); + map.put(RestStatus.SEE_OTHER, HttpResponseStatus.SEE_OTHER); + map.put(RestStatus.NOT_MODIFIED, HttpResponseStatus.NOT_MODIFIED); + map.put(RestStatus.USE_PROXY, HttpResponseStatus.USE_PROXY); + map.put(RestStatus.TEMPORARY_REDIRECT, HttpResponseStatus.TEMPORARY_REDIRECT); + map.put(RestStatus.BAD_REQUEST, HttpResponseStatus.BAD_REQUEST); + map.put(RestStatus.UNAUTHORIZED, HttpResponseStatus.UNAUTHORIZED); + map.put(RestStatus.PAYMENT_REQUIRED, HttpResponseStatus.PAYMENT_REQUIRED); + map.put(RestStatus.FORBIDDEN, HttpResponseStatus.FORBIDDEN); + map.put(RestStatus.NOT_FOUND, HttpResponseStatus.NOT_FOUND); + map.put(RestStatus.METHOD_NOT_ALLOWED, HttpResponseStatus.METHOD_NOT_ALLOWED); + map.put(RestStatus.NOT_ACCEPTABLE, HttpResponseStatus.NOT_ACCEPTABLE); + map.put(RestStatus.PROXY_AUTHENTICATION, HttpResponseStatus.PROXY_AUTHENTICATION_REQUIRED); + map.put(RestStatus.REQUEST_TIMEOUT, HttpResponseStatus.REQUEST_TIMEOUT); + map.put(RestStatus.CONFLICT, HttpResponseStatus.CONFLICT); + map.put(RestStatus.GONE, HttpResponseStatus.GONE); + map.put(RestStatus.LENGTH_REQUIRED, HttpResponseStatus.LENGTH_REQUIRED); + map.put(RestStatus.PRECONDITION_FAILED, HttpResponseStatus.PRECONDITION_FAILED); + map.put(RestStatus.REQUEST_ENTITY_TOO_LARGE, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE); + map.put(RestStatus.REQUEST_URI_TOO_LONG, HttpResponseStatus.REQUEST_URI_TOO_LONG); + map.put(RestStatus.UNSUPPORTED_MEDIA_TYPE, HttpResponseStatus.UNSUPPORTED_MEDIA_TYPE); + map.put(RestStatus.REQUESTED_RANGE_NOT_SATISFIED, HttpResponseStatus.REQUESTED_RANGE_NOT_SATISFIABLE); + map.put(RestStatus.EXPECTATION_FAILED, HttpResponseStatus.EXPECTATION_FAILED); + map.put(RestStatus.UNPROCESSABLE_ENTITY, HttpResponseStatus.BAD_REQUEST); + map.put(RestStatus.LOCKED, HttpResponseStatus.BAD_REQUEST); + map.put(RestStatus.FAILED_DEPENDENCY, HttpResponseStatus.BAD_REQUEST); + map.put(RestStatus.TOO_MANY_REQUESTS, TOO_MANY_REQUESTS); + map.put(RestStatus.INTERNAL_SERVER_ERROR, HttpResponseStatus.INTERNAL_SERVER_ERROR); + map.put(RestStatus.NOT_IMPLEMENTED, HttpResponseStatus.NOT_IMPLEMENTED); + map.put(RestStatus.BAD_GATEWAY, HttpResponseStatus.BAD_GATEWAY); + map.put(RestStatus.SERVICE_UNAVAILABLE, HttpResponseStatus.SERVICE_UNAVAILABLE); + map.put(RestStatus.GATEWAY_TIMEOUT, HttpResponseStatus.GATEWAY_TIMEOUT); + map.put(RestStatus.HTTP_VERSION_NOT_SUPPORTED, HttpResponseStatus.HTTP_VERSION_NOT_SUPPORTED); + MAP = Collections.unmodifiableMap(map); + } + + private static HttpResponseStatus getStatus(RestStatus status) { + return MAP.getOrDefault(status, HttpResponseStatus.INTERNAL_SERVER_ERROR); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java new file mode 100644 index 00000000000..949faa848e7 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java @@ -0,0 +1,134 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpRequest; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.transport.netty4.Netty4Utils; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.support.RestUtils; +import io.netty.channel.Channel; +import io.netty.handler.codec.http.HttpMethod; + +import java.net.SocketAddress; +import java.util.HashMap; +import java.util.Map; + +class Netty4HttpRequest extends RestRequest { + + private final FullHttpRequest request; + private final Channel channel; + private final BytesReference content; + + Netty4HttpRequest(FullHttpRequest request, Channel channel) { + super(request.uri()); + this.request = request; + this.channel = channel; + if (request.content().isReadable()) { + this.content = Netty4Utils.toBytesReference(request.content()); + } else { + this.content = BytesArray.EMPTY; + } + } + + public FullHttpRequest request() { + return this.request; + } + + @Override + public Method method() { + HttpMethod httpMethod = request.method(); + if (httpMethod == HttpMethod.GET) + return Method.GET; + + if (httpMethod == HttpMethod.POST) + return Method.POST; + + if (httpMethod == HttpMethod.PUT) + return Method.PUT; + + if (httpMethod == HttpMethod.DELETE) + return Method.DELETE; + + if (httpMethod == HttpMethod.HEAD) { + return Method.HEAD; + } + + if (httpMethod == HttpMethod.OPTIONS) { + return Method.OPTIONS; + } + + return Method.GET; + } + + @Override + public String uri() { + return request.uri(); + } + + @Override + public boolean hasContent() { + return content.length() > 0; + } + + @Override + public BytesReference content() { + return content; + } + + /** + * Returns the remote address where this rest request channel is "connected to". The + * returned {@link SocketAddress} is supposed to be down-cast into more + * concrete type such as {@link java.net.InetSocketAddress} to retrieve + * the detailed information. + */ + @Override + public SocketAddress getRemoteAddress() { + return channel.remoteAddress(); + } + + /** + * Returns the local address where this request channel is bound to. The returned + * {@link SocketAddress} is supposed to be down-cast into more concrete + * type such as {@link java.net.InetSocketAddress} to retrieve the detailed + * information. + */ + @Override + public SocketAddress getLocalAddress() { + return channel.localAddress(); + } + + public Channel getChannel() { + return channel; + } + + @Override + public String header(String name) { + return request.headers().get(name); + } + + @Override + public Iterable> headers() { + return request.headers().entries(); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java new file mode 100644 index 00000000000..a4402d4816f --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.http.netty4.pipelining.HttpPipelinedRequest; + +@ChannelHandler.Sharable +class Netty4HttpRequestHandler extends SimpleChannelInboundHandler { + + private final Netty4HttpServerTransport serverTransport; + private final boolean httpPipeliningEnabled; + private final boolean detailedErrorsEnabled; + private final ThreadContext threadContext; + + Netty4HttpRequestHandler(Netty4HttpServerTransport serverTransport, boolean detailedErrorsEnabled, ThreadContext threadContext) { + this.serverTransport = serverTransport; + this.httpPipeliningEnabled = serverTransport.pipelining; + this.detailedErrorsEnabled = detailedErrorsEnabled; + this.threadContext = threadContext; + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception { + final FullHttpRequest request; + final HttpPipelinedRequest pipelinedRequest; + if (this.httpPipeliningEnabled && msg instanceof HttpPipelinedRequest) { + pipelinedRequest = (HttpPipelinedRequest) msg; + request = (FullHttpRequest) pipelinedRequest.last(); + } else { + pipelinedRequest = null; + request = (FullHttpRequest) msg; + } + + final FullHttpRequest copy = + new DefaultFullHttpRequest( + request.protocolVersion(), + request.method(), + request.uri(), + Unpooled.copiedBuffer(request.content()), + request.headers(), + request.trailingHeaders()); + + final Netty4HttpRequest httpRequest = new Netty4HttpRequest(copy, ctx.channel()); + serverTransport.dispatchRequest( + httpRequest, + new Netty4HttpChannel(serverTransport, httpRequest, pipelinedRequest, detailedErrorsEnabled, threadContext)); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + serverTransport.exceptionCaught(ctx, cause); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java new file mode 100644 index 00000000000..910a499a194 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -0,0 +1,571 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import com.carrotsearch.hppc.IntHashSet; +import com.carrotsearch.hppc.IntSet; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.AdaptiveRecvByteBufAllocator; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.FixedRecvByteBufAllocator; +import io.netty.channel.RecvByteBufAllocator; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.oio.OioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.oio.OioServerSocketChannel; +import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.handler.codec.http.HttpContentCompressor; +import io.netty.handler.codec.http.HttpContentDecompressor; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpRequestDecoder; +import io.netty.handler.codec.http.HttpResponseEncoder; +import io.netty.handler.timeout.ReadTimeoutException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.BoundTransportAddress; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.NetworkExceptionHelper; +import org.elasticsearch.common.transport.PortsRange; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.http.BindHttpException; +import org.elasticsearch.http.HttpInfo; +import org.elasticsearch.http.HttpServerAdapter; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.http.HttpStats; +import org.elasticsearch.http.netty4.cors.Netty4CorsConfig; +import org.elasticsearch.http.netty4.cors.Netty4CorsConfigBuilder; +import org.elasticsearch.http.netty4.cors.Netty4CorsHandler; +import org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler; +import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.support.RestUtils; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.BindTransportException; +import org.elasticsearch.transport.netty4.Netty4OpenChannelsHandler; +import org.elasticsearch.transport.netty4.Netty4Utils; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; + +import static org.elasticsearch.common.settings.Setting.boolSetting; +import static org.elasticsearch.common.settings.Setting.byteSizeSetting; +import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_BIND_HOST; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION_LEVEL; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PORT; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_PORT; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_RESET_COOKIES; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_PIPELINING; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS; +import static org.elasticsearch.http.netty4.cors.Netty4CorsHandler.ANY_ORIGIN; + +public class Netty4HttpServerTransport extends AbstractLifecycleComponent implements HttpServerTransport { + + static { + Netty4Utils.setup(); + } + + public static Setting SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY = + Setting.byteSizeSetting("http.netty.max_cumulation_buffer_capacity", new ByteSizeValue(-1), + Property.NodeScope, Property.Shared); + public static Setting SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = + Setting.intSetting("http.netty.max_composite_buffer_components", -1, Property.NodeScope, Property.Shared); + + public static final Setting SETTING_HTTP_WORKER_COUNT = new Setting<>("http.netty.worker_count", + (s) -> Integer.toString(EsExecutors.boundedNumberOfProcessors(s) * 2), + (s) -> Setting.parseInt(s, 1, "http.netty.worker_count"), Property.NodeScope, Property.Shared); + + public static final Setting SETTING_HTTP_TCP_NO_DELAY = + boolSetting("http.tcp_no_delay", NetworkService.TcpSettings.TCP_NO_DELAY, Property.NodeScope, Property.Shared); + public static final Setting SETTING_HTTP_TCP_KEEP_ALIVE = + boolSetting("http.tcp.keep_alive", NetworkService.TcpSettings.TCP_KEEP_ALIVE, Property.NodeScope, Property.Shared); + public static final Setting SETTING_HTTP_TCP_BLOCKING_SERVER = + boolSetting("http.tcp.blocking_server", NetworkService.TcpSettings.TCP_BLOCKING_SERVER, Property.NodeScope, Property.Shared); + public static final Setting SETTING_HTTP_TCP_REUSE_ADDRESS = + boolSetting("http.tcp.reuse_address", NetworkService.TcpSettings.TCP_REUSE_ADDRESS, Property.NodeScope, Property.Shared); + + public static final Setting SETTING_HTTP_TCP_SEND_BUFFER_SIZE = + Setting.byteSizeSetting("http.tcp.send_buffer_size", NetworkService.TcpSettings.TCP_SEND_BUFFER_SIZE, + Property.NodeScope, Property.Shared); + public static final Setting SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE = + Setting.byteSizeSetting("http.tcp.receive_buffer_size", NetworkService.TcpSettings.TCP_RECEIVE_BUFFER_SIZE, + Property.NodeScope, Property.Shared); + public static final Setting SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE = + Setting.byteSizeSetting("transport.netty.receive_predictor_size", + settings -> { + long defaultReceiverPredictor = 512 * 1024; + if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) { + // we can guess a better default... + long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / SETTING_HTTP_WORKER_COUNT.get + (settings)); + defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024)); + } + return new ByteSizeValue(defaultReceiverPredictor).toString(); + }, Property.NodeScope); + public static final Setting SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN = + byteSizeSetting("http.netty.receive_predictor_min", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope); + public static final Setting SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX = + byteSizeSetting("http.netty.receive_predictor_max", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope); + + + protected final NetworkService networkService; + protected final BigArrays bigArrays; + + protected final ByteSizeValue maxContentLength; + protected final ByteSizeValue maxInitialLineLength; + protected final ByteSizeValue maxHeaderSize; + protected final ByteSizeValue maxChunkSize; + + protected final int workerCount; + + protected final boolean blockingServer; + + protected final boolean pipelining; + + protected final int pipeliningMaxEvents; + + protected final boolean compression; + + protected final int compressionLevel; + + protected final boolean resetCookies; + + protected final PortsRange port; + + protected final String bindHosts[]; + + protected final String publishHosts[]; + + protected final boolean detailedErrorsEnabled; + protected final ThreadPool threadPool; + + protected final boolean tcpNoDelay; + protected final boolean tcpKeepAlive; + protected final boolean reuseAddress; + + protected final ByteSizeValue tcpSendBufferSize; + protected final ByteSizeValue tcpReceiveBufferSize; + protected final RecvByteBufAllocator recvByteBufAllocator; + + protected final ByteSizeValue maxCumulationBufferCapacity; + protected final int maxCompositeBufferComponents; + + protected volatile ServerBootstrap serverBootstrap; + + protected volatile BoundTransportAddress boundAddress; + + protected final List serverChannels = new ArrayList<>(); + + // package private for testing + Netty4OpenChannelsHandler serverOpenChannels; + + protected volatile HttpServerAdapter httpServerAdapter; + + private final Netty4CorsConfig corsConfig; + + @Inject + public Netty4HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool) { + super(settings); + this.networkService = networkService; + this.bigArrays = bigArrays; + this.threadPool = threadPool; + + ByteSizeValue maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings); + this.maxChunkSize = SETTING_HTTP_MAX_CHUNK_SIZE.get(settings); + this.maxHeaderSize = SETTING_HTTP_MAX_HEADER_SIZE.get(settings); + this.maxInitialLineLength = SETTING_HTTP_MAX_INITIAL_LINE_LENGTH.get(settings); + this.resetCookies = SETTING_HTTP_RESET_COOKIES.get(settings); + this.maxCumulationBufferCapacity = SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY.get(settings); + this.maxCompositeBufferComponents = SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS.get(settings); + this.workerCount = SETTING_HTTP_WORKER_COUNT.get(settings); + this.blockingServer = SETTING_HTTP_TCP_BLOCKING_SERVER.get(settings); + this.port = SETTING_HTTP_PORT.get(settings); + this.bindHosts = SETTING_HTTP_BIND_HOST.get(settings).toArray(Strings.EMPTY_ARRAY); + this.publishHosts = SETTING_HTTP_PUBLISH_HOST.get(settings).toArray(Strings.EMPTY_ARRAY); + this.tcpNoDelay = SETTING_HTTP_TCP_NO_DELAY.get(settings); + this.tcpKeepAlive = SETTING_HTTP_TCP_KEEP_ALIVE.get(settings); + this.reuseAddress = SETTING_HTTP_TCP_REUSE_ADDRESS.get(settings); + this.tcpSendBufferSize = SETTING_HTTP_TCP_SEND_BUFFER_SIZE.get(settings); + this.tcpReceiveBufferSize = SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE.get(settings); + this.detailedErrorsEnabled = SETTING_HTTP_DETAILED_ERRORS_ENABLED.get(settings); + + // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one + ByteSizeValue receivePredictorMin = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN.get(settings); + ByteSizeValue receivePredictorMax = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX.get(settings); + if (receivePredictorMax.bytes() == receivePredictorMin.bytes()) { + recvByteBufAllocator = new FixedRecvByteBufAllocator(Math.toIntExact(receivePredictorMax.bytes())); + } else { + recvByteBufAllocator = new AdaptiveRecvByteBufAllocator( + Math.toIntExact(receivePredictorMin.bytes()), + Math.toIntExact(receivePredictorMin.bytes()), + Math.toIntExact(receivePredictorMax.bytes())); + } + + this.compression = SETTING_HTTP_COMPRESSION.get(settings); + this.compressionLevel = SETTING_HTTP_COMPRESSION_LEVEL.get(settings); + this.pipelining = SETTING_PIPELINING.get(settings); + this.pipeliningMaxEvents = SETTING_PIPELINING_MAX_EVENTS.get(settings); + this.corsConfig = buildCorsConfig(settings); + + // validate max content length + if (maxContentLength.bytes() > Integer.MAX_VALUE) { + logger.warn("maxContentLength[{}] set to high value, resetting it to [100mb]", maxContentLength); + maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB); + } + this.maxContentLength = maxContentLength; + + logger.debug("using max_chunk_size[{}], max_header_size[{}], max_initial_line_length[{}], max_content_length[{}], " + + "receive_predictor[{}->{}], pipelining[{}], pipelining_max_events[{}]", + maxChunkSize, maxHeaderSize, maxInitialLineLength, this.maxContentLength, + receivePredictorMin, receivePredictorMax, pipelining, pipeliningMaxEvents); + } + + public Settings settings() { + return this.settings; + } + + @Override + public void httpServerAdapter(HttpServerAdapter httpServerAdapter) { + this.httpServerAdapter = httpServerAdapter; + } + + @Override + protected void doStart() { + this.serverOpenChannels = new Netty4OpenChannelsHandler(logger); + + serverBootstrap = new ServerBootstrap(); + if (blockingServer) { + serverBootstrap.group(new OioEventLoopGroup(workerCount, daemonThreadFactory(settings, "http_server_worker"))); + serverBootstrap.channel(OioServerSocketChannel.class); + } else { + serverBootstrap.group(new NioEventLoopGroup(workerCount, daemonThreadFactory(settings, "http_server_worker"))); + serverBootstrap.channel(NioServerSocketChannel.class); + } + + serverBootstrap.childHandler(configureServerChannelHandler()); + + serverBootstrap.childOption(ChannelOption.TCP_NODELAY, SETTING_HTTP_TCP_NO_DELAY.get(settings)); + serverBootstrap.childOption(ChannelOption.SO_KEEPALIVE, SETTING_HTTP_TCP_KEEP_ALIVE.get(settings)); + + final ByteSizeValue tcpSendBufferSize = SETTING_HTTP_TCP_SEND_BUFFER_SIZE.get(settings); + if (tcpSendBufferSize.bytes() > 0) { + serverBootstrap.childOption(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.bytes())); + } + + final ByteSizeValue tcpReceiveBufferSize = SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE.get(settings); + if (tcpReceiveBufferSize.bytes() > 0) { + serverBootstrap.childOption(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.bytes())); + } + + serverBootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator); + serverBootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator); + + final boolean reuseAddress = SETTING_HTTP_TCP_REUSE_ADDRESS.get(settings); + serverBootstrap.option(ChannelOption.SO_REUSEADDR, reuseAddress); + serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, reuseAddress); + + this.boundAddress = createBoundHttpAddress(); + } + + private BoundTransportAddress createBoundHttpAddress() { + // Bind and start to accept incoming connections. + InetAddress hostAddresses[]; + try { + hostAddresses = networkService.resolveBindHostAddresses(bindHosts); + } catch (IOException e) { + throw new BindHttpException("Failed to resolve host [" + Arrays.toString(bindHosts) + "]", e); + } + + List boundAddresses = new ArrayList<>(hostAddresses.length); + for (InetAddress address : hostAddresses) { + boundAddresses.add(bindAddress(address)); + } + + final InetAddress publishInetAddress; + try { + publishInetAddress = networkService.resolvePublishHostAddresses(publishHosts); + } catch (Exception e) { + throw new BindTransportException("Failed to resolve publish address", e); + } + + final int publishPort = resolvePublishPort(settings, boundAddresses, publishInetAddress); + final InetSocketAddress publishAddress = new InetSocketAddress(publishInetAddress, publishPort); + return new BoundTransportAddress(boundAddresses.toArray(new TransportAddress[0]), new InetSocketTransportAddress(publishAddress)); + } + + // package private for tests + static int resolvePublishPort(Settings settings, List boundAddresses, InetAddress publishInetAddress) { + int publishPort = SETTING_HTTP_PUBLISH_PORT.get(settings); + + if (publishPort < 0) { + for (InetSocketTransportAddress boundAddress : boundAddresses) { + InetAddress boundInetAddress = boundAddress.address().getAddress(); + if (boundInetAddress.isAnyLocalAddress() || boundInetAddress.equals(publishInetAddress)) { + publishPort = boundAddress.getPort(); + break; + } + } + } + + // if no matching boundAddress found, check if there is a unique port for all bound addresses + if (publishPort < 0) { + final IntSet ports = new IntHashSet(); + for (InetSocketTransportAddress boundAddress : boundAddresses) { + ports.add(boundAddress.getPort()); + } + if (ports.size() == 1) { + publishPort = ports.iterator().next().value; + } + } + + if (publishPort < 0) { + throw new BindHttpException("Failed to auto-resolve http publish port, multiple bound addresses " + boundAddresses + + " with distinct ports and none of them matched the publish address (" + publishInetAddress + "). " + + "Please specify a unique port by setting " + SETTING_HTTP_PORT.getKey() + " or " + SETTING_HTTP_PUBLISH_PORT.getKey()); + } + return publishPort; + } + + private Netty4CorsConfig buildCorsConfig(Settings settings) { + if (SETTING_CORS_ENABLED.get(settings) == false) { + return Netty4CorsConfigBuilder.forOrigins().disable().build(); + } + String origin = SETTING_CORS_ALLOW_ORIGIN.get(settings); + final Netty4CorsConfigBuilder builder; + if (Strings.isNullOrEmpty(origin)) { + builder = Netty4CorsConfigBuilder.forOrigins(); + } else if (origin.equals(ANY_ORIGIN)) { + builder = Netty4CorsConfigBuilder.forAnyOrigin(); + } else { + Pattern p = RestUtils.checkCorsSettingForRegex(origin); + if (p == null) { + builder = Netty4CorsConfigBuilder.forOrigins(RestUtils.corsSettingAsArray(origin)); + } else { + builder = Netty4CorsConfigBuilder.forPattern(p); + } + } + if (SETTING_CORS_ALLOW_CREDENTIALS.get(settings)) { + builder.allowCredentials(); + } + String[] strMethods = settings.getAsArray(SETTING_CORS_ALLOW_METHODS.getKey()); + HttpMethod[] methods = Arrays.asList(strMethods) + .stream() + .map(HttpMethod::valueOf) + .toArray(size -> new HttpMethod[size]); + return builder.allowedRequestMethods(methods) + .maxAge(SETTING_CORS_MAX_AGE.get(settings)) + .allowedRequestHeaders(settings.getAsArray(SETTING_CORS_ALLOW_HEADERS.getKey())) + .shortCircuit() + .build(); + } + + private InetSocketTransportAddress bindAddress(final InetAddress hostAddress) { + final AtomicReference lastException = new AtomicReference<>(); + final AtomicReference boundSocket = new AtomicReference<>(); + boolean success = port.iterate(new PortsRange.PortCallback() { + @Override + public boolean onPortNumber(int portNumber) { + try { + synchronized (serverChannels) { + ChannelFuture future = serverBootstrap.bind(new InetSocketAddress(hostAddress, portNumber)).sync(); + serverChannels.add(future.channel()); + boundSocket.set((InetSocketAddress) future.channel().localAddress()); + } + } catch (Exception e) { + lastException.set(e); + return false; + } + return true; + } + }); + if (!success) { + throw new BindHttpException("Failed to bind to [" + port + "]", lastException.get()); + } + + if (logger.isDebugEnabled()) { + logger.debug("Bound http to address {{}}", NetworkAddress.format(boundSocket.get())); + } + return new InetSocketTransportAddress(boundSocket.get()); + } + + @Override + protected void doStop() { + synchronized (serverChannels) { + if (!serverChannels.isEmpty()) { + try { + Netty4Utils.closeChannels(serverChannels); + } catch (IOException e) { + logger.trace("exception while closing channels", e); + } + serverChannels.clear(); + } + } + + if (serverOpenChannels != null) { + serverOpenChannels.close(); + serverOpenChannels = null; + } + + if (serverBootstrap != null) { + serverBootstrap.config().group().shutdownGracefully(0, 5, TimeUnit.SECONDS).awaitUninterruptibly(); + serverBootstrap = null; + } + } + + @Override + protected void doClose() { + } + + @Override + public BoundTransportAddress boundAddress() { + return this.boundAddress; + } + + @Override + public HttpInfo info() { + BoundTransportAddress boundTransportAddress = boundAddress(); + if (boundTransportAddress == null) { + return null; + } + return new HttpInfo(boundTransportAddress, maxContentLength.bytes()); + } + + @Override + public HttpStats stats() { + Netty4OpenChannelsHandler channels = serverOpenChannels; + return new HttpStats(channels == null ? 0 : channels.numberOfOpenChannels(), channels == null ? 0 : channels.totalChannels()); + } + + public Netty4CorsConfig getCorsConfig() { + return corsConfig; + } + + protected void dispatchRequest(RestRequest request, RestChannel channel) { + httpServerAdapter.dispatchRequest(request, channel, threadPool.getThreadContext()); + } + + protected void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + if (cause instanceof ReadTimeoutException) { + if (logger.isTraceEnabled()) { + logger.trace("Connection timeout [{}]", ctx.channel().remoteAddress()); + } + ctx.channel().close(); + } else { + if (!lifecycle.started()) { + // ignore + return; + } + if (!NetworkExceptionHelper.isCloseConnectionException(cause)) { + logger.warn("caught exception while handling client http traffic, closing connection {}", cause, ctx.channel()); + ctx.channel().close(); + } else { + logger.debug("caught exception while handling client http traffic, closing connection {}", cause, ctx.channel()); + ctx.channel().close(); + } + } + } + + public ChannelHandler configureServerChannelHandler() { + return new HttpChannelHandler(this, detailedErrorsEnabled, threadPool.getThreadContext()); + } + + static class HttpChannelHandler extends ChannelInitializer { + + private final Netty4HttpServerTransport transport; + private final Netty4HttpRequestHandler requestHandler; + + HttpChannelHandler( + final Netty4HttpServerTransport transport, + final boolean detailedErrorsEnabled, + final ThreadContext threadContext) { + this.transport = transport; + this.requestHandler = new Netty4HttpRequestHandler(transport, detailedErrorsEnabled, threadContext); + } + + @Override + protected void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast("openChannels", transport.serverOpenChannels); + final HttpRequestDecoder decoder = new HttpRequestDecoder( + Math.toIntExact(transport.maxInitialLineLength.bytes()), + Math.toIntExact(transport.maxHeaderSize.bytes()), + Math.toIntExact(transport.maxChunkSize.bytes())); + decoder.setCumulator(ByteToMessageDecoder.COMPOSITE_CUMULATOR); + ch.pipeline().addLast("decoder", decoder); + ch.pipeline().addLast("decoder_compress", new HttpContentDecompressor()); + final HttpObjectAggregator aggregator = new HttpObjectAggregator(Math.toIntExact(transport.maxContentLength.bytes())); + if (transport.maxCompositeBufferComponents != -1) { + aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); + } + ch.pipeline().addLast("aggregator", aggregator); + ch.pipeline().addLast("encoder", new HttpResponseEncoder()); + if (transport.compression) { + ch.pipeline().addLast("encoder_compress", new HttpContentCompressor(transport.compressionLevel)); + } + if (SETTING_CORS_ENABLED.get(transport.settings())) { + ch.pipeline().addLast("cors", new Netty4CorsHandler(transport.getCorsConfig())); + } + if (transport.pipelining) { + ch.pipeline().addLast("pipelining", new HttpPipeliningHandler(transport.pipeliningMaxEvents)); + } + ch.pipeline().addLast("handler", requestHandler); + } + + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfig.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfig.java new file mode 100644 index 00000000000..9c81c07e663 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfig.java @@ -0,0 +1,235 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4.cors; + +import io.netty.handler.codec.http.DefaultHttpHeaders; +import io.netty.handler.codec.http.EmptyHttpHeaders; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMethod; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.regex.Pattern; + +/** + * Configuration for Cross-Origin Resource Sharing (CORS). + * + * This class was lifted from the Netty project: + * https://github.com/netty/netty + */ +public final class Netty4CorsConfig { + + private final Optional> origins; + private final Optional pattern; + private final boolean anyOrigin; + private final boolean enabled; + private final boolean allowCredentials; + private final long maxAge; + private final Set allowedRequestMethods; + private final Set allowedRequestHeaders; + private final boolean allowNullOrigin; + private final Map> preflightHeaders; + private final boolean shortCircuit; + + Netty4CorsConfig(final Netty4CorsConfigBuilder builder) { + origins = builder.origins.map(s -> new LinkedHashSet<>(s)); + pattern = builder.pattern; + anyOrigin = builder.anyOrigin; + enabled = builder.enabled; + allowCredentials = builder.allowCredentials; + maxAge = builder.maxAge; + allowedRequestMethods = builder.requestMethods; + allowedRequestHeaders = builder.requestHeaders; + allowNullOrigin = builder.allowNullOrigin; + preflightHeaders = builder.preflightHeaders; + shortCircuit = builder.shortCircuit; + } + + /** + * Determines if support for CORS is enabled. + * + * @return {@code true} if support for CORS is enabled, false otherwise. + */ + public boolean isCorsSupportEnabled() { + return enabled; + } + + /** + * Determines whether a wildcard origin, '*', is supported. + * + * @return {@code boolean} true if any origin is allowed. + */ + public boolean isAnyOriginSupported() { + return anyOrigin; + } + + /** + * Returns the set of allowed origins. + * + * @return {@code Set} the allowed origins. + */ + public Optional> origins() { + return origins; + } + + /** + * Returns whether the input origin is allowed by this configuration. + * + * @return {@code true} if the origin is allowed, otherwise {@code false} + */ + public boolean isOriginAllowed(final String origin) { + if (origins.isPresent()) { + return origins.get().contains(origin); + } else if (pattern.isPresent()) { + return pattern.get().matcher(origin).matches(); + } + return false; + } + + /** + * Web browsers may set the 'Origin' request header to 'null' if a resource is loaded + * from the local file system. + * + * If isNullOriginAllowed is true then the server will response with the wildcard for the + * the CORS response header 'Access-Control-Allow-Origin'. + * + * @return {@code true} if a 'null' origin should be supported. + */ + public boolean isNullOriginAllowed() { + return allowNullOrigin; + } + + /** + * Determines if cookies are supported for CORS requests. + * + * By default cookies are not included in CORS requests but if isCredentialsAllowed returns + * true cookies will be added to CORS requests. Setting this value to true will set the + * CORS 'Access-Control-Allow-Credentials' response header to true. + * + * Please note that cookie support needs to be enabled on the client side as well. + * The client needs to opt-in to send cookies by calling: + * 
+     * xhr.withCredentials = true;
+     *
+ * The default value for 'withCredentials' is false in which case no cookies are sent. + * Setting this to true will included cookies in cross origin requests. + * + * @return {@code true} if cookies are supported. + */ + public boolean isCredentialsAllowed() { + return allowCredentials; + } + + /** + * Gets the maxAge setting. + * + * When making a preflight request the client has to perform two request with can be inefficient. + * This setting will set the CORS 'Access-Control-Max-Age' response header and enables the + * caching of the preflight response for the specified time. During this time no preflight + * request will be made. + * + * @return {@code long} the time in seconds that a preflight request may be cached. + */ + public long maxAge() { + return maxAge; + } + + /** + * Returns the allowed set of Request Methods. The Http methods that should be returned in the + * CORS 'Access-Control-Request-Method' response header. + * + * @return {@code Set} of {@link HttpMethod}s that represent the allowed Request Methods. + */ + public Set allowedRequestMethods() { + return Collections.unmodifiableSet(allowedRequestMethods); + } + + /** + * Returns the allowed set of Request Headers. + * + * The header names returned from this method will be used to set the CORS + * 'Access-Control-Allow-Headers' response header. + * + * @return {@code Set} of strings that represent the allowed Request Headers. + */ + public Set allowedRequestHeaders() { + return Collections.unmodifiableSet(allowedRequestHeaders); + } + + /** + * Returns HTTP response headers that should be added to a CORS preflight response. + * + * @return {@link HttpHeaders} the HTTP response headers to be added. + */ + public HttpHeaders preflightResponseHeaders() { + if (preflightHeaders.isEmpty()) { + return EmptyHttpHeaders.INSTANCE; + } + final HttpHeaders preflightHeaders = new DefaultHttpHeaders(); + for (Map.Entry> entry : this.preflightHeaders.entrySet()) { + final Object value = getValue(entry.getValue()); + if (value instanceof Iterable) { + preflightHeaders.add(entry.getKey().toString(), (Iterable) value); + } else { + preflightHeaders.add(entry.getKey().toString(), value); + } + } + return preflightHeaders; + } + + /** + * Determines whether a CORS request should be rejected if it's invalid before being + * further processing. + * + * CORS headers are set after a request is processed. This may not always be desired + * and this setting will check that the Origin is valid and if it is not valid no + * further processing will take place, and a error will be returned to the calling client. + * + * @return {@code true} if a CORS request should short-circuit upon receiving an invalid Origin header. + */ + public boolean isShortCircuit() { + return shortCircuit; + } + + private static T getValue(final Callable callable) { + try { + return callable.call(); + } catch (final Exception e) { + throw new IllegalStateException("Could not generate value for callable [" + callable + ']', e); + } + } + + @Override + public String toString() { + return "CorsConfig[enabled=" + enabled + + ", origins=" + origins + + ", anyOrigin=" + anyOrigin + + ", isCredentialsAllowed=" + allowCredentials + + ", maxAge=" + maxAge + + ", allowedRequestMethods=" + allowedRequestMethods + + ", allowedRequestHeaders=" + allowedRequestHeaders + + ", preflightHeaders=" + preflightHeaders + ']'; + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfigBuilder.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfigBuilder.java new file mode 100644 index 00000000000..4989cd35f7b --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfigBuilder.java @@ -0,0 +1,357 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4.cors; + +import io.netty.handler.codec.http.HttpMethod; + +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.regex.Pattern; + +/** + * Builder used to configure and build a {@link Netty4CorsConfig} instance. + * + * This class was lifted from the Netty project: + * https://github.com/netty/netty + */ +public final class Netty4CorsConfigBuilder { + + /** + * Creates a Builder instance with it's origin set to '*'. + * + * @return Builder to support method chaining. + */ + public static Netty4CorsConfigBuilder forAnyOrigin() { + return new Netty4CorsConfigBuilder(); + } + + /** + * Creates a {@link Netty4CorsConfigBuilder} instance with the specified origin. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public static Netty4CorsConfigBuilder forOrigin(final String origin) { + if ("*".equals(origin)) { + return new Netty4CorsConfigBuilder(); + } + return new Netty4CorsConfigBuilder(origin); + } + + + /** + * Create a {@link Netty4CorsConfigBuilder} instance with the specified pattern origin. + * + * @param pattern the regular expression pattern to match incoming origins on. + * @return {@link Netty4CorsConfigBuilder} with the configured origin pattern. + */ + public static Netty4CorsConfigBuilder forPattern(final Pattern pattern) { + if (pattern == null) { + throw new IllegalArgumentException("CORS pattern cannot be null"); + } + return new Netty4CorsConfigBuilder(pattern); + } + + /** + * Creates a {@link Netty4CorsConfigBuilder} instance with the specified origins. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public static Netty4CorsConfigBuilder forOrigins(final String... origins) { + return new Netty4CorsConfigBuilder(origins); + } + + Optional> origins; + Optional pattern; + final boolean anyOrigin; + boolean allowNullOrigin; + boolean enabled = true; + boolean allowCredentials; + long maxAge; + final Set requestMethods = new HashSet<>(); + final Set requestHeaders = new HashSet<>(); + final Map> preflightHeaders = new HashMap<>(); + private boolean noPreflightHeaders; + boolean shortCircuit; + + /** + * Creates a new Builder instance with the origin passed in. + * + * @param origins the origin to be used for this builder. + */ + Netty4CorsConfigBuilder(final String... origins) { + this.origins = Optional.of(new LinkedHashSet<>(Arrays.asList(origins))); + pattern = Optional.empty(); + anyOrigin = false; + } + + /** + * Creates a new Builder instance allowing any origin, "*" which is the + * wildcard origin. + * + */ + Netty4CorsConfigBuilder() { + anyOrigin = true; + origins = Optional.empty(); + pattern = Optional.empty(); + } + + /** + * Creates a new Builder instance allowing any origin that matches the pattern. + * + * @param pattern the pattern to match against for incoming origins. + */ + Netty4CorsConfigBuilder(final Pattern pattern) { + this.pattern = Optional.of(pattern); + origins = Optional.empty(); + anyOrigin = false; + } + + /** + * Web browsers may set the 'Origin' request header to 'null' if a resource is loaded + * from the local file system. Calling this method will enable a successful CORS response + * with a wildcard for the CORS response header 'Access-Control-Allow-Origin'. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + Netty4CorsConfigBuilder allowNullOrigin() { + allowNullOrigin = true; + return this; + } + + /** + * Disables CORS support. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder disable() { + enabled = false; + return this; + } + + /** + * By default cookies are not included in CORS requests, but this method will enable cookies to + * be added to CORS requests. Calling this method will set the CORS 'Access-Control-Allow-Credentials' + * response header to true. + * + * Please note, that cookie support needs to be enabled on the client side as well. + * The client needs to opt-in to send cookies by calling: + * 
+     * xhr.withCredentials = true;
+     *
+ * The default value for 'withCredentials' is false in which case no cookies are sent. + * Setting this to true will included cookies in cross origin requests. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder allowCredentials() { + allowCredentials = true; + return this; + } + + /** + * When making a preflight request the client has to perform two request with can be inefficient. + * This setting will set the CORS 'Access-Control-Max-Age' response header and enables the + * caching of the preflight response for the specified time. During this time no preflight + * request will be made. + * + * @param max the maximum time, in seconds, that the preflight response may be cached. + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder maxAge(final long max) { + maxAge = max; + return this; + } + + /** + * Specifies the allowed set of HTTP Request Methods that should be returned in the + * CORS 'Access-Control-Request-Method' response header. + * + * @param methods the {@link HttpMethod}s that should be allowed. + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder allowedRequestMethods(final HttpMethod... methods) { + requestMethods.addAll(Arrays.asList(methods)); + return this; + } + + /** + * Specifies the if headers that should be returned in the CORS 'Access-Control-Allow-Headers' + * response header. + * + * If a client specifies headers on the request, for example by calling: + * 
+     * xhr.setRequestHeader('My-Custom-Header', "SomeValue");
+     *
+ * the server will receive the above header name in the 'Access-Control-Request-Headers' of the + * preflight request. The server will then decide if it allows this header to be sent for the + * real request (remember that a preflight is not the real request but a request asking the server + * if it allow a request). + * + * @param headers the headers to be added to the preflight 'Access-Control-Allow-Headers' response header. + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder allowedRequestHeaders(final String... headers) { + requestHeaders.addAll(Arrays.asList(headers)); + return this; + } + + /** + * Returns HTTP response headers that should be added to a CORS preflight response. + * + * An intermediary like a load balancer might require that a CORS preflight request + * have certain headers set. This enables such headers to be added. + * + * @param name the name of the HTTP header. + * @param values the values for the HTTP header. + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder preflightResponseHeader(final CharSequence name, final Object... values) { + if (values.length == 1) { + preflightHeaders.put(name, new ConstantValueGenerator(values[0])); + } else { + preflightResponseHeader(name, Arrays.asList(values)); + } + return this; + } + + /** + * Returns HTTP response headers that should be added to a CORS preflight response. + * + * An intermediary like a load balancer might require that a CORS preflight request + * have certain headers set. This enables such headers to be added. + * + * @param name the name of the HTTP header. + * @param value the values for the HTTP header. + * @param the type of values that the Iterable contains. + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder preflightResponseHeader(final CharSequence name, final Iterable value) { + preflightHeaders.put(name, new ConstantValueGenerator(value)); + return this; + } + + /** + * Returns HTTP response headers that should be added to a CORS preflight response. + * + * An intermediary like a load balancer might require that a CORS preflight request + * have certain headers set. This enables such headers to be added. + * + * Some values must be dynamically created when the HTTP response is created, for + * example the 'Date' response header. This can be accomplished by using a Callable + * which will have its 'call' method invoked when the HTTP response is created. + * + * @param name the name of the HTTP header. + * @param valueGenerator a Callable which will be invoked at HTTP response creation. + * @param the type of the value that the Callable can return. + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder preflightResponseHeader(final CharSequence name, final Callable valueGenerator) { + preflightHeaders.put(name, valueGenerator); + return this; + } + + /** + * Specifies that no preflight response headers should be added to a preflight response. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder noPreflightResponseHeaders() { + noPreflightHeaders = true; + return this; + } + + /** + * Specifies that a CORS request should be rejected if it's invalid before being + * further processing. + * + * CORS headers are set after a request is processed. This may not always be desired + * and this setting will check that the Origin is valid and if it is not valid no + * further processing will take place, and a error will be returned to the calling client. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder shortCircuit() { + shortCircuit = true; + return this; + } + + /** + * Builds a {@link Netty4CorsConfig} with settings specified by previous method calls. + * + * @return {@link Netty4CorsConfig} the configured CorsConfig instance. + */ + public Netty4CorsConfig build() { + if (preflightHeaders.isEmpty() && !noPreflightHeaders) { + preflightHeaders.put("date", DateValueGenerator.INSTANCE); + preflightHeaders.put("content-length", new ConstantValueGenerator("0")); + } + return new Netty4CorsConfig(this); + } + + /** + * This class is used for preflight HTTP response values that do not need to be + * generated, but instead the value is "static" in that the same value will be returned + * for each call. + */ + private static final class ConstantValueGenerator implements Callable { + + private final Object value; + + /** + * Sole constructor. + * + * @param value the value that will be returned when the call method is invoked. + */ + private ConstantValueGenerator(final Object value) { + if (value == null) { + throw new IllegalArgumentException("value must not be null"); + } + this.value = value; + } + + @Override + public Object call() { + return value; + } + } + + /** + * This callable is used for the DATE preflight HTTP response HTTP header. + * It's value must be generated when the response is generated, hence will be + * different for every call. + */ + private static final class DateValueGenerator implements Callable { + + static final DateValueGenerator INSTANCE = new DateValueGenerator(); + + @Override + public Date call() throws Exception { + return new Date(); + } + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java new file mode 100644 index 00000000000..779eb4fe2e4 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java @@ -0,0 +1,235 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4.cors; + +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import org.elasticsearch.common.Strings; + +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Handles Cross Origin Resource Sharing (CORS) requests. + *

+ * This handler can be configured using a {@link Netty4CorsConfig}, please + * refer to this class for details about the configuration options available. + * + * This code was borrowed from Netty 4 and refactored to work for Elasticsearch's Netty 3 setup. + */ +public class Netty4CorsHandler extends ChannelDuplexHandler { + + public static final String ANY_ORIGIN = "*"; + private static Pattern SCHEME_PATTERN = Pattern.compile("^https?://"); + + private final Netty4CorsConfig config; + private HttpRequest request; + + /** + * Creates a new instance with the specified {@link Netty4CorsConfig}. + */ + public Netty4CorsHandler(final Netty4CorsConfig config) { + if (config == null) { + throw new NullPointerException(); + } + this.config = config; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (config.isCorsSupportEnabled() && msg instanceof HttpRequest) { + request = (HttpRequest) msg; + if (isPreflightRequest(request)) { + handlePreflight(ctx, request); + return; + } + if (config.isShortCircuit() && !validateOrigin()) { + forbidden(ctx, request); + return; + } + } + ctx.fireChannelRead(msg); + } + + public static void setCorsResponseHeaders(HttpRequest request, HttpResponse resp, Netty4CorsConfig config) { + if (!config.isCorsSupportEnabled()) { + return; + } + String originHeader = request.headers().get(HttpHeaderNames.ORIGIN); + if (!Strings.isNullOrEmpty(originHeader)) { + final String originHeaderVal; + if (config.isAnyOriginSupported()) { + originHeaderVal = ANY_ORIGIN; + } else if (config.isOriginAllowed(originHeader) || isSameOrigin(originHeader, request.headers().get(HttpHeaderNames.HOST))) { + originHeaderVal = originHeader; + } else { + originHeaderVal = null; + } + if (originHeaderVal != null) { + resp.headers().add(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN, originHeaderVal); + } + } + if (config.isCredentialsAllowed()) { + resp.headers().add(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS, "true"); + } + } + + private void handlePreflight(final ChannelHandlerContext ctx, final HttpRequest request) { + final HttpResponse response = new DefaultFullHttpResponse(request.protocolVersion(), HttpResponseStatus.OK, true, true); + if (setOrigin(response)) { + setAllowMethods(response); + setAllowHeaders(response); + setAllowCredentials(response); + setMaxAge(response); + setPreflightHeaders(response); + ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE); + } else { + forbidden(ctx, request); + } + } + + private static void forbidden(final ChannelHandlerContext ctx, final HttpRequest request) { + ctx.writeAndFlush(new DefaultFullHttpResponse(request.protocolVersion(), HttpResponseStatus.FORBIDDEN)) + .addListener(ChannelFutureListener.CLOSE); + } + + private static boolean isSameOrigin(final String origin, final String host) { + if (Strings.isNullOrEmpty(host) == false) { + // strip protocol from origin + final String originDomain = SCHEME_PATTERN.matcher(origin).replaceFirst(""); + if (host.equals(originDomain)) { + return true; + } + } + return false; + } + + /** + * This is a non CORS specification feature which enables the setting of preflight + * response headers that might be required by intermediaries. + * + * @param response the HttpResponse to which the preflight response headers should be added. + */ + private void setPreflightHeaders(final HttpResponse response) { + response.headers().add(config.preflightResponseHeaders()); + } + + private boolean setOrigin(final HttpResponse response) { + final String origin = request.headers().get(HttpHeaderNames.ORIGIN); + if (!Strings.isNullOrEmpty(origin)) { + if ("null".equals(origin) && config.isNullOriginAllowed()) { + setAnyOrigin(response); + return true; + } + + if (config.isAnyOriginSupported()) { + if (config.isCredentialsAllowed()) { + echoRequestOrigin(response); + setVaryHeader(response); + } else { + setAnyOrigin(response); + } + return true; + } + if (config.isOriginAllowed(origin)) { + setOrigin(response, origin); + setVaryHeader(response); + return true; + } + } + return false; + } + + private boolean validateOrigin() { + if (config.isAnyOriginSupported()) { + return true; + } + + final String origin = request.headers().get(HttpHeaderNames.ORIGIN); + if (Strings.isNullOrEmpty(origin)) { + // Not a CORS request so we cannot validate it. It may be a non CORS request. + return true; + } + + if ("null".equals(origin) && config.isNullOriginAllowed()) { + return true; + } + + // if the origin is the same as the host of the request, then allow + if (isSameOrigin(origin, request.headers().get(HttpHeaderNames.HOST))) { + return true; + } + + return config.isOriginAllowed(origin); + } + + private void echoRequestOrigin(final HttpResponse response) { + setOrigin(response, request.headers().get(HttpHeaderNames.ORIGIN)); + } + + private static void setVaryHeader(final HttpResponse response) { + response.headers().set(HttpHeaderNames.VARY, HttpHeaderNames.ORIGIN); + } + + private static void setAnyOrigin(final HttpResponse response) { + setOrigin(response, ANY_ORIGIN); + } + + private static void setOrigin(final HttpResponse response, final String origin) { + response.headers().set(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN, origin); + } + + private void setAllowCredentials(final HttpResponse response) { + if (config.isCredentialsAllowed() + && !response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN).equals(ANY_ORIGIN)) { + response.headers().set(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS, "true"); + } + } + + private static boolean isPreflightRequest(final HttpRequest request) { + final HttpHeaders headers = request.headers(); + return request.method().equals(HttpMethod.OPTIONS) && + headers.contains(HttpHeaderNames.ORIGIN) && + headers.contains(HttpHeaderNames.ACCESS_CONTROL_REQUEST_METHOD); + } + + private void setAllowMethods(final HttpResponse response) { + response.headers().set(HttpHeaderNames.ACCESS_CONTROL_ALLOW_METHODS, config.allowedRequestMethods().stream() + .map(m -> m.name().trim()) + .collect(Collectors.toList())); + } + + private void setAllowHeaders(final HttpResponse response) { + response.headers().set(HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS, config.allowedRequestHeaders()); + } + + private void setMaxAge(final HttpResponse response) { + response.headers().set(HttpHeaderNames.ACCESS_CONTROL_MAX_AGE, config.maxAge()); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedRequest.java new file mode 100644 index 00000000000..792397a3c3d --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedRequest.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4.pipelining; + +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.util.ReferenceCounted; + +/** + * Permits downstream channel events to be ordered and signalled as to whether more are to come for + * a given sequence. + */ +public class HttpPipelinedRequest implements ReferenceCounted { + + private final LastHttpContent last; + private final int sequence; + + + HttpPipelinedRequest(final LastHttpContent last, final int sequence) { + this.last = last; + this.sequence = sequence; + } + + public LastHttpContent last() { + return last; + } + + public HttpPipelinedResponse createHttpResponse(final FullHttpResponse response, final ChannelPromise promise) { + return new HttpPipelinedResponse(response, promise, sequence); + } + + @Override + public int refCnt() { + return last.refCnt(); + } + + @Override + public ReferenceCounted retain() { + last.retain(); + return this; + } + + @Override + public ReferenceCounted retain(int increment) { + last.retain(increment); + return this; + } + + @Override + public ReferenceCounted touch() { + last.touch(); + return this; + } + + @Override + public ReferenceCounted touch(Object hint) { + last.touch(hint); + return this; + } + + @Override + public boolean release() { + return last.release(); + } + + @Override + public boolean release(int decrement) { + return last.release(decrement); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedResponse.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedResponse.java new file mode 100644 index 00000000000..21659d5fbdf --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedResponse.java @@ -0,0 +1,95 @@ +package org.elasticsearch.http.netty4.pipelining; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.util.ReferenceCounted; + +class HttpPipelinedResponse implements Comparable, ReferenceCounted { + + private final FullHttpResponse response; + private final ChannelPromise promise; + private final int sequence; + + HttpPipelinedResponse(FullHttpResponse response, ChannelPromise promise, int sequence) { + this.response = response; + this.promise = promise; + this.sequence = sequence; + } + + public FullHttpResponse response() { + return response; + } + + public ChannelPromise promise() { + return promise; + } + + public int sequence() { + return sequence; + } + + @Override + public int compareTo(HttpPipelinedResponse o) { + return Integer.compare(sequence, o.sequence); + } + + @Override + public int refCnt() { + return response.refCnt(); + } + + @Override + public ReferenceCounted retain() { + response.retain(); + return this; + } + + @Override + public ReferenceCounted retain(int increment) { + response.retain(increment); + return this; + } + + @Override + public ReferenceCounted touch() { + response.touch(); + return this; + } + + @Override + public ReferenceCounted touch(Object hint) { + response.touch(hint); + return this; + } + + @Override + public boolean release() { + return response.release(); + } + + @Override + public boolean release(int decrement) { + return response.release(decrement); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipeliningHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipeliningHandler.java new file mode 100644 index 00000000000..b96b7f5b322 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipeliningHandler.java @@ -0,0 +1,108 @@ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4.pipelining; + +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.util.ReferenceCountUtil; +import org.elasticsearch.action.termvectors.TermVectorsFilter; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.transport.netty4.Netty4Utils; + +import java.util.Collections; +import java.util.PriorityQueue; +import java.util.Queue; + +/** + * Implements HTTP pipelining ordering, ensuring that responses are completely served in the same order as their + * corresponding requests. NOTE: A side effect of using this handler is that upstream HttpRequest objects will + * cause the original message event to be effectively transformed into an OrderedUpstreamMessageEvent. Conversely + * OrderedDownstreamChannelEvent objects are expected to be received for the correlating response objects. + */ +public class HttpPipeliningHandler extends ChannelDuplexHandler { + + private static final int INITIAL_EVENTS_HELD = 3; + + private final int maxEventsHeld; + + private int readSequence; + private int writeSequence; + + private final Queue holdingQueue; + + /** + * @param maxEventsHeld the maximum number of channel events that will be retained prior to aborting the channel + * connection. This is required as events cannot queue up indefinitely; we would run out of + * memory if this was the case. + */ + public HttpPipeliningHandler(final int maxEventsHeld) { + this.maxEventsHeld = maxEventsHeld; + this.holdingQueue = new PriorityQueue<>(INITIAL_EVENTS_HELD); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (msg instanceof LastHttpContent) { + ctx.fireChannelRead(new HttpPipelinedRequest(((LastHttpContent) msg).retain(), readSequence++)); + } else { + ctx.fireChannelRead(msg); + } + } + + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { + if (msg instanceof HttpPipelinedResponse) { + boolean channelShouldClose = false; + + synchronized (holdingQueue) { + if (holdingQueue.size() < maxEventsHeld) { + holdingQueue.add((HttpPipelinedResponse) msg); + + while (!holdingQueue.isEmpty()) { + final HttpPipelinedResponse response = holdingQueue.peek(); + if (response.sequence() != writeSequence) { + break; + } + holdingQueue.remove(); + ctx.write(response.response(), response.promise()); + writeSequence++; + } + } else { + channelShouldClose = true; + } + } + + if (channelShouldClose) { + try { + Netty4Utils.closeChannels(Collections.singletonList(ctx.channel())); + } finally { + ((HttpPipelinedResponse) msg).release(); + promise.setSuccess(); + } + } + } else { + ctx.write(msg, promise); + } + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java new file mode 100644 index 00000000000..1ec2852fd1d --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java @@ -0,0 +1,94 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport; + +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.http.netty4.Netty4HttpServerTransport; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.transport.netty4.Netty4Transport; + +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Arrays; +import java.util.List; + +public class Netty4Plugin extends Plugin { + + public static final String NETTY_TRANSPORT_NAME = "netty4"; + public static final String NETTY_HTTP_TRANSPORT_NAME = "netty4"; + + public Netty4Plugin(Settings settings) { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + AccessController.doPrivileged((PrivilegedAction) () -> { + try { + Class.forName("io.netty.channel.nio.NioEventLoop"); + } catch (ClassNotFoundException e) { + throw new AssertionError(e); // we don't do anything with this + } + return null; + }); + /* + * Asserts that sun.nio.ch.bugLevel has been set to a non-null value. This assertion will fail if the corresponding code + * is not executed in a doPrivileged block. This can be disabled via `netty.assert.buglevel` setting which isn't registered + * by default but test can do so if they depend on the jar instead of the module. + */ + //TODO Once we have no jar level dependency we can get rid of this. + if (settings.getAsBoolean("netty.assert.buglevel", true)) { + assert System.getProperty("sun.nio.ch.bugLevel") != null : + "sun.nio.ch.bugLevel is null somebody pulls in SelectorUtil without doing stuff in a doPrivileged block?"; + } + } + + @Override + public List> getSettings() { + return Arrays.asList( + Netty4HttpServerTransport.SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY, + Netty4HttpServerTransport.SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS, + Netty4HttpServerTransport.SETTING_HTTP_WORKER_COUNT, + Netty4HttpServerTransport.SETTING_HTTP_TCP_NO_DELAY, + Netty4HttpServerTransport.SETTING_HTTP_TCP_KEEP_ALIVE, + Netty4HttpServerTransport.SETTING_HTTP_TCP_BLOCKING_SERVER, + Netty4HttpServerTransport.SETTING_HTTP_TCP_REUSE_ADDRESS, + Netty4HttpServerTransport.SETTING_HTTP_TCP_SEND_BUFFER_SIZE, + Netty4HttpServerTransport.SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE, + Netty4Transport.WORKER_COUNT, + Netty4Transport.NETTY_MAX_CUMULATION_BUFFER_CAPACITY, + Netty4Transport.NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS, + Netty4Transport.NETTY_RECEIVE_PREDICTOR_SIZE, + Netty4Transport.NETTY_RECEIVE_PREDICTOR_MIN, + Netty4Transport.NETTY_RECEIVE_PREDICTOR_MAX, + Netty4Transport.NETTY_BOSS_COUNT + ); + } + + public void onModule(NetworkModule networkModule) { + if (networkModule.canRegisterHttpExtensions()) { + networkModule.registerHttpTransport(NETTY_HTTP_TRANSPORT_NAME, Netty4HttpServerTransport.class); + } + networkModule.registerTransport(NETTY_TRANSPORT_NAME, Netty4Transport.class); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufBytesReference.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufBytesReference.java new file mode 100644 index 00000000000..3b4aba10281 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufBytesReference.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +final class ByteBufBytesReference extends BytesReference { + + private final ByteBuf buffer; + private final int length; + private final int offset; + + ByteBufBytesReference(ByteBuf buffer, int length) { + this.buffer = buffer; + this.length = length; + this.offset = buffer.readerIndex(); + assert length <= buffer.readableBytes() : "length[" + length +"] > " + buffer.readableBytes(); + } + + @Override + public byte get(int index) { + return buffer.getByte(offset + index); + } + + @Override + public int length() { + return length; + } + + @Override + public BytesReference slice(int from, int length) { + return new ByteBufBytesReference(buffer.slice(offset + from, length), length); + } + + @Override + public StreamInput streamInput() { + return new ByteBufStreamInput(buffer.duplicate(), length); + } + + @Override + public void writeTo(OutputStream os) throws IOException { + buffer.getBytes(offset, os, length); + } + + ByteBuf toByteBuf() { + return buffer.duplicate(); + } + + @Override + public String utf8ToString() { + return buffer.toString(offset, length, StandardCharsets.UTF_8); + } + + @Override + public BytesRef toBytesRef() { + if (buffer.hasArray()) { + return new BytesRef(buffer.array(), buffer.arrayOffset() + offset, length); + } + final byte[] copy = new byte[length]; + buffer.getBytes(offset, copy); + return new BytesRef(copy); + } + + @Override + public long ramBytesUsed() { + return buffer.capacity(); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java new file mode 100644 index 00000000000..2219ce31ff6 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; + +/** + * A Netty {@link io.netty.buffer.ByteBuf} based {@link org.elasticsearch.common.io.stream.StreamInput}. + */ +class ByteBufStreamInput extends StreamInput { + + private final ByteBuf buffer; + private final int startIndex; + private final int endIndex; + + public ByteBufStreamInput(ByteBuf buffer, int length) { + if (length > buffer.readableBytes()) { + throw new IndexOutOfBoundsException(); + } + this.buffer = buffer; + startIndex = buffer.readerIndex(); + endIndex = startIndex + length; + buffer.markReaderIndex(); + } + + @Override + public BytesReference readBytesReference(int length) throws IOException { + BytesReference ref = Netty4Utils.toBytesReference(buffer.slice(buffer.readerIndex(), length)); + buffer.skipBytes(length); + return ref; + } + + @Override + public BytesRef readBytesRef(int length) throws IOException { + if (!buffer.hasArray()) { + return super.readBytesRef(length); + } + BytesRef bytesRef = new BytesRef(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), length); + buffer.skipBytes(length); + return bytesRef; + } + + @Override + public int available() throws IOException { + return endIndex - buffer.readerIndex(); + } + + @Override + public void mark(int readlimit) { + buffer.markReaderIndex(); + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public int read() throws IOException { + if (available() == 0) { + return -1; + } + return buffer.readByte() & 0xff; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (len == 0) { + return 0; + } + int available = available(); + if (available == 0) { + return -1; + } + + len = Math.min(available, len); + buffer.readBytes(b, off, len); + return len; + } + + @Override + public void reset() throws IOException { + buffer.resetReaderIndex(); + } + + @Override + public long skip(long n) throws IOException { + if (n > Integer.MAX_VALUE) { + return skipBytes(Integer.MAX_VALUE); + } else { + return skipBytes((int) n); + } + } + + public int skipBytes(int n) throws IOException { + int nBytes = Math.min(available(), n); + buffer.skipBytes(nBytes); + return nBytes; + } + + + @Override + public byte readByte() throws IOException { + return buffer.readByte(); + } + + @Override + public void readBytes(byte[] b, int offset, int len) throws IOException { + int read = read(b, offset, len); + if (read < len) { + throw new IndexOutOfBoundsException(); + } + } + + @Override + public void close() throws IOException { + // nothing to do here + } +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java new file mode 100644 index 00000000000..61555294018 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java @@ -0,0 +1,187 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.util.internal.logging.AbstractInternalLogger; +import org.elasticsearch.common.SuppressLoggerChecks; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; + +@SuppressLoggerChecks(reason = "safely delegates to logger") +class Netty4InternalESLogger extends AbstractInternalLogger { + + private final ESLogger logger; + + Netty4InternalESLogger(final String name) { + super(name); + this.logger = Loggers.getLogger(name); + } + + @Override + public boolean isTraceEnabled() { + return logger.isTraceEnabled(); + } + + @Override + public void trace(String msg) { + logger.trace(msg); + } + + @Override + public void trace(String format, Object arg) { + logger.trace(format, arg); + } + + @Override + public void trace(String format, Object argA, Object argB) { + logger.trace(format, argA, argB); + } + + @Override + public void trace(String format, Object... arguments) { + logger.trace(format, arguments); + } + + @Override + public void trace(String msg, Throwable t) { + logger.trace(msg, t); + } + + @Override + public boolean isDebugEnabled() { + return logger.isDebugEnabled(); + } + + @Override + public void debug(String msg) { + logger.debug(msg); + } + + @Override + public void debug(String format, Object arg) { + logger.debug(format, arg); + } + + @Override + public void debug(String format, Object argA, Object argB) { + logger.debug(format, argA, argB); + } + + @Override + public void debug(String format, Object... arguments) { + logger.debug(format, arguments); + } + + @Override + public void debug(String msg, Throwable t) { + logger.debug(msg, t); + } + + @Override + public boolean isInfoEnabled() { + return logger.isInfoEnabled(); + } + + @Override + public void info(String msg) { + logger.info(msg); + } + + @Override + public void info(String format, Object arg) { + logger.info(format, arg); + } + + @Override + public void info(String format, Object argA, Object argB) { + logger.info(format, argA, argB); + } + + @Override + public void info(String format, Object... arguments) { + logger.info(format, arguments); + } + + @Override + public void info(String msg, Throwable t) { + logger.info(msg, t); + } + + @Override + public boolean isWarnEnabled() { + return logger.isWarnEnabled(); + } + + @Override + public void warn(String msg) { + logger.warn(msg); + } + + @Override + public void warn(String format, Object arg) { + logger.warn(format, arg); + } + + @Override + public void warn(String format, Object... arguments) { + logger.warn(format, arguments); + } + + @Override + public void warn(String format, Object argA, Object argB) { + logger.warn(format, argA, argB); + } + + @Override + public void warn(String msg, Throwable t) { + logger.warn(msg, t); + } + + @Override + public boolean isErrorEnabled() { + return logger.isErrorEnabled(); + } + + @Override + public void error(String msg) { + logger.equals(msg); + } + + @Override + public void error(String format, Object arg) { + logger.error(format, arg); + } + + @Override + public void error(String format, Object argA, Object argB) { + logger.error(format, argA, argB); + } + + @Override + public void error(String format, Object... arguments) { + logger.error(format, arguments); + } + + @Override + public void error(String msg, Throwable t) { + logger.error(msg, t); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java new file mode 100644 index 00000000000..cddbdac3e76 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; +import io.netty.util.ReferenceCountUtil; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.transport.TcpHeader; +import org.elasticsearch.transport.TransportServiceAdapter; +import org.elasticsearch.transport.Transports; + +import java.net.InetSocketAddress; + +/** + * A handler (must be the last one!) that does size based frame decoding and forwards the actual message + * to the relevant action. + */ +final class Netty4MessageChannelHandler extends ChannelDuplexHandler { + + private final TransportServiceAdapter transportServiceAdapter; + private final Netty4Transport transport; + private final String profileName; + + Netty4MessageChannelHandler(Netty4Transport transport, String profileName) { + this.transportServiceAdapter = transport.transportServiceAdapter(); + this.transport = transport; + this.profileName = profileName; + } + + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { + if (msg instanceof ByteBuf && transportServiceAdapter != null) { + // record the number of bytes send on the channel + promise.addListener(f -> transportServiceAdapter.addBytesSent(((ByteBuf) msg).readableBytes())); + } + ctx.write(msg, promise); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + Transports.assertTransportThread(); + if (!(msg instanceof ByteBuf)) { + ctx.fireChannelRead(msg); + return; + } + final ByteBuf buffer = (ByteBuf) msg; + final int remainingMessageSize = buffer.getInt(buffer.readerIndex() - TcpHeader.MESSAGE_LENGTH_SIZE); + final int expectedReaderIndex = buffer.readerIndex() + remainingMessageSize; + InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress(); + try { + // netty always copies a buffer, either in NioWorker in its read handler, where it copies to a fresh + // buffer, or in the cumulation buffer, which is cleaned each time so it could be bigger than the actual size + BytesReference reference = Netty4Utils.toBytesReference(buffer, remainingMessageSize); + transport.messageReceived(reference, ctx.channel(), profileName, remoteAddress, remainingMessageSize); + } finally { + // Set the expected position of the buffer, no matter what happened + buffer.readerIndex(expectedReaderIndex); + } + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + transport.exceptionCaught(ctx, cause); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4OpenChannelsHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4OpenChannelsHandler.java new file mode 100644 index 00000000000..0562a0d4661 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4OpenChannelsHandler.java @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.metrics.CounterMetric; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +@ChannelHandler.Sharable +public class Netty4OpenChannelsHandler extends ChannelInboundHandlerAdapter implements Releasable { + + final Set openChannels = Collections.newSetFromMap(new ConcurrentHashMap<>()); + final CounterMetric openChannelsMetric = new CounterMetric(); + final CounterMetric totalChannelsMetric = new CounterMetric(); + + final ESLogger logger; + + public Netty4OpenChannelsHandler(ESLogger logger) { + this.logger = logger; + } + + final ChannelFutureListener remover = new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + boolean removed = openChannels.remove(future.channel()); + if (removed) { + openChannelsMetric.dec(); + } + if (logger.isTraceEnabled()) { + logger.trace("channel closed: {}", future.channel()); + } + } + }; + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + if (logger.isTraceEnabled()) { + logger.trace("channel opened: {}", ctx.channel()); + } + final boolean added = openChannels.add(ctx.channel()); + if (added) { + openChannelsMetric.inc(); + totalChannelsMetric.inc(); + ctx.channel().closeFuture().addListener(remover); + } + + super.channelActive(ctx); + } + + public long numberOfOpenChannels() { + return openChannelsMetric.count(); + } + + public long totalChannels() { + return totalChannelsMetric.count(); + } + + @Override + public void close() { + try { + Netty4Utils.closeChannels(openChannels); + } catch (IOException e) { + logger.trace("exception while closing channels", e); + } + openChannels.clear(); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoder.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoder.java new file mode 100644 index 00000000000..144c036f08a --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoder.java @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.handler.codec.TooLongFrameException; +import org.elasticsearch.transport.TcpHeader; +import org.elasticsearch.transport.TcpTransport; + +import java.util.List; + +final class Netty4SizeHeaderFrameDecoder extends ByteToMessageDecoder { + + @Override + protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { + try { + boolean continueProcessing = TcpTransport.validateMessageHeader(Netty4Utils.toBytesReference(in)); + final ByteBuf message = in.skipBytes(TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE); + if (!continueProcessing) return; + out.add(message); + } catch (IllegalArgumentException ex) { + throw new TooLongFrameException(ex); + } catch (IllegalStateException ex) { + /* decode will be called until the ByteBuf is fully consumed; when it is fully + * consumed, transport#validateMessageHeader will throw an IllegalStateException which + * is okay, it means we have finished consuming the ByteBuf and we can get out + */ + } + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java new file mode 100644 index 00000000000..c1b2ef10211 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java @@ -0,0 +1,506 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.bootstrap.Bootstrap; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.AdaptiveRecvByteBufAllocator; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.FixedRecvByteBufAllocator; +import io.netty.channel.RecvByteBufAllocator; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.oio.OioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.channel.socket.oio.OioServerSocketChannel; +import io.netty.channel.socket.oio.OioSocketChannel; +import io.netty.util.concurrent.Future; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.network.NetworkService.TcpSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.FutureUtils; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.ConnectTransportException; +import org.elasticsearch.transport.TcpTransport; +import org.elasticsearch.transport.TransportServiceAdapter; +import org.elasticsearch.transport.TransportSettings; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.common.settings.Setting.byteSizeSetting; +import static org.elasticsearch.common.settings.Setting.intSetting; +import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap; +import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; + +/** + * There are 4 types of connections per node, low/med/high/ping. Low if for batch oriented APIs (like recovery or + * batch) with high payload that will cause regular request. (like search or single index) to take + * longer. Med is for the typical search / single doc index. And High for things like cluster state. Ping is reserved for + * sending out ping requests to other nodes. + */ +public class Netty4Transport extends TcpTransport { + + static { + Netty4Utils.setup(); + } + + public static final Setting WORKER_COUNT = + new Setting<>("transport.netty.worker_count", + (s) -> Integer.toString(EsExecutors.boundedNumberOfProcessors(s) * 2), + (s) -> Setting.parseInt(s, 1, "transport.netty.worker_count"), Property.NodeScope, Property.Shared); + + public static final Setting NETTY_MAX_CUMULATION_BUFFER_CAPACITY = + Setting.byteSizeSetting( + "transport.netty.max_cumulation_buffer_capacity", + new ByteSizeValue(-1), + Property.NodeScope, + Property.Shared); + public static final Setting NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = + Setting.intSetting("transport.netty.max_composite_buffer_components", -1, -1, Property.NodeScope, Property.Shared); + + // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one + public static final Setting NETTY_RECEIVE_PREDICTOR_SIZE = Setting.byteSizeSetting( + "transport.netty.receive_predictor_size", + settings -> { + long defaultReceiverPredictor = 512 * 1024; + if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) { + // we can guess a better default... + long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / WORKER_COUNT.get(settings)); + defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024)); + } + return new ByteSizeValue(defaultReceiverPredictor).toString(); + }, + Property.NodeScope, + Property.Shared); + public static final Setting NETTY_RECEIVE_PREDICTOR_MIN = + byteSizeSetting("transport.netty.receive_predictor_min", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared); + public static final Setting NETTY_RECEIVE_PREDICTOR_MAX = + byteSizeSetting("transport.netty.receive_predictor_max", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared); + public static final Setting NETTY_BOSS_COUNT = + intSetting("transport.netty.boss_count", 1, 1, Property.NodeScope, Property.Shared); + + + protected final ByteSizeValue maxCumulationBufferCapacity; + protected final int maxCompositeBufferComponents; + protected final RecvByteBufAllocator recvByteBufAllocator; + protected final int workerCount; + protected final ByteSizeValue receivePredictorMin; + protected final ByteSizeValue receivePredictorMax; + // package private for testing + volatile Netty4OpenChannelsHandler serverOpenChannels; + protected volatile Bootstrap bootstrap; + protected final Map serverBootstraps = newConcurrentMap(); + + @Inject + public Netty4Transport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays, + NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService) { + super("netty", settings, threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, networkService); + this.workerCount = WORKER_COUNT.get(settings); + this.maxCumulationBufferCapacity = NETTY_MAX_CUMULATION_BUFFER_CAPACITY.get(settings); + this.maxCompositeBufferComponents = NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS.get(settings); + + // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one + this.receivePredictorMin = NETTY_RECEIVE_PREDICTOR_MIN.get(settings); + this.receivePredictorMax = NETTY_RECEIVE_PREDICTOR_MAX.get(settings); + if (receivePredictorMax.bytes() == receivePredictorMin.bytes()) { + recvByteBufAllocator = new FixedRecvByteBufAllocator((int) receivePredictorMax.bytes()); + } else { + recvByteBufAllocator = new AdaptiveRecvByteBufAllocator((int) receivePredictorMin.bytes(), + (int) receivePredictorMin.bytes(), (int) receivePredictorMax.bytes()); + } + } + + TransportServiceAdapter transportServiceAdapter() { + return transportServiceAdapter; + } + + @Override + protected void doStart() { + boolean success = false; + try { + bootstrap = createBootstrap(); + if (NetworkService.NETWORK_SERVER.get(settings)) { + final Netty4OpenChannelsHandler openChannels = new Netty4OpenChannelsHandler(logger); + this.serverOpenChannels = openChannels; + // loop through all profiles and start them up, special handling for default one + for (Map.Entry entry : buildProfileSettings().entrySet()) { + // merge fallback settings with default settings with profile settings so we have complete settings with default values + final Settings settings = Settings.builder() + .put(createFallbackSettings()) + .put(entry.getValue()).build(); + createServerBootstrap(entry.getKey(), settings); + bindServer(entry.getKey(), settings); + } + } + super.doStart(); + success = true; + } finally { + if (success == false) { + doStop(); + } + } + } + + private Bootstrap createBootstrap() { + final Bootstrap bootstrap = new Bootstrap(); + if (TCP_BLOCKING_CLIENT.get(settings)) { + bootstrap.group(new OioEventLoopGroup(1, daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX))); + bootstrap.channel(OioSocketChannel.class); + } else { + bootstrap.group(new NioEventLoopGroup(workerCount, daemonThreadFactory(settings, TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX))); + bootstrap.channel(NioSocketChannel.class); + } + + bootstrap.handler(new ChannelInitializer() { + + @Override + protected void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast("size", new Netty4SizeHeaderFrameDecoder()); + // using a dot as a prefix means this cannot come from any settings parsed + ch.pipeline().addLast("dispatcher", new Netty4MessageChannelHandler(Netty4Transport.this, ".client")); + } + + }); + + bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, Math.toIntExact(connectTimeout.millis())); + bootstrap.option(ChannelOption.TCP_NODELAY, TCP_NO_DELAY.get(settings)); + bootstrap.option(ChannelOption.SO_KEEPALIVE, TCP_KEEP_ALIVE.get(settings)); + + final ByteSizeValue tcpSendBufferSize = TCP_SEND_BUFFER_SIZE.get(settings); + if (tcpSendBufferSize.bytes() > 0) { + bootstrap.option(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.bytes())); + } + + final ByteSizeValue tcpReceiveBufferSize = TCP_RECEIVE_BUFFER_SIZE.get(settings); + if (tcpReceiveBufferSize.bytes() > 0) { + bootstrap.option(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.bytes())); + } + + bootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator); + + final boolean reuseAddress = TCP_REUSE_ADDRESS.get(settings); + bootstrap.option(ChannelOption.SO_REUSEADDR, reuseAddress); + + bootstrap.validate(); + + return bootstrap; + } + + private Settings createFallbackSettings() { + Settings.Builder fallbackSettingsBuilder = Settings.builder(); + + List fallbackBindHost = TransportSettings.BIND_HOST.get(settings); + if (fallbackBindHost.isEmpty() == false) { + fallbackSettingsBuilder.putArray("bind_host", fallbackBindHost); + } + + List fallbackPublishHost = TransportSettings.PUBLISH_HOST.get(settings); + if (fallbackPublishHost.isEmpty() == false) { + fallbackSettingsBuilder.putArray("publish_host", fallbackPublishHost); + } + + boolean fallbackTcpNoDelay = settings.getAsBoolean("transport.netty.tcp_no_delay", TcpSettings.TCP_NO_DELAY.get(settings)); + fallbackSettingsBuilder.put("tcp_no_delay", fallbackTcpNoDelay); + + boolean fallbackTcpKeepAlive = settings.getAsBoolean("transport.netty.tcp_keep_alive", TcpSettings.TCP_KEEP_ALIVE.get(settings)); + fallbackSettingsBuilder.put("tcp_keep_alive", fallbackTcpKeepAlive); + + boolean fallbackReuseAddress = settings.getAsBoolean("transport.netty.reuse_address", TcpSettings.TCP_REUSE_ADDRESS.get(settings)); + fallbackSettingsBuilder.put("reuse_address", fallbackReuseAddress); + + ByteSizeValue fallbackTcpSendBufferSize = settings.getAsBytesSize("transport.netty.tcp_send_buffer_size", + TCP_SEND_BUFFER_SIZE.get(settings)); + if (fallbackTcpSendBufferSize.bytes() >= 0) { + fallbackSettingsBuilder.put("tcp_send_buffer_size", fallbackTcpSendBufferSize); + } + + ByteSizeValue fallbackTcpBufferSize = settings.getAsBytesSize("transport.netty.tcp_receive_buffer_size", + TCP_RECEIVE_BUFFER_SIZE.get(settings)); + if (fallbackTcpBufferSize.bytes() >= 0) { + fallbackSettingsBuilder.put("tcp_receive_buffer_size", fallbackTcpBufferSize); + } + + return fallbackSettingsBuilder.build(); + } + + private void createServerBootstrap(String name, Settings settings) { + if (logger.isDebugEnabled()) { + logger.debug("using profile[{}], worker_count[{}], port[{}], bind_host[{}], publish_host[{}], compress[{}], " + + "connect_timeout[{}], connections_per_node[{}/{}/{}/{}/{}], receive_predictor[{}->{}]", + name, workerCount, settings.get("port"), settings.get("bind_host"), settings.get("publish_host"), compress, + connectTimeout, connectionsPerNodeRecovery, connectionsPerNodeBulk, connectionsPerNodeReg, connectionsPerNodeState, + connectionsPerNodePing, receivePredictorMin, receivePredictorMax); + } + + final ThreadFactory workerFactory = daemonThreadFactory(this.settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX, name); + + final ServerBootstrap serverBootstrap = new ServerBootstrap(); + + if (TCP_BLOCKING_SERVER.get(settings)) { + serverBootstrap.group(new OioEventLoopGroup(workerCount, workerFactory)); + serverBootstrap.channel(OioServerSocketChannel.class); + } else { + serverBootstrap.group(new NioEventLoopGroup(workerCount, workerFactory)); + serverBootstrap.channel(NioServerSocketChannel.class); + } + + serverBootstrap.childHandler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast("open_channels", Netty4Transport.this.serverOpenChannels); + ch.pipeline().addLast("size", new Netty4SizeHeaderFrameDecoder()); + ch.pipeline().addLast("dispatcher", new Netty4MessageChannelHandler(Netty4Transport.this, name)); + } + }); + + serverBootstrap.childOption(ChannelOption.TCP_NODELAY, TCP_NO_DELAY.get(settings)); + serverBootstrap.childOption(ChannelOption.SO_KEEPALIVE, TCP_KEEP_ALIVE.get(settings)); + + final ByteSizeValue tcpSendBufferSize = TCP_SEND_BUFFER_SIZE.getDefault(settings); + if (tcpSendBufferSize != null && tcpSendBufferSize.bytes() > 0) { + serverBootstrap.childOption(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.bytes())); + } + + final ByteSizeValue tcpReceiveBufferSize = TCP_RECEIVE_BUFFER_SIZE.getDefault(settings); + if (tcpReceiveBufferSize != null && tcpReceiveBufferSize.bytes() > 0) { + serverBootstrap.childOption(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.bytesAsInt())); + } + + serverBootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator); + serverBootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator); + + final boolean reuseAddress = TCP_REUSE_ADDRESS.get(settings); + serverBootstrap.option(ChannelOption.SO_REUSEADDR, reuseAddress); + serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, reuseAddress); + + serverBootstrap.validate(); + + serverBootstraps.put(name, serverBootstrap); + } + + protected final void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + final Throwable unwrapped = ExceptionsHelper.unwrap(cause, ElasticsearchException.class); + final Throwable t = unwrapped != null ? unwrapped : cause; + onException(ctx.channel(), t instanceof Exception ? (Exception) t : new ElasticsearchException(t)); + } + + @Override + public long serverOpen() { + Netty4OpenChannelsHandler channels = serverOpenChannels; + return channels == null ? 0 : channels.numberOfOpenChannels(); + } + + protected NodeChannels connectToChannelsLight(DiscoveryNode node) { + InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address(); + ChannelFuture connect = bootstrap.connect(address); + connect.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5)); + if (!connect.isSuccess()) { + throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connect.cause()); + } + Channel[] channels = new Channel[1]; + channels[0] = connect.channel(); + channels[0].closeFuture().addListener(new ChannelCloseListener(node)); + return new NodeChannels(channels, channels, channels, channels, channels); + } + + protected NodeChannels connectToChannels(DiscoveryNode node) { + final NodeChannels nodeChannels = + new NodeChannels( + new Channel[connectionsPerNodeRecovery], + new Channel[connectionsPerNodeBulk], + new Channel[connectionsPerNodeReg], + new Channel[connectionsPerNodeState], + new Channel[connectionsPerNodePing]); + boolean success = false; + try { + int numConnections = + connectionsPerNodeRecovery + + connectionsPerNodeBulk + + connectionsPerNodeReg + + connectionsPerNodeState + + connectionsPerNodeRecovery; + final ArrayList connections = new ArrayList<>(numConnections); + final InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address(); + for (int i = 0; i < numConnections; i++) { + connections.add(bootstrap.connect(address)); + } + final Iterator iterator = connections.iterator(); + try { + for (Channel[] channels : nodeChannels.getChannelArrays()) { + for (int i = 0; i < channels.length; i++) { + assert iterator.hasNext(); + ChannelFuture future = iterator.next(); + future.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5)); + if (!future.isSuccess()) { + throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", future.cause()); + } + channels[i] = future.channel(); + channels[i].closeFuture().addListener(new ChannelCloseListener(node)); + } + } + if (nodeChannels.recovery.length == 0) { + if (nodeChannels.bulk.length > 0) { + nodeChannels.recovery = nodeChannels.bulk; + } else { + nodeChannels.recovery = nodeChannels.reg; + } + } + if (nodeChannels.bulk.length == 0) { + nodeChannels.bulk = nodeChannels.reg; + } + } catch (final RuntimeException e) { + for (final ChannelFuture future : Collections.unmodifiableList(connections)) { + FutureUtils.cancel(future); + if (future.channel() != null && future.channel().isOpen()) { + try { + future.channel().close(); + } catch (Exception inner) { + e.addSuppressed(inner); + } + } + } + throw e; + } + success = true; + } finally { + if (success == false) { + try { + nodeChannels.close(); + } catch (IOException e) { + logger.trace("exception while closing channels", e); + } + } + } + return nodeChannels; + } + + private class ChannelCloseListener implements ChannelFutureListener { + + private final DiscoveryNode node; + + private ChannelCloseListener(DiscoveryNode node) { + this.node = node; + } + + @Override + public void operationComplete(final ChannelFuture future) throws Exception { + NodeChannels nodeChannels = connectedNodes.get(node); + if (nodeChannels != null && nodeChannels.hasChannel(future.channel())) { + threadPool.generic().execute(() -> disconnectFromNode(node, future.channel(), "channel closed event")); + } + } + } + + @Override + protected void sendMessage(Channel channel, BytesReference reference, Runnable sendListener, boolean close) { + final ChannelFuture future = channel.writeAndFlush(Netty4Utils.toByteBuf(reference)); + if (close) { + future.addListener(f -> { + try { + sendListener.run(); + } finally { + future.channel().close(); + } + }); + } else { + future.addListener(f -> sendListener.run()); + } + } + + @Override + protected void closeChannels(final List channels) throws IOException { + Netty4Utils.closeChannels(channels); + } + + @Override + protected InetSocketAddress getLocalAddress(Channel channel) { + return (InetSocketAddress) channel.localAddress(); + } + + @Override + protected Channel bind(String name, InetSocketAddress address) { + return serverBootstraps.get(name).bind(address).syncUninterruptibly().channel(); + } + + ScheduledPing getPing() { + return scheduledPing; + } + + @Override + protected boolean isOpen(Channel channel) { + return channel.isOpen(); + } + + @Override + @SuppressForbidden(reason = "debug") + protected void stopInternal() { + Releasables.close(serverOpenChannels, () -> { + final List>> serverBootstrapCloseFutures = new ArrayList<>(serverBootstraps.size()); + for (final Map.Entry entry : serverBootstraps.entrySet()) { + serverBootstrapCloseFutures.add( + Tuple.tuple(entry.getKey(), entry.getValue().config().group().shutdownGracefully(0, 5, TimeUnit.SECONDS))); + } + for (final Tuple> future : serverBootstrapCloseFutures) { + future.v2().awaitUninterruptibly(); + if (!future.v2().isSuccess()) { + logger.debug("Error closing server bootstrap for profile [{}]", future.v2().cause(), future.v1()); + } + } + serverBootstraps.clear(); + + if (bootstrap != null) { + bootstrap.config().group().shutdownGracefully(0, 5, TimeUnit.SECONDS).awaitUninterruptibly(); + bootstrap = null; + } + }); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java new file mode 100644 index 00000000000..53cf1b329aa --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java @@ -0,0 +1,121 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.CompositeByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.util.internal.logging.InternalLogger; +import io.netty.util.internal.logging.InternalLoggerFactory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.elasticsearch.common.bytes.BytesReference; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class Netty4Utils { + + static { + InternalLoggerFactory.setDefaultFactory(new InternalLoggerFactory() { + + @Override + public InternalLogger newInstance(final String name) { + return new Netty4InternalESLogger(name.replace("io.netty.", "netty.")); + } + + }); + } + + public static void setup() { + + } + + /** + * Turns the given BytesReference into a ByteBuf. Note: the returned ByteBuf will reference the internal + * pages of the BytesReference. Don't free the bytes of reference before the ByteBuf goes out of scope. + */ + public static ByteBuf toByteBuf(final BytesReference reference) { + if (reference.length() == 0) { + return Unpooled.EMPTY_BUFFER; + } + if (reference instanceof ByteBufBytesReference) { + return ((ByteBufBytesReference) reference).toByteBuf(); + } else { + final BytesRefIterator iterator = reference.iterator(); + // usually we have one, two, or three components + // from the header, the message, and a buffer + final List buffers = new ArrayList<>(3); + try { + BytesRef slice; + while ((slice = iterator.next()) != null) { + buffers.add(Unpooled.wrappedBuffer(slice.bytes, slice.offset, slice.length)); + } + final CompositeByteBuf composite = Unpooled.compositeBuffer(buffers.size()); + composite.addComponents(true, buffers); + return composite; + } catch (IOException ex) { + throw new AssertionError("no IO happens here", ex); + } + } + } + + /** + * Wraps the given ChannelBuffer with a BytesReference + */ + public static BytesReference toBytesReference(final ByteBuf buffer) { + return toBytesReference(buffer, buffer.readableBytes()); + } + + /** + * Wraps the given ChannelBuffer with a BytesReference of a given size + */ + static BytesReference toBytesReference(final ByteBuf buffer, final int size) { + return new ByteBufBytesReference(buffer, size); + } + + public static void closeChannels(final Collection channels) throws IOException { + IOException closingExceptions = null; + final List futures = new ArrayList<>(); + for (final Channel channel : channels) { + try { + if (channel != null && channel.isOpen()) { + futures.add(channel.close()); + } + } catch (Exception e) { + if (closingExceptions == null) { + closingExceptions = new IOException("failed to close channels"); + } + closingExceptions.addSuppressed(e); + } + } + for (final ChannelFuture future : futures) { + future.awaitUninterruptibly(); + } + + if (closingExceptions != null) { + throw closingExceptions; + } + } + +} diff --git a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 00000000000..45c8cd923aa --- /dev/null +++ b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,24 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +grant { + // Netty SelectorUtil wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854 + // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! + permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; +}; \ No newline at end of file diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java b/modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java new file mode 100644 index 00000000000..1feded41ed5 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch; + +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.transport.Netty4Plugin; +import org.elasticsearch.transport.netty4.Netty4Transport; + +import java.util.Collection; + +@ESIntegTestCase.SuppressLocalMode +public abstract class ESNetty4IntegTestCase extends ESIntegTestCase { + + @Override + protected boolean ignoreExternalCluster() { + return true; + } + + @Override + protected boolean addMockTransportService() { + return false; + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); + // randomize netty settings + if (randomBoolean()) { + builder.put(Netty4Transport.WORKER_COUNT.getKey(), random().nextInt(3) + 1); + } + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME); + builder.put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME); + return builder.build(); + } + + @Override + protected Settings transportClientSettings() { + Settings.Builder builder = Settings.builder().put(super.transportClientSettings()); + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME); + return builder.build(); + } + + @Override + protected Collection> nodePlugins() { + return pluginList(Netty4Plugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return pluginList(Netty4Plugin.class); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java new file mode 100644 index 00000000000..58c6a476fe8 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java @@ -0,0 +1,490 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelConfig; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelId; +import io.netty.channel.ChannelMetadata; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.ChannelProgressivePromise; +import io.netty.channel.ChannelPromise; +import io.netty.channel.EventLoop; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.util.Attribute; +import io.netty.util.AttributeKey; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.http.HttpTransportSettings; +import org.elasticsearch.http.netty4.cors.Netty4CorsHandler; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.netty4.Netty4Utils; +import org.junit.After; +import org.junit.Before; + +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class Netty4HttpChannelTests extends ESTestCase { + + private NetworkService networkService; + private ThreadPool threadPool; + private MockBigArrays bigArrays; + + @Before + public void setup() throws Exception { + networkService = new NetworkService(Settings.EMPTY); + threadPool = new TestThreadPool("test"); + bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()); + } + + @After + public void shutdown() throws Exception { + if (threadPool != null) { + threadPool.shutdownNow(); + } + } + + public void testResponse() { + final FullHttpResponse response = executeRequest(Settings.EMPTY, "request-host"); + assertThat(response.content(), equalTo(Netty4Utils.toByteBuf(new TestResponse().content()))); + } + + public void testCorsEnabledWithoutAllowOrigins() { + // Set up a HTTP transport with only the CORS enabled setting + Settings settings = Settings.builder() + .put(HttpTransportSettings.SETTING_CORS_ENABLED.getKey(), true) + .build(); + HttpResponse response = executeRequest(settings, "remote-host", "request-host"); + // inspect response and validate + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), nullValue()); + } + + public void testCorsEnabledWithAllowOrigins() { + final String originValue = "remote-host"; + // create a http transport with CORS enabled and allow origin configured + Settings settings = Settings.builder() + .put(SETTING_CORS_ENABLED.getKey(), true) + .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue) + .build(); + HttpResponse response = executeRequest(settings, originValue, "request-host"); + // inspect response and validate + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + } + + public void testCorsAllowOriginWithSameHost() { + String originValue = "remote-host"; + String host = "remote-host"; + // create a http transport with CORS enabled + Settings settings = Settings.builder() + .put(SETTING_CORS_ENABLED.getKey(), true) + .build(); + HttpResponse response = executeRequest(settings, originValue, host); + // inspect response and validate + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + + originValue = "http://" + originValue; + response = executeRequest(settings, originValue, host); + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + + originValue = originValue + ":5555"; + host = host + ":5555"; + response = executeRequest(settings, originValue, host); + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + + originValue = originValue.replace("http", "https"); + response = executeRequest(settings, originValue, host); + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + } + + public void testThatStringLiteralWorksOnMatch() { + final String originValue = "remote-host"; + Settings settings = Settings.builder() + .put(SETTING_CORS_ENABLED.getKey(), true) + .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue) + .put(SETTING_CORS_ALLOW_METHODS.getKey(), "get, options, post") + .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true) + .build(); + HttpResponse response = executeRequest(settings, originValue, "request-host"); + // inspect response and validate + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS), equalTo("true")); + } + + public void testThatAnyOriginWorks() { + final String originValue = Netty4CorsHandler.ANY_ORIGIN; + Settings settings = Settings.builder() + .put(SETTING_CORS_ENABLED.getKey(), true) + .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue) + .build(); + HttpResponse response = executeRequest(settings, originValue, "request-host"); + // inspect response and validate + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS), nullValue()); + } + + public void testHeadersSet() { + Settings settings = Settings.builder().build(); + try (Netty4HttpServerTransport httpServerTransport = + new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool)) { + httpServerTransport.start(); + final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/"); + httpRequest.headers().add(HttpHeaderNames.ORIGIN, "remote"); + final WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel(); + Netty4HttpRequest request = new Netty4HttpRequest(httpRequest, writeCapturingChannel); + + // send a response + Netty4HttpChannel channel = + new Netty4HttpChannel(httpServerTransport, request, null, randomBoolean(), threadPool.getThreadContext()); + TestResponse resp = new TestResponse(); + final String customHeader = "custom-header"; + final String customHeaderValue = "xyz"; + resp.addHeader(customHeader, customHeaderValue); + channel.sendResponse(resp); + + // inspect what was written + List writtenObjects = writeCapturingChannel.getWrittenObjects(); + assertThat(writtenObjects.size(), is(1)); + HttpResponse response = (HttpResponse) writtenObjects.get(0); + assertThat(response.headers().get("non-existent-header"), nullValue()); + assertThat(response.headers().get(customHeader), equalTo(customHeaderValue)); + assertThat(response.headers().get(HttpHeaderNames.CONTENT_LENGTH), equalTo(Integer.toString(resp.content().length()))); + assertThat(response.headers().get(HttpHeaderNames.CONTENT_TYPE), equalTo(resp.contentType())); + } + } + + private FullHttpResponse executeRequest(final Settings settings, final String host) { + return executeRequest(settings, null, host); + } + + private FullHttpResponse executeRequest(final Settings settings, final String originValue, final String host) { + // construct request and send it over the transport layer + try (Netty4HttpServerTransport httpServerTransport = + new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool)) { + httpServerTransport.start(); + final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/"); + if (originValue != null) { + httpRequest.headers().add(HttpHeaderNames.ORIGIN, originValue); + } + httpRequest.headers().add(HttpHeaderNames.HOST, host); + final WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel(); + final Netty4HttpRequest request = new Netty4HttpRequest(httpRequest, writeCapturingChannel); + + Netty4HttpChannel channel = + new Netty4HttpChannel(httpServerTransport, request, null, randomBoolean(), threadPool.getThreadContext()); + channel.sendResponse(new TestResponse()); + + // get the response + List writtenObjects = writeCapturingChannel.getWrittenObjects(); + assertThat(writtenObjects.size(), is(1)); + return (FullHttpResponse) writtenObjects.get(0); + } + } + + private static class WriteCapturingChannel implements Channel { + + private List writtenObjects = new ArrayList<>(); + + @Override + public ChannelId id() { + return null; + } + + @Override + public EventLoop eventLoop() { + return null; + } + + @Override + public Channel parent() { + return null; + } + + @Override + public ChannelConfig config() { + return null; + } + + @Override + public boolean isOpen() { + return false; + } + + @Override + public boolean isRegistered() { + return false; + } + + @Override + public boolean isActive() { + return false; + } + + @Override + public ChannelMetadata metadata() { + return null; + } + + @Override + public SocketAddress localAddress() { + return null; + } + + @Override + public SocketAddress remoteAddress() { + return null; + } + + @Override + public ChannelFuture closeFuture() { + return null; + } + + @Override + public boolean isWritable() { + return false; + } + + @Override + public long bytesBeforeUnwritable() { + return 0; + } + + @Override + public long bytesBeforeWritable() { + return 0; + } + + @Override + public Unsafe unsafe() { + return null; + } + + @Override + public ChannelPipeline pipeline() { + return null; + } + + @Override + public ByteBufAllocator alloc() { + return null; + } + + @Override + public Channel read() { + return null; + } + + @Override + public Channel flush() { + return null; + } + + @Override + public ChannelFuture bind(SocketAddress localAddress) { + return null; + } + + @Override + public ChannelFuture connect(SocketAddress remoteAddress) { + return null; + } + + @Override + public ChannelFuture connect(SocketAddress remoteAddress, SocketAddress localAddress) { + return null; + } + + @Override + public ChannelFuture disconnect() { + return null; + } + + @Override + public ChannelFuture close() { + return null; + } + + @Override + public ChannelFuture deregister() { + return null; + } + + @Override + public ChannelFuture bind(SocketAddress localAddress, ChannelPromise promise) { + return null; + } + + @Override + public ChannelFuture connect(SocketAddress remoteAddress, ChannelPromise promise) { + return null; + } + + @Override + public ChannelFuture connect(SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) { + return null; + } + + @Override + public ChannelFuture disconnect(ChannelPromise promise) { + return null; + } + + @Override + public ChannelFuture close(ChannelPromise promise) { + return null; + } + + @Override + public ChannelFuture deregister(ChannelPromise promise) { + return null; + } + + @Override + public ChannelFuture write(Object msg) { + writtenObjects.add(msg); + return null; + } + + @Override + public ChannelFuture write(Object msg, ChannelPromise promise) { + writtenObjects.add(msg); + return null; + } + + @Override + public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) { + writtenObjects.add(msg); + return null; + } + + @Override + public ChannelFuture writeAndFlush(Object msg) { + writtenObjects.add(msg); + return null; + } + + @Override + public ChannelPromise newPromise() { + return null; + } + + @Override + public ChannelProgressivePromise newProgressivePromise() { + return null; + } + + @Override + public ChannelFuture newSucceededFuture() { + return null; + } + + @Override + public ChannelFuture newFailedFuture(Throwable cause) { + return null; + } + + @Override + public ChannelPromise voidPromise() { + return null; + } + + @Override + public Attribute attr(AttributeKey key) { + return null; + } + + @Override + public boolean hasAttr(AttributeKey key) { + return false; + } + + @Override + public int compareTo(Channel o) { + return 0; + } + + List getWrittenObjects() { + return writtenObjects; + } + + } + + private static class TestResponse extends RestResponse { + + @Override + public String contentType() { + return "text"; + } + + @Override + public BytesReference content() { + return Netty4Utils.toBytesReference(Unpooled.copiedBuffer("content", StandardCharsets.UTF_8)); + } + + @Override + public RestStatus status() { + return RestStatus.OK; + } + + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java new file mode 100644 index 00000000000..31e7a6112f9 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java @@ -0,0 +1,190 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.http.netty4; + +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpObject; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpRequestEncoder; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseDecoder; +import io.netty.handler.codec.http.HttpVersion; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; + +import java.io.Closeable; +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import static io.netty.handler.codec.http.HttpHeaderNames.HOST; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; + +/** + * Tiny helper to send http requests over netty. + */ +class Netty4HttpClient implements Closeable { + + static Collection returnHttpResponseBodies(Collection responses) { + List list = new ArrayList<>(responses.size()); + for (FullHttpResponse response : responses) { + list.add(response.content().toString(StandardCharsets.UTF_8)); + } + return list; + } + + static Collection returnOpaqueIds(Collection responses) { + List list = new ArrayList<>(responses.size()); + for (HttpResponse response : responses) { + list.add(response.headers().get("X-Opaque-Id")); + } + return list; + } + + private final Bootstrap clientBootstrap; + + Netty4HttpClient() { + clientBootstrap = new Bootstrap().channel(NioSocketChannel.class).group(new NioEventLoopGroup()); + } + + public Collection get(SocketAddress remoteAddress, String... uris) throws InterruptedException { + Collection requests = new ArrayList<>(uris.length); + for (int i = 0; i < uris.length; i++) { + final HttpRequest httpRequest = new DefaultFullHttpRequest(HTTP_1_1, HttpMethod.GET, uris[i]); + httpRequest.headers().add(HOST, "localhost"); + httpRequest.headers().add("X-Opaque-ID", String.valueOf(i)); + requests.add(httpRequest); + } + return sendRequests(remoteAddress, requests); + } + + @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods. + public final Collection post(SocketAddress remoteAddress, Tuple... urisAndBodies) + throws InterruptedException { + return processRequestsWithBody(HttpMethod.POST, remoteAddress, urisAndBodies); + } + + @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods. + public final Collection put(SocketAddress remoteAddress, Tuple... urisAndBodies) + throws InterruptedException { + return processRequestsWithBody(HttpMethod.PUT, remoteAddress, urisAndBodies); + } + + private Collection processRequestsWithBody(HttpMethod method, SocketAddress remoteAddress, Tuple... urisAndBodies) throws InterruptedException { + Collection requests = new ArrayList<>(urisAndBodies.length); + for (Tuple uriAndBody : urisAndBodies) { + ByteBuf content = Unpooled.copiedBuffer(uriAndBody.v2(), StandardCharsets.UTF_8); + HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, method, uriAndBody.v1(), content); + request.headers().add(HttpHeaderNames.HOST, "localhost"); + request.headers().add(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes()); + requests.add(request); + } + return sendRequests(remoteAddress, requests); + } + + private synchronized Collection sendRequests( + final SocketAddress remoteAddress, + final Collection requests) throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(requests.size()); + final Collection content = Collections.synchronizedList(new ArrayList<>(requests.size())); + + clientBootstrap.handler(new CountDownLatchHandler(latch, content)); + + ChannelFuture channelFuture = null; + try { + channelFuture = clientBootstrap.connect(remoteAddress); + channelFuture.sync(); + + for (HttpRequest request : requests) { + channelFuture.channel().writeAndFlush(request); + } + latch.await(); + + } finally { + if (channelFuture != null) { + channelFuture.channel().close().sync(); + } + } + + return content; + } + + @Override + public void close() { + clientBootstrap.config().group().shutdownGracefully().awaitUninterruptibly(); + } + + /** + * helper factory which adds returned data to a list and uses a count down latch to decide when done + */ + private static class CountDownLatchHandler extends ChannelInitializer { + + private final CountDownLatch latch; + private final Collection content; + + CountDownLatchHandler(final CountDownLatch latch, final Collection content) { + this.latch = latch; + this.content = content; + } + + @Override + protected void initChannel(SocketChannel ch) throws Exception { + final int maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB).bytesAsInt(); + ch.pipeline().addLast(new HttpResponseDecoder()); + ch.pipeline().addLast(new HttpRequestEncoder()); + ch.pipeline().addLast(new HttpObjectAggregator(maxContentLength)); + ch.pipeline().addLast(new SimpleChannelInboundHandler() { + @Override + protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception { + final FullHttpResponse response = (FullHttpResponse) msg; + content.add(response.copy()); + latch.countDown(); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + super.exceptionCaught(ctx, cause); + latch.countDown(); + } + }); + } + + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPublishPortTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPublishPortTests.java new file mode 100644 index 00000000000..03e09f28d25 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPublishPortTests.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import org.elasticsearch.common.network.NetworkUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.http.BindHttpException; +import org.elasticsearch.http.HttpTransportSettings; +import org.elasticsearch.test.ESTestCase; + +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; + +import static java.net.InetAddress.getByName; +import static java.util.Arrays.asList; +import static org.elasticsearch.http.netty4.Netty4HttpServerTransport.resolvePublishPort; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class Netty4HttpPublishPortTests extends ESTestCase { + + public void testHttpPublishPort() throws Exception { + int boundPort = randomIntBetween(9000, 9100); + int otherBoundPort = randomIntBetween(9200, 9300); + + int publishPort = resolvePublishPort(Settings.builder().put(HttpTransportSettings.SETTING_HTTP_PUBLISH_PORT.getKey(), 9080).build(), + randomAddresses(), getByName("127.0.0.2")); + assertThat("Publish port should be explicitly set to 9080", publishPort, equalTo(9080)); + + publishPort = resolvePublishPort(Settings.EMPTY, asList(address("127.0.0.1", boundPort), address("127.0.0.2", otherBoundPort)), + getByName("127.0.0.1")); + assertThat("Publish port should be derived from matched address", publishPort, equalTo(boundPort)); + + publishPort = resolvePublishPort(Settings.EMPTY, asList(address("127.0.0.1", boundPort), address("127.0.0.2", boundPort)), + getByName("127.0.0.3")); + assertThat("Publish port should be derived from unique port of bound addresses", publishPort, equalTo(boundPort)); + + final BindHttpException e = + expectThrows(BindHttpException.class, + () -> resolvePublishPort( + Settings.EMPTY, + asList(address("127.0.0.1", boundPort), address("127.0.0.2", otherBoundPort)), + getByName("127.0.0.3"))); + assertThat(e.getMessage(), containsString("Failed to auto-resolve http publish port")); + + publishPort = resolvePublishPort(Settings.EMPTY, asList(address("0.0.0.0", boundPort), address("127.0.0.2", otherBoundPort)), + getByName("127.0.0.1")); + assertThat("Publish port should be derived from matching wildcard address", publishPort, equalTo(boundPort)); + + if (NetworkUtils.SUPPORTS_V6) { + publishPort = resolvePublishPort(Settings.EMPTY, asList(address("0.0.0.0", boundPort), address("127.0.0.2", otherBoundPort)), + getByName("::1")); + assertThat("Publish port should be derived from matching wildcard address", publishPort, equalTo(boundPort)); + } + } + + private InetSocketTransportAddress address(String host, int port) throws UnknownHostException { + return new InetSocketTransportAddress(getByName(host), port); + } + + private InetSocketTransportAddress randomAddress() throws UnknownHostException { + return address("127.0.0." + randomIntBetween(1, 100), randomIntBetween(9200, 9300)); + } + + private List randomAddresses() throws UnknownHostException { + List addresses = new ArrayList<>(); + for (int i = 0; i < randomIntBetween(1, 5); i++) { + addresses.add(randomAddress()); + } + return addresses; + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java new file mode 100644 index 00000000000..f1e2e922cbd --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java @@ -0,0 +1,131 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.handler.codec.http.FullHttpResponse; +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; + +import java.util.Collection; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; + +/** + * This test checks that in-flight requests are limited on HTTP level and that requests that are excluded from limiting can pass. + * + * As the same setting is also used to limit in-flight requests on transport level, we avoid transport messages by forcing + * a single node "cluster". We also force test infrastructure to use the node client instead of the transport client for the same reason. + */ +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numClientNodes = 0, numDataNodes = 1, transportClientRatio = 0) +public class Netty4HttpRequestSizeLimitIT extends ESNetty4IntegTestCase { + + private static final ByteSizeValue LIMIT = new ByteSizeValue(2, ByteSizeUnit.KB); + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.HTTP_ENABLED.getKey(), true) + .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), LIMIT) + .build(); + } + + public void testLimitsInFlightRequests() throws Exception { + ensureGreen(); + + // we use the limit size as a (very) rough indication on how many requests we should sent to hit the limit + int numRequests = LIMIT.bytesAsInt() / 100; + + StringBuilder bulkRequest = new StringBuilder(); + for (int i = 0; i < numRequests; i++) { + bulkRequest.append("{\"index\": {}}"); + bulkRequest.append(System.lineSeparator()); + bulkRequest.append("{ \"field\" : \"value\" }"); + bulkRequest.append(System.lineSeparator()); + } + + @SuppressWarnings("unchecked") + Tuple[] requests = new Tuple[150]; + for (int i = 0; i < requests.length; i++) { + requests[i] = Tuple.tuple("/index/type/_bulk", bulkRequest); + } + + HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class); + InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress + ().boundAddresses()); + + try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) { + Collection singleResponse = nettyHttpClient.post(inetSocketTransportAddress.address(), requests[0]); + assertThat(singleResponse, hasSize(1)); + assertAtLeastOnceExpectedStatus(singleResponse, HttpResponseStatus.OK); + + Collection multipleResponses = nettyHttpClient.post(inetSocketTransportAddress.address(), requests); + assertThat(multipleResponses, hasSize(requests.length)); + assertAtLeastOnceExpectedStatus(multipleResponses, HttpResponseStatus.SERVICE_UNAVAILABLE); + } + } + + public void testDoesNotLimitExcludedRequests() throws Exception { + ensureGreen(); + + @SuppressWarnings("unchecked") + Tuple[] requestUris = new Tuple[1500]; + for (int i = 0; i < requestUris.length; i++) { + requestUris[i] = Tuple.tuple("/_cluster/settings", + "{ \"transient\": {\"indices.ttl.interval\": \"40s\" } }"); + } + + HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class); + InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress + ().boundAddresses()); + + try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) { + Collection responses = nettyHttpClient.put(inetSocketTransportAddress.address(), requestUris); + assertThat(responses, hasSize(requestUris.length)); + assertAllInExpectedStatus(responses, HttpResponseStatus.OK); + } + } + + private void assertAtLeastOnceExpectedStatus(Collection responses, HttpResponseStatus expectedStatus) { + long countExpectedStatus = responses.stream().filter(r -> r.status().equals(expectedStatus)).count(); + assertThat("Expected at least one request with status [" + expectedStatus + "]", countExpectedStatus, greaterThan(0L)); + } + + private void assertAllInExpectedStatus(Collection responses, HttpResponseStatus expectedStatus) { + long countUnexpectedStatus = responses.stream().filter(r -> r.status().equals(expectedStatus) == false).count(); + assertThat("Expected all requests with status [" + expectedStatus + "] but [" + countUnexpectedStatus + + "] requests had a different one", countUnexpectedStatus, equalTo(0L)); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java new file mode 100644 index 00000000000..d8e61908011 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java @@ -0,0 +1,260 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.QueryStringDecoder; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.http.netty4.pipelining.HttpPipelinedRequest; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.hasSize; + +/** + * This test just tests, if he pipelining works in general with out any connection the Elasticsearch handler + */ +public class Netty4HttpServerPipeliningTests extends ESTestCase { + private NetworkService networkService; + private ThreadPool threadPool; + private MockBigArrays bigArrays; + + @Before + public void setup() throws Exception { + networkService = new NetworkService(Settings.EMPTY); + threadPool = new TestThreadPool("test"); + bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()); + } + + @After + public void shutdown() throws Exception { + if (threadPool != null) { + threadPool.shutdownNow(); + } + } + + public void testThatHttpPipeliningWorksWhenEnabled() throws Exception { + final Settings settings = Settings.builder() + .put("http.pipelining", true) + .put("http.port", "0") + .build(); + try (final HttpServerTransport httpServerTransport = new CustomNettyHttpServerTransport(settings)) { + httpServerTransport.start(); + final InetSocketTransportAddress transportAddress = + (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress().boundAddresses()); + + final int numberOfRequests = randomIntBetween(4, 16); + final List requests = new ArrayList<>(numberOfRequests); + for (int i = 0; i < numberOfRequests; i++) { + if (rarely()) { + requests.add("/slow?sleep=" + scaledRandomIntBetween(500, 1000)); + } else { + requests.add("/" + i); + } + } + + try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) { + Collection responses = nettyHttpClient.get(transportAddress.address(), requests.toArray(new String[]{})); + Collection responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses); + assertThat(responseBodies, contains(requests.toArray())); + } + } + } + + public void testThatHttpPipeliningCanBeDisabled() throws Exception { + final Settings settings = Settings.builder() + .put("http.pipelining", false) + .put("http.port", "0") + .build(); + try (final HttpServerTransport httpServerTransport = new CustomNettyHttpServerTransport(settings)) { + httpServerTransport.start(); + final InetSocketTransportAddress transportAddress = + (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress().boundAddresses()); + + final int numberOfRequests = randomIntBetween(4, 16); + final int numberOfSlowRequests = scaledRandomIntBetween(1, numberOfRequests); + final List requests = new ArrayList<>(numberOfRequests); + for (int i = 0; i < numberOfRequests - numberOfSlowRequests; i++) { + requests.add("/" + i); + } + for (int i = 0; i < numberOfSlowRequests; i++) { + requests.add("/slow?sleep=" + sleep(i)); + } + + try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) { + Collection responses = nettyHttpClient.get(transportAddress.address(), requests.toArray(new String[]{})); + List responseBodies = new ArrayList<>(Netty4HttpClient.returnHttpResponseBodies(responses)); + // we cannot be sure about the order of the fast requests, but the slow ones should have to be last + assertThat(responseBodies, hasSize(numberOfRequests)); + for (int i = 0; i < numberOfSlowRequests; i++) { + assertThat(responseBodies.get(numberOfRequests - numberOfSlowRequests + i), equalTo("/slow?sleep=" + sleep(i))); + } + } + } + } + + + private int sleep(int index) { + return 500 + 100 * (index + 1); + } + + class CustomNettyHttpServerTransport extends Netty4HttpServerTransport { + + private final ExecutorService executorService = Executors.newCachedThreadPool(); + + CustomNettyHttpServerTransport(final Settings settings) { + super(settings, + Netty4HttpServerPipeliningTests.this.networkService, + Netty4HttpServerPipeliningTests.this.bigArrays, + Netty4HttpServerPipeliningTests.this.threadPool); + } + + @Override + public ChannelHandler configureServerChannelHandler() { + return new CustomHttpChannelHandler(this, executorService, Netty4HttpServerPipeliningTests.this.threadPool.getThreadContext()); + } + + @Override + protected void doClose() { + executorService.shutdown(); + super.doClose(); + } + + } + + private class CustomHttpChannelHandler extends Netty4HttpServerTransport.HttpChannelHandler { + + private final ExecutorService executorService; + + CustomHttpChannelHandler(Netty4HttpServerTransport transport, ExecutorService executorService, ThreadContext threadContext) { + super(transport, randomBoolean(), threadContext); + this.executorService = executorService; + } + + @Override + protected void initChannel(SocketChannel ch) throws Exception { + super.initChannel(ch); + ch.pipeline().replace("handler", "handler", new PossiblySlowUpstreamHandler(executorService)); + } + + } + + class PossiblySlowUpstreamHandler extends SimpleChannelInboundHandler { + + private final ExecutorService executorService; + + PossiblySlowUpstreamHandler(ExecutorService executorService) { + this.executorService = executorService; + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception { + executorService.submit(new PossiblySlowRunnable(ctx, msg)); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + logger.info("Caught exception", cause); + ctx.channel().close().sync(); + } + + } + + class PossiblySlowRunnable implements Runnable { + + private ChannelHandlerContext ctx; + private HttpPipelinedRequest pipelinedRequest; + private FullHttpRequest fullHttpRequest; + + PossiblySlowRunnable(ChannelHandlerContext ctx, Object msg) { + this.ctx = ctx; + if (msg instanceof HttpPipelinedRequest) { + this.pipelinedRequest = (HttpPipelinedRequest) msg; + } else if (msg instanceof FullHttpRequest) { + this.fullHttpRequest = (FullHttpRequest) msg; + } + } + + @Override + public void run() { + final String uri; + if (pipelinedRequest != null && pipelinedRequest.last() instanceof FullHttpRequest) { + uri = ((FullHttpRequest) pipelinedRequest.last()).uri(); + } else { + uri = fullHttpRequest.uri(); + } + + final ByteBuf buffer = Unpooled.copiedBuffer(uri, StandardCharsets.UTF_8); + + final DefaultFullHttpResponse httpResponse = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK, buffer); + httpResponse.headers().add(HttpHeaderNames.CONTENT_LENGTH, buffer.readableBytes()); + + final QueryStringDecoder decoder = new QueryStringDecoder(uri); + + final int timeout = + uri.startsWith("/slow") && decoder.parameters().containsKey("sleep") ? + Integer.valueOf(decoder.parameters().get("sleep").get(0)) : 0; + if (timeout > 0) { + try { + Thread.sleep(timeout); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + if (pipelinedRequest != null) { + ctx.writeAndFlush(pipelinedRequest.createHttpResponse(httpResponse, ctx.channel().newPromise())); + } else { + ctx.writeAndFlush(httpResponse); + } + } + + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java new file mode 100644 index 00000000000..dc63907265c --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.handler.codec.http.HttpMethod; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.http.netty4.cors.Netty4CorsConfig; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED; +import static org.hamcrest.Matchers.equalTo; + +/** + * Tests for the {@link Netty4HttpServerTransport} class. + */ +public class Netty4HttpServerTransportTests extends ESTestCase { + + private NetworkService networkService; + private ThreadPool threadPool; + private MockBigArrays bigArrays; + + @Before + public void setup() throws Exception { + networkService = new NetworkService(Settings.EMPTY); + threadPool = new TestThreadPool("test"); + bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()); + } + + @After + public void shutdown() throws Exception { + if (threadPool != null) { + threadPool.shutdownNow(); + } + threadPool = null; + networkService = null; + bigArrays = null; + } + + public void testCorsConfig() { + final Set methods = new HashSet<>(Arrays.asList("get", "options", "post")); + final Set headers = new HashSet<>(Arrays.asList("Content-Type", "Content-Length")); + final Settings settings = Settings.builder() + .put(SETTING_CORS_ENABLED.getKey(), true) + .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*") + .put(SETTING_CORS_ALLOW_METHODS.getKey(), Strings.collectionToCommaDelimitedString(methods)) + .put(SETTING_CORS_ALLOW_HEADERS.getKey(), Strings.collectionToCommaDelimitedString(headers)) + .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true) + .build(); + final Netty4HttpServerTransport transport = new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool); + final Netty4CorsConfig corsConfig = transport.getCorsConfig(); + assertThat(corsConfig.isAnyOriginSupported(), equalTo(true)); + assertThat(corsConfig.allowedRequestHeaders(), equalTo(headers)); + assertThat(corsConfig.allowedRequestMethods().stream().map(HttpMethod::name).collect(Collectors.toSet()), equalTo(methods)); + transport.close(); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningDisabledIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningDisabledIT.java new file mode 100644 index 00000000000..4ea46c651fc --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningDisabledIT.java @@ -0,0 +1,77 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.http.netty4; + +import io.netty.handler.codec.http.FullHttpResponse; +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Locale; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasSize; + +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) +public class Netty4PipeliningDisabledIT extends ESNetty4IntegTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.HTTP_ENABLED.getKey(), true) + .put("http.pipelining", false) + .build(); + } + + public void testThatNettyHttpServerDoesNotSupportPipelining() throws Exception { + ensureGreen(); + String[] requests = new String[] {"/", "/_nodes/stats", "/", "/_cluster/state", "/", "/_nodes", "/"}; + + HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class); + TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses(); + InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) randomFrom(boundAddresses); + + try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) { + Collection responses = nettyHttpClient.get(inetSocketTransportAddress.address(), requests); + assertThat(responses, hasSize(requests.length)); + + List opaqueIds = new ArrayList<>(Netty4HttpClient.returnOpaqueIds(responses)); + + assertResponsesOutOfOrder(opaqueIds); + } + } + + /** + * checks if all responses are there, but also tests that they are out of order because pipelining is disabled + */ + private void assertResponsesOutOfOrder(List opaqueIds) { + String message = String.format(Locale.ROOT, "Expected returned http message ids to be in any order of: %s", opaqueIds); + assertThat(message, opaqueIds, containsInAnyOrder("0", "1", "2", "3", "4", "5", "6")); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningEnabledIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningEnabledIT.java new file mode 100644 index 00000000000..c2222562d72 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningEnabledIT.java @@ -0,0 +1,76 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.handler.codec.http.FullHttpResponse; +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; + +import java.util.Collection; +import java.util.Locale; + +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; + +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) +public class Netty4PipeliningEnabledIT extends ESNetty4IntegTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.HTTP_ENABLED.getKey(), true) + .put("http.pipelining", true) + .build(); + } + + public void testThatNettyHttpServerSupportsPipelining() throws Exception { + String[] requests = new String[]{"/", "/_nodes/stats", "/", "/_cluster/state", "/"}; + + HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class); + TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses(); + InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) randomFrom(boundAddresses); + + try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) { + Collection responses = nettyHttpClient.get(inetSocketTransportAddress.address(), requests); + assertThat(responses, hasSize(5)); + + Collection opaqueIds = Netty4HttpClient.returnOpaqueIds(responses); + assertOpaqueIdsInOrder(opaqueIds); + } + } + + private void assertOpaqueIdsInOrder(Collection opaqueIds) { + // check if opaque ids are monotonically increasing + int i = 0; + String msg = String.format(Locale.ROOT, "Expected list of opaque ids to be monotonically increasing, got [%s]", opaqueIds); + for (String opaqueId : opaqueIds) { + assertThat(msg, opaqueId, is(String.valueOf(i++))); + } + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4RestIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4RestIT.java new file mode 100644 index 00000000000..6566428efa3 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4RestIT.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.parser.RestTestParseException; + +import java.io.IOException; + +public class Netty4RestIT extends ESRestTestCase { + + public Netty4RestIT(@Name("yaml") RestTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return ESRestTestCase.createParameters(0, 1); + } + +} \ No newline at end of file diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/pipelining/Netty4HttpPipeliningHandlerTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/pipelining/Netty4HttpPipeliningHandlerTests.java new file mode 100644 index 00000000000..1feb92223a3 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/pipelining/Netty4HttpPipeliningHandlerTests.java @@ -0,0 +1,262 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4.pipelining; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufUtil; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.DefaultHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.codec.http.QueryStringDecoder; +import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; +import static io.netty.handler.codec.http.HttpResponseStatus.OK; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static org.hamcrest.core.Is.is; + +public class Netty4HttpPipeliningHandlerTests extends ESTestCase { + + private ExecutorService executorService = Executors.newFixedThreadPool(randomIntBetween(4, 8)); + private Map waitingRequests = new ConcurrentHashMap<>(); + private Map finishingRequests = new ConcurrentHashMap<>(); + + @After + public void tearDown() throws Exception { + waitingRequests.keySet().forEach(this::finishRequest); + shutdownExecutorService(); + super.tearDown(); + } + + private CountDownLatch finishRequest(String url) { + waitingRequests.get(url).countDown(); + return finishingRequests.get(url); + } + + private void shutdownExecutorService() throws InterruptedException { + if (!executorService.isShutdown()) { + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); + } + } + + public void testThatPipeliningWorksWithFastSerializedRequests() throws InterruptedException { + final int numberOfRequests = randomIntBetween(2, 128); + final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(numberOfRequests), new WorkEmulatorHandler()); + + for (int i = 0; i < numberOfRequests; i++) { + embeddedChannel.writeInbound(createHttpRequest("/" + String.valueOf(i))); + } + + final List latches = new ArrayList<>(); + for (final String url : waitingRequests.keySet()) { + latches.add(finishRequest(url)); + } + + for (final CountDownLatch latch : latches) { + latch.await(); + } + + embeddedChannel.flush(); + + for (int i = 0; i < numberOfRequests; i++) { + assertReadHttpMessageHasContent(embeddedChannel, String.valueOf(i)); + } + + assertTrue(embeddedChannel.isOpen()); + } + + public void testThatPipeliningWorksWhenSlowRequestsInDifferentOrder() throws InterruptedException { + final int numberOfRequests = randomIntBetween(2, 128); + final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(numberOfRequests), new WorkEmulatorHandler()); + + for (int i = 0; i < numberOfRequests; i++) { + embeddedChannel.writeInbound(createHttpRequest("/" + String.valueOf(i))); + } + + // random order execution + final List urls = new ArrayList<>(waitingRequests.keySet()); + Randomness.shuffle(urls); + final List latches = new ArrayList<>(); + for (final String url : urls) { + latches.add(finishRequest(url)); + } + + for (final CountDownLatch latch : latches) { + latch.await(); + } + + embeddedChannel.flush(); + + for (int i = 0; i < numberOfRequests; i++) { + assertReadHttpMessageHasContent(embeddedChannel, String.valueOf(i)); + } + + assertTrue(embeddedChannel.isOpen()); + } + + public void testThatPipeliningWorksWithChunkedRequests() throws InterruptedException { + final int numberOfRequests = randomIntBetween(2, 128); + final EmbeddedChannel embeddedChannel = + new EmbeddedChannel( + new AggregateUrisAndHeadersHandler(), + new HttpPipeliningHandler(numberOfRequests), + new WorkEmulatorHandler()); + + for (int i = 0; i < numberOfRequests; i++) { + final DefaultHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/" + i); + embeddedChannel.writeInbound(request); + embeddedChannel.writeInbound(LastHttpContent.EMPTY_LAST_CONTENT); + } + + final List latches = new ArrayList<>(); + for (int i = numberOfRequests - 1; i >= 0; i--) { + latches.add(finishRequest(Integer.toString(i))); + } + + for (final CountDownLatch latch : latches) { + latch.await(); + } + + embeddedChannel.flush(); + + for (int i = 0; i < numberOfRequests; i++) { + assertReadHttpMessageHasContent(embeddedChannel, Integer.toString(i)); + } + + assertTrue(embeddedChannel.isOpen()); + } + + public void testThatPipeliningClosesConnectionWithTooManyEvents() throws InterruptedException { + final int numberOfRequests = randomIntBetween(2, 128); + final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(numberOfRequests), new WorkEmulatorHandler()); + + for (int i = 0; i < 1 + numberOfRequests + 1; i++) { + embeddedChannel.writeInbound(createHttpRequest("/" + Integer.toString(i))); + } + + final List latches = new ArrayList<>(); + final List requests = IntStream.range(1, numberOfRequests + 1).mapToObj(r -> r).collect(Collectors.toList()); + Randomness.shuffle(requests); + + for (final Integer request : requests) { + latches.add(finishRequest(request.toString())); + } + + for (final CountDownLatch latch : latches) { + latch.await(); + } + + finishRequest(Integer.toString(numberOfRequests + 1)).await(); + + embeddedChannel.flush(); + + assertFalse(embeddedChannel.isOpen()); + } + + + private void assertReadHttpMessageHasContent(EmbeddedChannel embeddedChannel, String expectedContent) { + FullHttpResponse response = (FullHttpResponse) embeddedChannel.outboundMessages().poll(); + assertNotNull("Expected response to exist, maybe you did not wait long enough?", response); + assertNotNull("Expected response to have content " + expectedContent, response.content()); + String data = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8); + assertThat(data, is(expectedContent)); + } + + private FullHttpRequest createHttpRequest(String uri) { + return new DefaultFullHttpRequest(HTTP_1_1, HttpMethod.GET, uri); + } + + private static class AggregateUrisAndHeadersHandler extends SimpleChannelInboundHandler { + + static final Queue QUEUE_URI = new LinkedTransferQueue<>(); + + @Override + protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) throws Exception { + QUEUE_URI.add(request.uri()); + } + + } + + private class WorkEmulatorHandler extends SimpleChannelInboundHandler { + + @Override + protected void channelRead0(final ChannelHandlerContext ctx, final HttpPipelinedRequest pipelinedRequest) throws Exception { + final QueryStringDecoder decoder; + if (pipelinedRequest.last() instanceof FullHttpRequest) { + final FullHttpRequest fullHttpRequest = (FullHttpRequest) pipelinedRequest.last(); + decoder = new QueryStringDecoder(fullHttpRequest.uri()); + } else { + decoder = new QueryStringDecoder(AggregateUrisAndHeadersHandler.QUEUE_URI.poll()); + } + + final String uri = decoder.path().replace("/", ""); + final ByteBuf content = Unpooled.copiedBuffer(uri, StandardCharsets.UTF_8); + final DefaultFullHttpResponse httpResponse = new DefaultFullHttpResponse(HTTP_1_1, OK, content); + httpResponse.headers().add(CONTENT_LENGTH, content.readableBytes()); + + final CountDownLatch waitingLatch = new CountDownLatch(1); + waitingRequests.put(uri, waitingLatch); + final CountDownLatch finishingLatch = new CountDownLatch(1); + finishingRequests.put(uri, finishingLatch); + + executorService.submit(() -> { + try { + waitingLatch.await(1000, TimeUnit.SECONDS); + final ChannelPromise promise = ctx.newPromise(); + ctx.write(pipelinedRequest.createHttpResponse(httpResponse, promise), promise); + finishingLatch.countDown(); + } catch (InterruptedException e) { + fail(e.toString()); + } + }); + } + + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java new file mode 100644 index 00000000000..bce875e8516 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; + +import java.io.IOException; + +public class ByteBufBytesReferenceTests extends AbstractBytesReferenceTestCase { + @Override + protected BytesReference newBytesReference(int length) throws IOException { + ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays); + for (int i = 0; i < length; i++) { + out.writeByte((byte) random().nextInt(1 << 8)); + } + assertEquals(out.size(), length); + BytesReference ref = out.bytes(); + assertEquals(ref.length(), length); + BytesRef bytesRef = ref.toBytesRef(); + final ByteBuf buffer = Unpooled.wrappedBuffer(bytesRef.bytes, bytesRef.offset, bytesRef.length); + return Netty4Utils.toBytesReference(buffer); + } + + public void testSliceOnAdvancedBuffer() throws IOException { + BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE)); + BytesRef bytesRef = bytesReference.toBytesRef(); + ByteBuf channelBuffer = Unpooled.wrappedBuffer(bytesRef.bytes, bytesRef.offset, + bytesRef.length); + int numBytesToRead = randomIntBetween(1, 5); + for (int i = 0; i < numBytesToRead; i++) { + channelBuffer.readByte(); + } + BytesReference other = Netty4Utils.toBytesReference(channelBuffer); + BytesReference slice = bytesReference.slice(numBytesToRead, bytesReference.length() - numBytesToRead); + assertEquals(other, slice); + assertEquals(other.slice(3, 1), slice.slice(3, 1)); + } + + public void testImmutable() throws IOException { + BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE)); + BytesRef bytesRef = BytesRef.deepCopyOf(bytesReference.toBytesRef()); + ByteBuf channelBuffer = Unpooled.wrappedBuffer(bytesRef.bytes, bytesRef.offset, + bytesRef.length); + ByteBufBytesReference byteBufBytesReference = new ByteBufBytesReference(channelBuffer, bytesRef.length); + assertEquals(byteBufBytesReference, bytesReference); + channelBuffer.readInt(); // this advances the index of the channel buffer + assertEquals(byteBufBytesReference, bytesReference); + assertEquals(bytesRef, byteBufBytesReference.toBytesRef()); + + BytesRef unicodeBytes = new BytesRef(randomUnicodeOfCodepointLength(100)); + channelBuffer = Unpooled.wrappedBuffer(unicodeBytes.bytes, unicodeBytes.offset, unicodeBytes.length); + byteBufBytesReference = new ByteBufBytesReference(channelBuffer, unicodeBytes.length); + String utf8ToString = byteBufBytesReference.utf8ToString(); + channelBuffer.readInt(); // this advances the index of the channel buffer + assertEquals(utf8ToString, byteBufBytesReference.utf8ToString()); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java new file mode 100644 index 00000000000..0e2ee23f950 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java @@ -0,0 +1,148 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TcpTransport; +import org.elasticsearch.transport.TransportChannel; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestHandler; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportResponseHandler; +import org.elasticsearch.transport.TransportResponseOptions; +import org.elasticsearch.transport.TransportSettings; + +import java.io.IOException; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +public class Netty4ScheduledPingTests extends ESTestCase { + public void testScheduledPing() throws Exception { + ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + Settings settings = Settings.builder() + .put(TcpTransport.PING_SCHEDULE.getKey(), "5ms") + .put(TransportSettings.PORT.getKey(), 0) + .put("cluster.name", "test") + .build(); + + CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService(); + + NamedWriteableRegistry registryA = new NamedWriteableRegistry(); + final Netty4Transport nettyA = new Netty4Transport(settings, threadPool, new NetworkService(settings), + BigArrays.NON_RECYCLING_INSTANCE, registryA, circuitBreakerService); + MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool); + serviceA.start(); + serviceA.acceptIncomingRequests(); + + NamedWriteableRegistry registryB = new NamedWriteableRegistry(); + final Netty4Transport nettyB = new Netty4Transport(settings, threadPool, new NetworkService(settings), + BigArrays.NON_RECYCLING_INSTANCE, registryB, circuitBreakerService); + MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool); + + serviceB.start(); + serviceB.acceptIncomingRequests(); + + DiscoveryNode nodeA = + new DiscoveryNode("TS_A", "TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode nodeB = + new DiscoveryNode("TS_B", "TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), Version.CURRENT); + + serviceA.connectToNode(nodeB); + serviceB.connectToNode(nodeA); + + assertBusy(new Runnable() { + @Override + public void run() { + assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(100L)); + assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(100L)); + } + }); + assertThat(nettyA.getPing().getFailedPings(), equalTo(0L)); + assertThat(nettyB.getPing().getFailedPings(), equalTo(0L)); + + serviceA.registerRequestHandler("sayHello", TransportRequest.Empty::new, ThreadPool.Names.GENERIC, + new TransportRequestHandler() { + @Override + public void messageReceived(TransportRequest.Empty request, TransportChannel channel) { + try { + channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.EMPTY); + } catch (IOException e) { + logger.error("Unexpected failure", e); + fail(e.getMessage()); + } + } + }); + + int rounds = scaledRandomIntBetween(100, 5000); + for (int i = 0; i < rounds; i++) { + serviceB.submitRequest(nodeA, "sayHello", + TransportRequest.Empty.INSTANCE, TransportRequestOptions.builder().withCompress(randomBoolean()).build(), + new TransportResponseHandler() { + @Override + public TransportResponse.Empty newInstance() { + return TransportResponse.Empty.INSTANCE; + } + + @Override + public String executor() { + return ThreadPool.Names.GENERIC; + } + + @Override + public void handleResponse(TransportResponse.Empty response) { + } + + @Override + public void handleException(TransportException exp) { + logger.error("Unexpected failure", exp); + fail("got exception instead of a response: " + exp.getMessage()); + } + }).txGet(); + } + + assertBusy(() -> { + assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(200L)); + assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(200L)); + }); + assertThat(nettyA.getPing().getFailedPings(), equalTo(0L)); + assertThat(nettyB.getPing().getFailedPings(), equalTo(0L)); + + Releasables.close(serviceA, serviceB); + terminate(threadPool); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java new file mode 100644 index 00000000000..8e7ac400f46 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java @@ -0,0 +1,107 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportSettings; +import org.junit.After; +import org.junit.Before; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.Socket; +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.Matchers.is; + +/** + * This test checks, if a HTTP look-alike request (starting with a HTTP method and a space) + * actually returns text response instead of just dropping the connection + */ +public class Netty4SizeHeaderFrameDecoderTests extends ESTestCase { + + private final Settings settings = Settings.builder() + .put("node.name", "NettySizeHeaderFrameDecoderTests") + .put(TransportSettings.BIND_HOST.getKey(), "127.0.0.1") + .put(TransportSettings.PORT.getKey(), "0") + .build(); + + private ThreadPool threadPool; + private Netty4Transport nettyTransport; + private int port; + private InetAddress host; + + @Before + public void startThreadPool() { + threadPool = new ThreadPool(settings); + NetworkService networkService = new NetworkService(settings); + BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()); + nettyTransport = new Netty4Transport(settings, threadPool, networkService, bigArrays, new NamedWriteableRegistry(), + new NoneCircuitBreakerService()); + nettyTransport.start(); + + TransportAddress[] boundAddresses = nettyTransport.boundAddress().boundAddresses(); + InetSocketTransportAddress transportAddress = (InetSocketTransportAddress) randomFrom(boundAddresses); + port = transportAddress.address().getPort(); + host = transportAddress.address().getAddress(); + } + + @After + public void terminateThreadPool() throws InterruptedException { + nettyTransport.stop(); + terminate(threadPool); + threadPool = null; + } + + public void testThatTextMessageIsReturnedOnHTTPLikeRequest() throws Exception { + String randomMethod = randomFrom("GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH"); + String data = randomMethod + " / HTTP/1.1"; + + try (Socket socket = new Socket(host, port)) { + socket.getOutputStream().write(data.getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().flush(); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))) { + assertThat(reader.readLine(), is("This is not a HTTP port")); + } + } + } + + public void testThatNothingIsReturnedForOtherInvalidPackets() throws Exception { + try (Socket socket = new Socket(host, port)) { + socket.getOutputStream().write("FOOBAR".getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().flush(); + + // end of stream + assertThat(socket.getInputStream().read(), is(-1)); + } + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java new file mode 100644 index 00000000000..b913fad933f --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java @@ -0,0 +1,124 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import io.netty.channel.Channel; +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportSettings; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) +public class Netty4TransportIT extends ESNetty4IntegTestCase { + // static so we can use it in anonymous classes + private static String channelProfileName = null; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.TRANSPORT_TYPE_KEY, "exception-throwing").build(); + } + + @Override + protected Collection> nodePlugins() { + List> list = new ArrayList<>(); + list.add(ExceptionThrowingNetty4Transport.TestPlugin.class); + list.addAll(super.nodePlugins()); + return Collections.unmodifiableCollection(list); + } + + public void testThatConnectionFailsAsIntended() throws Exception { + Client transportClient = internalCluster().transportClient(); + ClusterHealthResponse clusterIndexHealths = transportClient.admin().cluster().prepareHealth().get(); + assertThat(clusterIndexHealths.getStatus(), is(ClusterHealthStatus.GREEN)); + try { + transportClient.filterWithHeader(Collections.singletonMap("ERROR", "MY MESSAGE")).admin().cluster().prepareHealth().get(); + fail("Expected exception, but didn't happen"); + } catch (ElasticsearchException e) { + assertThat(e.getMessage(), containsString("MY MESSAGE")); + assertThat(channelProfileName, is(TransportSettings.DEFAULT_PROFILE)); + } + } + + public static final class ExceptionThrowingNetty4Transport extends Netty4Transport { + + public static class TestPlugin extends Plugin { + public void onModule(NetworkModule module) { + module.registerTransport("exception-throwing", ExceptionThrowingNetty4Transport.class); + } + } + + @Inject + public ExceptionThrowingNetty4Transport( + Settings settings, + ThreadPool threadPool, + NetworkService networkService, + BigArrays bigArrays, + NamedWriteableRegistry namedWriteableRegistry, + CircuitBreakerService circuitBreakerService) { + super(settings, threadPool, networkService, bigArrays, namedWriteableRegistry, circuitBreakerService); + } + + protected String handleRequest(Channel channel, String profileName, + StreamInput stream, long requestId, int messageLengthBytes, Version version, + InetSocketAddress remoteAddress) throws IOException { + String action = super.handleRequest(channel, profileName, stream, requestId, messageLengthBytes, version, + remoteAddress); + channelProfileName = TransportSettings.DEFAULT_PROFILE; + return action; + } + + @Override + protected void validateRequest(StreamInput buffer, long requestId, String action) + throws IOException { + super.validateRequest(buffer, requestId, action); + String error = threadPool.getThreadContext().getHeader("ERROR"); + if (error != null) { + throw new ElasticsearchException(error); + } + } + + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java new file mode 100644 index 00000000000..32268d1f5d8 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java @@ -0,0 +1,111 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.BoundTransportAddress; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.test.junit.annotations.Network; +import org.elasticsearch.transport.MockTransportClient; +import org.elasticsearch.transport.Netty4Plugin; + +import java.net.InetAddress; +import java.util.Locale; + +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1, numClientNodes = 0) +public class Netty4TransportMultiPortIntegrationIT extends ESNetty4IntegTestCase { + + private static int randomPort = -1; + private static String randomPortRange; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + if (randomPort == -1) { + randomPort = randomIntBetween(49152, 65525); + randomPortRange = String.format(Locale.ROOT, "%s-%s", randomPort, randomPort+10); + } + Settings.Builder builder = Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("network.host", "127.0.0.1") + .put("transport.profiles.client1.port", randomPortRange) + .put("transport.profiles.client1.publish_host", "127.0.0.7") + .put("transport.profiles.client1.publish_port", "4321") + .put("transport.profiles.client1.reuse_address", true); + return builder.build(); + } + + public void testThatTransportClientCanConnect() throws Exception { + Settings settings = Settings.builder() + .put("cluster.name", internalCluster().getClusterName()) + .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .build(); + try (TransportClient transportClient = new MockTransportClient(settings, Netty4Plugin.class)) { + transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), randomPort)); + ClusterHealthResponse response = transportClient.admin().cluster().prepareHealth().get(); + assertThat(response.getStatus(), is(ClusterHealthStatus.GREEN)); + } + } + + @Network + public void testThatInfosAreExposed() throws Exception { + NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().clear().setTransport(true).get(); + for (NodeInfo nodeInfo : response.getNodes()) { + assertThat(nodeInfo.getTransport().getProfileAddresses().keySet(), hasSize(1)); + assertThat(nodeInfo.getTransport().getProfileAddresses(), hasKey("client1")); + BoundTransportAddress boundTransportAddress = nodeInfo.getTransport().getProfileAddresses().get("client1"); + for (TransportAddress transportAddress : boundTransportAddress.boundAddresses()) { + assertThat(transportAddress, instanceOf(InetSocketTransportAddress.class)); + } + + // bound addresses + for (TransportAddress transportAddress : boundTransportAddress.boundAddresses()) { + assertThat(transportAddress, instanceOf(InetSocketTransportAddress.class)); + assertThat(((InetSocketTransportAddress) transportAddress).address().getPort(), + is(allOf(greaterThanOrEqualTo(randomPort), lessThanOrEqualTo(randomPort + 10)))); + } + + // publish address + assertThat(boundTransportAddress.publishAddress(), instanceOf(InetSocketTransportAddress.class)); + InetSocketTransportAddress publishAddress = (InetSocketTransportAddress) boundTransportAddress.publishAddress(); + assertThat(NetworkAddress.format(publishAddress.address().getAddress()), is("127.0.0.7")); + assertThat(publishAddress.address().getPort(), is(4321)); + } + } +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java new file mode 100644 index 00000000000..0b8e4ed85b5 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.network.NetworkUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.BoundTransportAddress; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.node.Node; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.transport.Netty4Plugin; + +import java.net.Inet4Address; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.instanceOf; + +/** + * Checks that Elasticsearch produces a sane publish_address when it binds to + * different ports on ipv4 and ipv6. + */ +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +public class Netty4TransportPublishAddressIT extends ESNetty4IntegTestCase { + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME) + .build(); + } + + public void testDifferentPorts() throws Exception { + if (!NetworkUtils.SUPPORTS_V6) { + return; + } + logger.info("--> starting a node on ipv4 only"); + Settings ipv4Settings = Settings.builder().put("network.host", "127.0.0.1").build(); + String ipv4OnlyNode = internalCluster().startNode(ipv4Settings); // should bind 127.0.0.1:XYZ + + logger.info("--> starting a node on ipv4 and ipv6"); + Settings bothSettings = Settings.builder().put("network.host", "_local_").build(); + internalCluster().startNode(bothSettings); // should bind [::1]:XYZ and 127.0.0.1:XYZ+1 + + logger.info("--> waiting for the cluster to declare itself stable"); + ensureStableCluster(2); // fails if port of publish address does not match corresponding bound address + + logger.info("--> checking if boundAddress matching publishAddress has same port"); + NodesInfoResponse nodesInfoResponse = client().admin().cluster().prepareNodesInfo().get(); + for (NodeInfo nodeInfo : nodesInfoResponse.getNodes()) { + BoundTransportAddress boundTransportAddress = nodeInfo.getTransport().getAddress(); + if (nodeInfo.getNode().getName().equals(ipv4OnlyNode)) { + assertThat(boundTransportAddress.boundAddresses().length, equalTo(1)); + assertThat(boundTransportAddress.boundAddresses()[0].getPort(), equalTo(boundTransportAddress.publishAddress().getPort())); + } else { + assertThat(boundTransportAddress.boundAddresses().length, greaterThan(1)); + for (TransportAddress boundAddress : boundTransportAddress.boundAddresses()) { + assertThat(boundAddress, instanceOf(InetSocketTransportAddress.class)); + InetSocketTransportAddress inetBoundAddress = (InetSocketTransportAddress) boundAddress; + if (inetBoundAddress.address().getAddress() instanceof Inet4Address) { + // IPv4 address is preferred publish address for _local_ + assertThat(inetBoundAddress.getPort(), equalTo(boundTransportAddress.publishAddress().getPort())); + } + } + } + } + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java new file mode 100644 index 00000000000..8372a8540b8 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.CompositeByteBuf; +import io.netty.buffer.Unpooled; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +public class Netty4UtilsTests extends ESTestCase { + + private static final int PAGE_SIZE = BigArrays.BYTE_PAGE_SIZE; + private final BigArrays bigarrays = new BigArrays(null, new NoneCircuitBreakerService(), false); + + public void testToChannelBufferWithEmptyRef() throws IOException { + ByteBuf buffer = Netty4Utils.toByteBuf(getRandomizedBytesReference(0)); + assertSame(Unpooled.EMPTY_BUFFER, buffer); + } + + public void testToChannelBufferWithSlice() throws IOException { + BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE)); + int sliceOffset = randomIntBetween(0, ref.length()); + int sliceLength = randomIntBetween(ref.length() - sliceOffset, ref.length() - sliceOffset); + BytesReference slice = ref.slice(sliceOffset, sliceLength); + ByteBuf buffer = Netty4Utils.toByteBuf(slice); + BytesReference bytesReference = Netty4Utils.toBytesReference(buffer); + assertArrayEquals(BytesReference.toBytes(slice), BytesReference.toBytes(bytesReference)); + } + + public void testToChannelBufferWithSliceAfter() throws IOException { + BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE)); + int sliceOffset = randomIntBetween(0, ref.length()); + int sliceLength = randomIntBetween(ref.length() - sliceOffset, ref.length() - sliceOffset); + ByteBuf buffer = Netty4Utils.toByteBuf(ref); + BytesReference bytesReference = Netty4Utils.toBytesReference(buffer); + assertArrayEquals(BytesReference.toBytes(ref.slice(sliceOffset, sliceLength)), + BytesReference.toBytes(bytesReference.slice(sliceOffset, sliceLength))); + } + + public void testToChannelBuffer() throws IOException { + BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE)); + ByteBuf buffer = Netty4Utils.toByteBuf(ref); + BytesReference bytesReference = Netty4Utils.toBytesReference(buffer); + if (ref instanceof ByteBufBytesReference) { + assertEquals(buffer, ((ByteBufBytesReference) ref).toByteBuf()); + } else if (AbstractBytesReferenceTestCase.getNumPages(ref) > 1) { // we gather the buffers into a channel buffer + assertTrue(buffer instanceof CompositeByteBuf); + } + assertArrayEquals(BytesReference.toBytes(ref), BytesReference.toBytes(bytesReference)); + } + + private BytesReference getRandomizedBytesReference(int length) throws IOException { + // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content + ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays); + for (int i = 0; i < length; i++) { + out.writeByte((byte) random().nextInt(1 << 8)); + } + assertEquals(out.size(), length); + BytesReference ref = out.bytes(); + assertEquals(ref.length(), length); + if (randomBoolean()) { + return new BytesArray(ref.toBytesRef()); + } else if (randomBoolean()) { + BytesRef bytesRef = ref.toBytesRef(); + return Netty4Utils.toBytesReference(Unpooled.wrappedBuffer(bytesRef.bytes, bytesRef.offset, + bytesRef.length)); + } else { + return ref; + } + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java new file mode 100644 index 00000000000..e76431daf8d --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java @@ -0,0 +1,146 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import org.elasticsearch.common.component.Lifecycle; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.network.NetworkUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TcpTransport; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.transport.TransportSettings; +import org.junit.Before; + +import static org.hamcrest.Matchers.is; + +public class NettyTransportMultiPortTests extends ESTestCase { + + private String host; + + @Before + public void setup() { + if (NetworkUtils.SUPPORTS_V6 && randomBoolean()) { + host = "::1"; + } else { + host = "127.0.0.1"; + } + } + + public void testThatNettyCanBindToMultiplePorts() throws Exception { + Settings settings = Settings.builder() + .put("network.host", host) + .put(TransportSettings.PORT.getKey(), 22) // will not actually bind to this + .put("transport.profiles.default.port", 0) + .put("transport.profiles.client1.port", 0) + .build(); + + ThreadPool threadPool = new TestThreadPool("tst"); + try (TcpTransport transport = startTransport(settings, threadPool)) { + assertEquals(1, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); + } finally { + terminate(threadPool); + } + } + + public void testThatDefaultProfileInheritsFromStandardSettings() throws Exception { + Settings settings = Settings.builder() + .put("network.host", host) + .put(TransportSettings.PORT.getKey(), 0) + .put("transport.profiles.client1.port", 0) + .build(); + + ThreadPool threadPool = new TestThreadPool("tst"); + try (TcpTransport transport = startTransport(settings, threadPool)) { + assertEquals(1, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); + } finally { + terminate(threadPool); + } + } + + public void testThatProfileWithoutPortSettingsFails() throws Exception { + + Settings settings = Settings.builder() + .put("network.host", host) + .put(TransportSettings.PORT.getKey(), 0) + .put("transport.profiles.client1.whatever", "foo") + .build(); + + ThreadPool threadPool = new TestThreadPool("tst"); + try (TcpTransport transport = startTransport(settings, threadPool)) { + assertEquals(0, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); + } finally { + terminate(threadPool); + } + } + + public void testThatDefaultProfilePortOverridesGeneralConfiguration() throws Exception { + Settings settings = Settings.builder() + .put("network.host", host) + .put(TransportSettings.PORT.getKey(), 22) // will not actually bind to this + .put("transport.profiles.default.port", 0) + .build(); + + ThreadPool threadPool = new TestThreadPool("tst"); + try (TcpTransport transport = startTransport(settings, threadPool)) { + assertEquals(0, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); + } finally { + terminate(threadPool); + } + } + + public void testThatProfileWithoutValidNameIsIgnored() throws Exception { + Settings settings = Settings.builder() + .put("network.host", host) + .put(TransportSettings.PORT.getKey(), 0) + // mimics someone trying to define a profile for .local which is the profile for a node request to itself + .put("transport.profiles." + TransportService.DIRECT_RESPONSE_PROFILE + ".port", 22) // will not actually bind to this + .put("transport.profiles..port", 23) // will not actually bind to this + .build(); + + ThreadPool threadPool = new TestThreadPool("tst"); + try (TcpTransport transport = startTransport(settings, threadPool)) { + assertEquals(0, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); + } finally { + terminate(threadPool); + } + } + + private TcpTransport startTransport(Settings settings, ThreadPool threadPool) { + BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()); + TcpTransport transport = new Netty4Transport(settings, threadPool, new NetworkService(settings), bigArrays, + new NamedWriteableRegistry(), new NoneCircuitBreakerService()); + transport.start(); + + assertThat(transport.lifecycleState(), is(Lifecycle.State.STARTED)); + return transport; + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java new file mode 100644 index 00000000000..bafec9d2ffa --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.AbstractSimpleTransportTestCase; +import org.elasticsearch.transport.ConnectTransportException; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportSettings; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.hamcrest.Matchers.containsString; + +public class SimpleNetty4TransportTests extends AbstractSimpleTransportTestCase { + + public static MockTransportService nettyFromThreadPool( + Settings settings, + ThreadPool threadPool, final Version version) { + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); + Transport transport = new Netty4Transport(settings, threadPool, new NetworkService(settings), BigArrays.NON_RECYCLING_INSTANCE, + namedWriteableRegistry, new NoneCircuitBreakerService()) { + @Override + protected Version getCurrentVersion() { + return version; + } + }; + return new MockTransportService(Settings.EMPTY, transport, threadPool); + } + + @Override + protected MockTransportService build(Settings settings, Version version) { + settings = Settings.builder().put(settings).put(TransportSettings.PORT.getKey(), "0").build(); + MockTransportService transportService = nettyFromThreadPool(settings, threadPool, version); + transportService.start(); + return transportService; + } + + public void testConnectException() throws UnknownHostException { + try { + serviceA.connectToNode(new DiscoveryNode("C", new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9876), + emptyMap(), emptySet(),Version.CURRENT)); + fail("Expected ConnectTransportException"); + } catch (ConnectTransportException e) { + assertThat(e.getMessage(), containsString("connect_timeout")); + assertThat(e.getMessage(), containsString("[127.0.0.1:9876]")); + } + } + +} diff --git a/modules/transport-netty4/src/test/resources/rest-api-spec/test/10_basic.yaml b/modules/transport-netty4/src/test/resources/rest-api-spec/test/10_basic.yaml new file mode 100644 index 00000000000..a7beac1491f --- /dev/null +++ b/modules/transport-netty4/src/test/resources/rest-api-spec/test/10_basic.yaml @@ -0,0 +1,13 @@ +# Integration tests for Netty transport +# +"Netty loaded": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - match: { nodes.$master.modules.1.name: transport-netty4 } diff --git a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java index 5f54d14eca2..96c1139c790 100644 --- a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java +++ b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.transport.MockTcpTransportPlugin; +import org.elasticsearch.transport.Netty4Plugin; import org.elasticsearch.transport.client.PreBuiltTransportClient; import org.junit.After; import org.junit.AfterClass; @@ -49,6 +50,8 @@ import java.util.Locale; import java.util.concurrent.atomic.AtomicInteger; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength; +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean; +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween; import static org.hamcrest.Matchers.notNullValue; /** @@ -82,13 +85,22 @@ public abstract class ESSmokeClientTestCase extends LuceneTestCase { .put("client.transport.ignore_cluster_name", true) .put(Environment.PATH_HOME_SETTING.getKey(), tempDir); final Collection> plugins; - if (random().nextBoolean()) { - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME); - plugins = Collections.singleton(MockTcpTransportPlugin.class); - } else { - plugins = Collections.emptyList(); + switch (randomIntBetween(0, 2)) { + case 0: + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME); + plugins = Collections.singleton(MockTcpTransportPlugin.class); + break; + case 1: + plugins = Collections.emptyList(); + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME); + break; + case 2: + plugins = Collections.emptyList(); + break; + default: + throw new AssertionError(); } - TransportClient client = new PreBuiltTransportClient(builder.build(), plugins).addTransportAddresses(transportAddresses); + TransportClient client = new PreBuiltTransportClient(builder.build(), plugins).addTransportAddresses(transportAddresses); logger.info("--> Elasticsearch Java TransportClient started"); diff --git a/qa/smoke-test-http/build.gradle b/qa/smoke-test-http/build.gradle index 7fa3205537a..038115c2d24 100644 --- a/qa/smoke-test-http/build.gradle +++ b/qa/smoke-test-http/build.gradle @@ -21,4 +21,5 @@ apply plugin: 'elasticsearch.rest-test' dependencies { testCompile project(path: ':modules:transport-netty3', configuration: 'runtime') // for http + testCompile project(path: ':modules:transport-netty4', configuration: 'runtime') // for http } \ No newline at end of file diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpCompressionIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpCompressionIT.java index e22b2e187ea..6e548dd2ca6 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpCompressionIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpCompressionIT.java @@ -65,4 +65,5 @@ public class HttpCompressionIT extends ESIntegTestCase { assertNull(response.getHeader(HttpHeaders.CONTENT_ENCODING)); } } + } diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java index 636d652feb1..03a7ba68a71 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java @@ -25,6 +25,8 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.transport.MockTcpTransportPlugin; import org.elasticsearch.transport.Netty3Plugin; +import org.elasticsearch.transport.Netty4Plugin; +import org.junit.BeforeClass; import java.util.Collection; import java.util.Collections; @@ -32,33 +34,56 @@ import java.util.List; public abstract class HttpSmokeTestCase extends ESIntegTestCase { + private static String nodeTransportTypeKey; + private static String nodeHttpTypeKey; + private static String clientTypeKey; + + @SuppressWarnings("unchecked") + @BeforeClass + public static void setUpTransport() { + nodeTransportTypeKey = getTypeKey(randomFrom(MockTcpTransportPlugin.class, Netty3Plugin.class, Netty4Plugin.class)); + nodeHttpTypeKey = getTypeKey(randomFrom(Netty3Plugin.class, Netty4Plugin.class)); + clientTypeKey = getTypeKey(randomFrom(MockTcpTransportPlugin.class, Netty3Plugin.class, Netty4Plugin.class)); + } + + private static String getTypeKey(Class clazz) { + if (clazz.equals(MockTcpTransportPlugin.class)) { + return MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME; + } else if (clazz.equals(Netty3Plugin.class)) { + return Netty3Plugin.NETTY_TRANSPORT_NAME; + } else { + assert clazz.equals(Netty4Plugin.class); + return Netty4Plugin.NETTY_TRANSPORT_NAME; + } + } + @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put("netty.assert.buglevel", false) - .put(NetworkModule.TRANSPORT_TYPE_KEY, randomFrom(Netty3Plugin.NETTY_TRANSPORT_NAME, - MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME)) - .put(NetworkModule.HTTP_ENABLED.getKey(), true).build(); + .put(super.nodeSettings(nodeOrdinal)) + .put("netty.assert.buglevel", false) + .put(NetworkModule.TRANSPORT_TYPE_KEY, nodeTransportTypeKey) + .put(NetworkModule.HTTP_TYPE_KEY, nodeHttpTypeKey) + .put(NetworkModule.HTTP_ENABLED.getKey(), true).build(); } @Override protected Collection> nodePlugins() { - return pluginList(MockTcpTransportPlugin.class, Netty3Plugin.class, BogusPlugin.class); + return pluginList(MockTcpTransportPlugin.class, Netty3Plugin.class, Netty4Plugin.class, BogusPlugin.class); } @Override protected Collection> transportClientPlugins() { - return pluginList(MockTcpTransportPlugin.class, Netty3Plugin.class, BogusPlugin.class); + return pluginList(MockTcpTransportPlugin.class, Netty3Plugin.class, Netty4Plugin.class, BogusPlugin.class); } @Override protected Settings transportClientSettings() { return Settings.builder() - .put(super.transportClientSettings()) - .put("netty.assert.buglevel", false) - .put(NetworkModule.TRANSPORT_TYPE_KEY, randomFrom(Netty3Plugin.NETTY_TRANSPORT_NAME, - MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME)).build(); + .put(super.transportClientSettings()) + .put("netty.assert.buglevel", false) + .put(NetworkModule.TRANSPORT_TYPE_KEY, clientTypeKey) + .build(); } @Override @@ -66,15 +91,18 @@ public abstract class HttpSmokeTestCase extends ESIntegTestCase { return true; } - public static final class BogusPlugin extends Plugin { - // see Netty3Plugin.... this runs without the permission from the netty3 module so it will fail since reindex can't set the property + + // this runs without the permission from the netty modules so it will fail since reindex can't set the property // to make it still work we disable that check but need to register the setting first - private static final Setting ASSERT_NETTY_BUGLEVEL = Setting.boolSetting("netty.assert.buglevel", true, - Setting.Property.NodeScope); + private static final Setting ASSERT_NETTY_BUGLEVEL = + Setting.boolSetting("netty.assert.buglevel", true, Setting.Property.NodeScope); + @Override public List> getSettings() { return Collections.singletonList(ASSERT_NETTY_BUGLEVEL); } + } + } diff --git a/settings.gradle b/settings.gradle index e070e761433..d04e4b233a6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -26,6 +26,7 @@ List projects = [ 'modules:lang-mustache', 'modules:lang-painless', 'modules:transport-netty3', + 'modules:transport-netty4', 'modules:reindex', 'modules:percolator', 'plugins:analysis-icu', diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java index 3414e9eda10..16647b04a47 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java @@ -242,7 +242,7 @@ public abstract class ESBackcompatTestCase extends ESIntegTestCase { protected Settings commonNodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder().put(requiredSettings()); - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, "netty3"); // run same transport / disco as external + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, randomBoolean() ? "netty3" : "netty4"); // run same transport / disco as external builder.put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "zen"); return builder.build(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java index b5a5778c1e5..cde4e5f6ac9 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.logging.ESLogger; @@ -51,7 +52,7 @@ final class ExternalNode implements Closeable { public static final Settings REQUIRED_SETTINGS = Settings.builder() .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "zen") - .put(NetworkModule.TRANSPORT_TYPE_KEY, "netty3").build(); // we need network mode for this + .put(NetworkModule.TRANSPORT_TYPE_KEY, Randomness.get().nextBoolean() ? "netty3" : "netty4").build(); // we need network mode for this private final Path path; private final Random random; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 16606e9fa91..bc16f4f7477 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -421,4 +421,5 @@ public abstract class ESRestTestCase extends ESTestCase { } return runningTasks; } + } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java index 3fc924176f0..f291e0bbeec 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java @@ -28,25 +28,19 @@ import java.util.Map; public class FakeRestRequest extends RestRequest { private final Map headers; - - private final Map params; - private final BytesReference content; - private final Method method; - private final String path; public FakeRestRequest() { this(new HashMap<>(), new HashMap<>(), null, Method.GET, "/"); } private FakeRestRequest(Map headers, Map params, BytesReference content, Method method, String path) { + super(params, path); this.headers = headers; - this.params = params; this.content = content; this.method = method; - this.path = path; } @Override @@ -56,12 +50,7 @@ public class FakeRestRequest extends RestRequest { @Override public String uri() { - return path; - } - - @Override - public String rawPath() { - return path; + return rawPath(); } @Override @@ -84,31 +73,8 @@ public class FakeRestRequest extends RestRequest { return headers.entrySet(); } - @Override - public boolean hasParam(String key) { - return params.containsKey(key); - } - - @Override - public String param(String key) { - return params.get(key); - } - - @Override - public String param(String key, String defaultValue) { - String value = params.get(key); - if (value == null) { - return defaultValue; - } - return value; - } - - @Override - public Map params() { - return params; - } - public static class Builder { + private Map headers = new HashMap<>(); private Map params = new HashMap<>(); @@ -147,6 +113,7 @@ public class FakeRestRequest extends RestRequest { public FakeRestRequest build() { return new FakeRestRequest(headers, params, content, method, path); } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java index eb4613f9e12..44a2bbd2166 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java @@ -43,6 +43,7 @@ import java.io.BufferedOutputStream; import java.io.Closeable; import java.io.IOException; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; @@ -146,8 +147,8 @@ public class MockTcpTransport extends TcpTransport output.write(minimalHeader); output.writeInt(msgSize); output.write(buffer); - BytesReference bytes = output.bytes(); - if (validateMessageHeader(bytes)) { + final BytesReference bytes = output.bytes(); + if (TcpTransport.validateMessageHeader(bytes)) { InetSocketAddress remoteAddress = (InetSocketAddress) socket.getRemoteSocketAddress(); messageReceived(bytes.slice(TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE, msgSize), mockChannel, mockChannel.profile, remoteAddress, msgSize); @@ -349,5 +350,6 @@ public class MockTcpTransport extends TcpTransport protected Version getCurrentVersion() { return mockVersion; } + } diff --git a/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java b/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java index a198ef77956..2b0f551dbb7 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java @@ -37,4 +37,5 @@ public class MockTransportClient extends TransportClient { public MockTransportClient(Settings settings, Collection> plugins) { super(settings, DEFAULT_SETTINGS, plugins); } + }