honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove `modules/transport_netty_3` in favor of `netty_4` (#21590) 

We kept `netty_3` as a fallback in the 5.x series but now that master
is 6.0 we don't need this or in other words all issues coming up with
netty 4 will be blockers for 6.0.
This commit is contained in:
Simon Willnauer 2016-11-17 12:44:42 +01:00 committed by GitHub
parent 116593e5f5
commit de04aad994
60 changed files with 15 additions and 6901 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
client/benchmark/build.gradle
View File

@ -55,7 +55,6 @@ dependencies {
// for transport client
compile("org.elasticsearch:elasticsearch:${version}")
compile("org.elasticsearch.client:transport:${version}")
compile project(path: ':modules:transport-netty3', configuration: 'runtime')
compile project(path: ':modules:transport-netty4', configuration: 'runtime')
compile project(path: ':modules:reindex', configuration: 'runtime')
compile project(path: ':modules:lang-mustache', configuration: 'runtime')

1
client/transport/build.gradle
View File

@ -27,7 +27,6 @@ group = 'org.elasticsearch.client'
dependencies {
compile "org.elasticsearch:elasticsearch:${version}"
compile "org.elasticsearch.plugin:transport-netty3-client:${version}"
compile "org.elasticsearch.plugin:transport-netty4-client:${version}"
compile "org.elasticsearch.plugin:reindex-client:${version}"
compile "org.elasticsearch.plugin:lang-mustache-client:${version}"

5
client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
View File

@ -23,25 +23,21 @@ import io.netty.util.ThreadDeathWatcher;
import io.netty.util.concurrent.GlobalEventExecutor;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.reindex.ReindexPlugin;
import org.elasticsearch.percolator.PercolatorPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.mustache.MustachePlugin;
import org.elasticsearch.transport.Netty3Plugin;
import org.elasticsearch.transport.Netty4Plugin;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
/**
* A builder to create an instance of {@link TransportClient}
* This class pre-installs the
* {@link Netty3Plugin},
* {@link Netty4Plugin},
* {@link ReindexPlugin},
* {@link PercolatorPlugin},
@ -54,7 +50,6 @@ public class PreBuiltTransportClient extends TransportClient {
private static final Collection<Class<? extends Plugin>> PRE_INSTALLED_PLUGINS =
Collections.unmodifiableList(
Arrays.asList(
Netty3Plugin.class,
Netty4Plugin.class,
ReindexPlugin.class,
PercolatorPlugin.class,

1
modules/reindex/build.gradle
View File

@ -38,7 +38,6 @@ run {
dependencies {
compile "org.elasticsearch.client:rest:${version}"
// for http - testing reindex from remote
testCompile project(path: ':modules:transport-netty3', configuration: 'runtime')
testCompile project(path: ':modules:transport-netty4', configuration: 'runtime')
}

8
modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
View File

@ -35,7 +35,6 @@ import org.elasticsearch.index.reindex.remote.RemoteInfo;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.Netty3Plugin;
import org.elasticsearch.transport.Netty4Plugin;
import org.junit.After;
import org.junit.Before;
@ -61,12 +60,10 @@ public class RetryTests extends ESSingleNodeTestCase {
private List<CyclicBarrier> blockedExecutors = new ArrayList<>();
private boolean useNetty3;
@Before
public void setUp() throws Exception {
super.setUp();
useNetty3 = randomBoolean();
createIndex("source");
// Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools.
BulkRequestBuilder bulk = client().prepareBulk();
@ -90,7 +87,6 @@ public class RetryTests extends ESSingleNodeTestCase {
protected Collection<Class<? extends Plugin>> getPlugins() {
return pluginList(
ReindexPlugin.class,
Netty3Plugin.class,
Netty4Plugin.class);
}
@ -110,10 +106,6 @@ public class RetryTests extends ESSingleNodeTestCase {
settings.put(NetworkModule.HTTP_ENABLED.getKey(), true);
// Whitelist reindexing from the http host we're going to use
settings.put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "127.0.0.1:*");
if (useNetty3) {
settings.put(NetworkModule.HTTP_TYPE_KEY, Netty3Plugin.NETTY_HTTP_TRANSPORT_NAME);
settings.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME);
}
return settings.build();
}

127
modules/transport-netty3/build.gradle
View File

@ -1,127 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/*
TODOs:
* fix permissions such that only netty3 can open sockets etc?
* fix the hack in the build framework that copies transport-netty3 into the integ test cluster
* maybe figure out a way to run all tests from core with netty3/network?
*/
esplugin {
description 'Netty 3 based transport implementation'
classname 'org.elasticsearch.transport.Netty3Plugin'
hasClientJar = true
}
compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
dependencies {
// network stack
compile 'io.netty:netty:3.10.6.Final'
}
integTest {
includePackaged = true
cluster {
setting 'http.type', 'netty3'
setting 'transport.type', 'netty3'
numNodes = 2
}
}
thirdPartyAudit.excludes = [
// uses internal java api: sun.security.x509 (X509CertInfo, X509CertImpl, X500Name)
'org.jboss.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
// classes are missing
// from org.jboss.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder (netty)
'com.google.protobuf.CodedInputStream',
// from org.jboss.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender (netty)
'com.google.protobuf.CodedOutputStream',
// from org.jboss.netty.handler.codec.protobuf.ProtobufDecoder (netty)
'com.google.protobuf.ExtensionRegistry',
'com.google.protobuf.MessageLite$Builder',
'com.google.protobuf.MessageLite',
'com.google.protobuf.Parser',
// from org.jboss.netty.channel.socket.http.HttpTunnelingServlet (netty)
'javax.servlet.ServletConfig',
'javax.servlet.ServletException',
'javax.servlet.ServletOutputStream',
'javax.servlet.http.HttpServlet',
'javax.servlet.http.HttpServletRequest',
'javax.servlet.http.HttpServletResponse',
// from org.jboss.netty.logging.CommonsLoggerFactory (netty)
'org.apache.commons.logging.Log',
'org.apache.commons.logging.LogFactory',
// from org.jboss.netty.handler.ssl.OpenSslEngine (netty)
'org.apache.tomcat.jni.Buffer',
'org.apache.tomcat.jni.Library',
'org.apache.tomcat.jni.Pool',
'org.apache.tomcat.jni.SSL',
'org.apache.tomcat.jni.SSLContext',
// from org.jboss.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty)
'org.bouncycastle.asn1.x500.X500Name',
'org.bouncycastle.cert.X509v3CertificateBuilder',
'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
'org.bouncycastle.jce.provider.BouncyCastleProvider',
'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
// from org.jboss.netty.handler.ssl.JettyNpnSslEngine (netty)
'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',
'org.eclipse.jetty.npn.NextProtoNego$ServerProvider',
'org.eclipse.jetty.npn.NextProtoNego',
// from org.jboss.netty.logging.JBossLoggerFactory (netty)
'org.jboss.logging.Logger',
// from org.jboss.netty.handler.codec.marshalling.ChannelBufferByteInput (netty)
'org.jboss.marshalling.ByteInput',
// from org.jboss.netty.handler.codec.marshalling.ChannelBufferByteOutput (netty)
'org.jboss.marshalling.ByteOutput',
// from org.jboss.netty.handler.codec.marshalling.CompatibleMarshallingEncoder (netty)
'org.jboss.marshalling.Marshaller',
// from org.jboss.netty.handler.codec.marshalling.ContextBoundUnmarshallerProvider (netty)
'org.jboss.marshalling.MarshallerFactory',
'org.jboss.marshalling.MarshallingConfiguration',
'org.jboss.marshalling.Unmarshaller',
// from org.jboss.netty.container.osgi.NettyBundleActivator (netty)
'org.osgi.framework.BundleActivator',
'org.osgi.framework.BundleContext',
// from org.jboss.netty.logging.OsgiLoggerFactory$1 (netty)
'org.osgi.framework.ServiceReference',
'org.osgi.service.log.LogService',
'org.osgi.util.tracker.ServiceTracker',
'org.osgi.util.tracker.ServiceTrackerCustomizer',
// from org.jboss.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional
'org.slf4j.Logger',
'org.slf4j.LoggerFactory'
]

1
modules/transport-netty3/licenses/netty-3.10.6.Final.jar.sha1
View File

@ -1 +0,0 @@
18ed04a0e502896552854926e908509db2987a00

202
modules/transport-netty3/licenses/netty-LICENSE.txt
View File

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

116
modules/transport-netty3/licenses/netty-NOTICE.txt
View File

@ -1,116 +0,0 @@
The Netty Project
=================
Please visit the Netty web site for more information:
* http://netty.io/
Copyright 2011 The Netty Project
The Netty Project licenses this file to you under the Apache License,
version 2.0 (the "License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at:
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Also, please refer to each LICENSE.<component>.txt file, which is located in
the 'license' directory of the distribution file, for the license terms of the
components that this product depends on.
-------------------------------------------------------------------------------
This product contains the extensions to Java Collections Framework which has
been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
* LICENSE:
* license/LICENSE.jsr166y.txt (Public Domain)
* HOMEPAGE:
* http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
* http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
This product contains a modified version of Robert Harder's Public Domain
Base64 Encoder and Decoder, which can be obtained at:
* LICENSE:
* license/LICENSE.base64.txt (Public Domain)
* HOMEPAGE:
* http://iharder.sourceforge.net/current/java/base64/
This product contains a modified version of 'JZlib', a re-implementation of
zlib in pure Java, which can be obtained at:
* LICENSE:
* license/LICENSE.jzlib.txt (BSD Style License)
* HOMEPAGE:
* http://www.jcraft.com/jzlib/
This product contains a modified version of 'Webbit', a Java event based
WebSocket and HTTP server:
* LICENSE:
* license/LICENSE.webbit.txt (BSD License)
* HOMEPAGE:
* https://github.com/joewalnes/webbit
This product optionally depends on 'Protocol Buffers', Google's data
interchange format, which can be obtained at:
* LICENSE:
* license/LICENSE.protobuf.txt (New BSD License)
* HOMEPAGE:
* http://code.google.com/p/protobuf/
This product optionally depends on 'Bouncy Castle Crypto APIs' to generate
a temporary self-signed X.509 certificate when the JVM does not provide the
equivalent functionality. It can be obtained at:
* LICENSE:
* license/LICENSE.bouncycastle.txt (MIT License)
* HOMEPAGE:
* http://www.bouncycastle.org/
This product optionally depends on 'SLF4J', a simple logging facade for Java,
which can be obtained at:
* LICENSE:
* license/LICENSE.slf4j.txt (MIT License)
* HOMEPAGE:
* http://www.slf4j.org/
This product optionally depends on 'Apache Commons Logging', a logging
framework, which can be obtained at:
* LICENSE:
* license/LICENSE.commons-logging.txt (Apache License 2.0)
* HOMEPAGE:
* http://commons.apache.org/logging/
This product optionally depends on 'Apache Log4J', a logging framework,
which can be obtained at:
* LICENSE:
* license/LICENSE.log4j.txt (Apache License 2.0)
* HOMEPAGE:
* http://logging.apache.org/log4j/
This product optionally depends on 'JBoss Logging', a logging framework,
which can be obtained at:
* LICENSE:
* license/LICENSE.jboss-logging.txt (GNU LGPL 2.1)
* HOMEPAGE:
* http://anonsvn.jboss.org/repos/common/common-logging-spi/
This product optionally depends on 'Apache Felix', an open source OSGi
framework implementation, which can be obtained at:
* LICENSE:
* license/LICENSE.felix.txt (Apache License 2.0)
* HOMEPAGE:
* http://felix.apache.org/

52
modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/ESNetty3HttpResponseEncoder.java
View File

@ -1,52 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import org.elasticsearch.transport.netty3.Netty3Utils;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.buffer.CompositeChannelBuffer;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.handler.codec.http.HttpResponseEncoder;
import java.util.List;
/**
* Wraps a netty {@link HttpResponseEncoder} and makes sure that if the resulting
* channel buffer is composite, it will use the correct gathering flag. See more
* at {@link Netty3Utils#DEFAULT_GATHERING}.
*/
public class ESNetty3HttpResponseEncoder extends HttpResponseEncoder {
@Override
protected Object encode(ChannelHandlerContext ctx, Channel channel, Object msg) throws Exception {
Object retVal = super.encode(ctx, channel, msg);
if (retVal instanceof CompositeChannelBuffer) {
CompositeChannelBuffer ccb = (CompositeChannelBuffer) retVal;
if (ccb.useGathering() != Netty3Utils.DEFAULT_GATHERING) {
List<ChannelBuffer> decompose = ccb.decompose(ccb.readerIndex(), ccb.readableBytes());
return ChannelBuffers.wrappedBuffer(Netty3Utils.DEFAULT_GATHERING,
decompose.toArray(new ChannelBuffer[decompose.size()]));
}
}
return retVal;
}
}

279
modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpChannel.java
View File

@ -1,279 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.transport.netty3.Netty3Utils;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.http.netty3.cors.Netty3CorsHandler;
import org.elasticsearch.http.netty3.pipelining.OrderedDownstreamChannelEvent;
import org.elasticsearch.http.netty3.pipelining.OrderedUpstreamMessageEvent;
import org.elasticsearch.rest.AbstractRestChannel;
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.RestStatus;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.handler.codec.http.Cookie;
import org.jboss.netty.handler.codec.http.CookieDecoder;
import org.jboss.netty.handler.codec.http.CookieEncoder;
import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
import org.jboss.netty.handler.codec.http.HttpHeaders;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpResponse;
import org.jboss.netty.handler.codec.http.HttpResponseStatus;
import org.jboss.netty.handler.codec.http.HttpVersion;
import java.util.Collections;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Values.KEEP_ALIVE;
public final class Netty3HttpChannel extends AbstractRestChannel {
private final Netty3HttpServerTransport transport;
private final Channel channel;
private final org.jboss.netty.handler.codec.http.HttpRequest nettyRequest;
private final OrderedUpstreamMessageEvent orderedUpstreamMessageEvent;
private final ThreadContext threadContext;
/**
* @param transport The corresponding <code>Netty3HttpServerTransport</code> where this channel belongs to.
* @param request The request that is handled by this channel.
* @param orderedUpstreamMessageEvent If HTTP pipelining is enabled provide the corresponding Netty upstream event. May be null if
* HTTP pipelining is disabled.
* @param detailedErrorsEnabled true iff error messages should include stack traces.
*/
public Netty3HttpChannel(Netty3HttpServerTransport transport, Netty3HttpRequest request,
@Nullable OrderedUpstreamMessageEvent orderedUpstreamMessageEvent,
boolean detailedErrorsEnabled, ThreadContext threadContext) {
super(request, detailedErrorsEnabled);
this.transport = transport;
this.channel = request.getChannel();
this.nettyRequest = request.request();
this.orderedUpstreamMessageEvent = orderedUpstreamMessageEvent;
this.threadContext = threadContext;
}
@Override
public BytesStreamOutput newBytesOutput() {
return new ReleasableBytesStreamOutput(transport.bigArrays);
}
@Override
public void sendResponse(RestResponse response) {
// if the response object was created upstream, then use it;
// otherwise, create a new one
HttpResponse resp = newResponse();
resp.setStatus(getStatus(response.status()));
Netty3CorsHandler.setCorsResponseHeaders(nettyRequest, resp, transport.getCorsConfig());
String opaque = nettyRequest.headers().get("X-Opaque-Id");
if (opaque != null) {
setHeaderField(resp, "X-Opaque-Id", opaque);
}
// Add all custom headers
addCustomHeaders(resp, response.getHeaders());
addCustomHeaders(resp, threadContext.getResponseHeaders());
BytesReference content = response.content();
ChannelBuffer buffer;
boolean addedReleaseListener = false;
try {
buffer = Netty3Utils.toChannelBuffer(content);
if (HttpMethod.HEAD.equals(nettyRequest.getMethod())) {
resp.setContent(ChannelBuffers.EMPTY_BUFFER);
} else {
resp.setContent(buffer);
}
// If our response doesn't specify a content-type header, set one
setHeaderField(resp, HttpHeaders.Names.CONTENT_TYPE, response.contentType(), false);
// If our response has no content-length, calculate and set one
setHeaderField(resp, HttpHeaders.Names.CONTENT_LENGTH, String.valueOf(buffer.readableBytes()), false);
addCookies(resp);
ChannelFuture future;
if (orderedUpstreamMessageEvent != null) {
OrderedDownstreamChannelEvent downstreamChannelEvent =
new OrderedDownstreamChannelEvent(orderedUpstreamMessageEvent, 0, true, resp);
future = downstreamChannelEvent.getFuture();
channel.getPipeline().sendDownstream(downstreamChannelEvent);
} else {
future = channel.write(resp);
}
if (content instanceof Releasable) {
future.addListener((x) -> ((Releasable)content).close());
addedReleaseListener = true;
}
if (isCloseConnection()) {
future.addListener(ChannelFutureListener.CLOSE);
}
} finally {
if (!addedReleaseListener && content instanceof Releasable) {
((Releasable) content).close();
}
}
}
private void setHeaderField(HttpResponse resp, String headerField, String value) {
setHeaderField(resp, headerField, value, true);
}
private void setHeaderField(HttpResponse resp, String headerField, String value, boolean override) {
if (override || !resp.headers().contains(headerField)) {
resp.headers().add(headerField, value);
}
}
private void addCookies(HttpResponse resp) {
if (transport.resetCookies) {
String cookieString = nettyRequest.headers().get(HttpHeaders.Names.COOKIE);
if (cookieString != null) {
CookieDecoder cookieDecoder = new CookieDecoder();
Set<Cookie> cookies = cookieDecoder.decode(cookieString);
if (!cookies.isEmpty()) {
// Reset the cookies if necessary.
CookieEncoder cookieEncoder = new CookieEncoder(true);
for (Cookie cookie : cookies) {
cookieEncoder.addCookie(cookie);
}
setHeaderField(resp, HttpHeaders.Names.SET_COOKIE, cookieEncoder.encode());
}
}
}
}
private void addCustomHeaders(HttpResponse resp, Map<String, List<String>> customHeaders) {
if (customHeaders != null) {
for (Map.Entry<String, List<String>> headerEntry : customHeaders.entrySet()) {
for (String headerValue : headerEntry.getValue()) {
setHeaderField(resp, headerEntry.getKey(), headerValue);
}
}
}
}
// Determine if the request protocol version is HTTP 1.0
private boolean isHttp10() {
return nettyRequest.getProtocolVersion().equals(HttpVersion.HTTP_1_0);
}
// Determine if the request connection should be closed on completion.
private boolean isCloseConnection() {
final boolean http10 = isHttp10();
return CLOSE.equalsIgnoreCase(nettyRequest.headers().get(CONNECTION)) ||
(http10 && !KEEP_ALIVE.equalsIgnoreCase(nettyRequest.headers().get(CONNECTION)));
}
// Create a new {@link HttpResponse} to transmit the response for the netty request.
private HttpResponse newResponse() {
final boolean http10 = isHttp10();
final boolean close = isCloseConnection();
// Build the response object.
HttpResponseStatus status = HttpResponseStatus.OK; // default to initialize
org.jboss.netty.handler.codec.http.HttpResponse resp;
if (http10) {
resp = new DefaultHttpResponse(HttpVersion.HTTP_1_0, status);
if (!close) {
resp.headers().add(CONNECTION, "Keep-Alive");
}
} else {
resp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, status);
}
return resp;
}
private static final HttpResponseStatus TOO_MANY_REQUESTS = new HttpResponseStatus(429, "Too Many Requests");
static Map<RestStatus, HttpResponseStatus> MAP;
static {
EnumMap<RestStatus, HttpResponseStatus> map = new EnumMap<>(RestStatus.class);
map.put(RestStatus.CONTINUE, HttpResponseStatus.CONTINUE);
map.put(RestStatus.SWITCHING_PROTOCOLS, HttpResponseStatus.SWITCHING_PROTOCOLS);
map.put(RestStatus.OK, HttpResponseStatus.OK);
map.put(RestStatus.CREATED, HttpResponseStatus.CREATED);
map.put(RestStatus.ACCEPTED, HttpResponseStatus.ACCEPTED);
map.put(RestStatus.NON_AUTHORITATIVE_INFORMATION, HttpResponseStatus.NON_AUTHORITATIVE_INFORMATION);
map.put(RestStatus.NO_CONTENT, HttpResponseStatus.NO_CONTENT);
map.put(RestStatus.RESET_CONTENT, HttpResponseStatus.RESET_CONTENT);
map.put(RestStatus.PARTIAL_CONTENT, HttpResponseStatus.PARTIAL_CONTENT);
map.put(RestStatus.MULTI_STATUS, HttpResponseStatus.INTERNAL_SERVER_ERROR); // no status for this??
map.put(RestStatus.MULTIPLE_CHOICES, HttpResponseStatus.MULTIPLE_CHOICES);
map.put(RestStatus.MOVED_PERMANENTLY, HttpResponseStatus.MOVED_PERMANENTLY);
map.put(RestStatus.FOUND, HttpResponseStatus.FOUND);
map.put(RestStatus.SEE_OTHER, HttpResponseStatus.SEE_OTHER);
map.put(RestStatus.NOT_MODIFIED, HttpResponseStatus.NOT_MODIFIED);
map.put(RestStatus.USE_PROXY, HttpResponseStatus.USE_PROXY);
map.put(RestStatus.TEMPORARY_REDIRECT, HttpResponseStatus.TEMPORARY_REDIRECT);
map.put(RestStatus.BAD_REQUEST, HttpResponseStatus.BAD_REQUEST);
map.put(RestStatus.UNAUTHORIZED, HttpResponseStatus.UNAUTHORIZED);
map.put(RestStatus.PAYMENT_REQUIRED, HttpResponseStatus.PAYMENT_REQUIRED);
map.put(RestStatus.FORBIDDEN, HttpResponseStatus.FORBIDDEN);
map.put(RestStatus.NOT_FOUND, HttpResponseStatus.NOT_FOUND);
map.put(RestStatus.METHOD_NOT_ALLOWED, HttpResponseStatus.METHOD_NOT_ALLOWED);
map.put(RestStatus.NOT_ACCEPTABLE, HttpResponseStatus.NOT_ACCEPTABLE);
map.put(RestStatus.PROXY_AUTHENTICATION, HttpResponseStatus.PROXY_AUTHENTICATION_REQUIRED);
map.put(RestStatus.REQUEST_TIMEOUT, HttpResponseStatus.REQUEST_TIMEOUT);
map.put(RestStatus.CONFLICT, HttpResponseStatus.CONFLICT);
map.put(RestStatus.GONE, HttpResponseStatus.GONE);
map.put(RestStatus.LENGTH_REQUIRED, HttpResponseStatus.LENGTH_REQUIRED);
map.put(RestStatus.PRECONDITION_FAILED, HttpResponseStatus.PRECONDITION_FAILED);
map.put(RestStatus.REQUEST_ENTITY_TOO_LARGE, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE);
map.put(RestStatus.REQUEST_URI_TOO_LONG, HttpResponseStatus.REQUEST_URI_TOO_LONG);
map.put(RestStatus.UNSUPPORTED_MEDIA_TYPE, HttpResponseStatus.UNSUPPORTED_MEDIA_TYPE);
map.put(RestStatus.REQUESTED_RANGE_NOT_SATISFIED, HttpResponseStatus.REQUESTED_RANGE_NOT_SATISFIABLE);
map.put(RestStatus.EXPECTATION_FAILED, HttpResponseStatus.EXPECTATION_FAILED);
map.put(RestStatus.UNPROCESSABLE_ENTITY, HttpResponseStatus.BAD_REQUEST);
map.put(RestStatus.LOCKED, HttpResponseStatus.BAD_REQUEST);
map.put(RestStatus.FAILED_DEPENDENCY, HttpResponseStatus.BAD_REQUEST);
map.put(RestStatus.TOO_MANY_REQUESTS, TOO_MANY_REQUESTS);
map.put(RestStatus.INTERNAL_SERVER_ERROR, HttpResponseStatus.INTERNAL_SERVER_ERROR);
map.put(RestStatus.NOT_IMPLEMENTED, HttpResponseStatus.NOT_IMPLEMENTED);
map.put(RestStatus.BAD_GATEWAY, HttpResponseStatus.BAD_GATEWAY);
map.put(RestStatus.SERVICE_UNAVAILABLE, HttpResponseStatus.SERVICE_UNAVAILABLE);
map.put(RestStatus.GATEWAY_TIMEOUT, HttpResponseStatus.GATEWAY_TIMEOUT);
map.put(RestStatus.HTTP_VERSION_NOT_SUPPORTED, HttpResponseStatus.HTTP_VERSION_NOT_SUPPORTED);
MAP = Collections.unmodifiableMap(map);
}
private static HttpResponseStatus getStatus(RestStatus status) {
return MAP.getOrDefault(status, HttpResponseStatus.INTERNAL_SERVER_ERROR);
}
}

133
modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java
View File

@ -1,133 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.transport.netty3.Netty3Utils;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.RestUtils;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpRequest;
import java.net.SocketAddress;
import java.util.HashMap;
import java.util.Map;
public class Netty3HttpRequest extends RestRequest {
private final HttpRequest request;
private final Channel channel;
private final BytesReference content;
public Netty3HttpRequest(HttpRequest request, Channel channel) {
super(request.getUri());
this.request = request;
this.channel = channel;
if (request.getContent().readable()) {
this.content = Netty3Utils.toBytesReference(request.getContent());
} else {
this.content = BytesArray.EMPTY;
}
}
public org.jboss.netty.handler.codec.http.HttpRequest request() {
return this.request;
}
@Override
public Method method() {
HttpMethod httpMethod = request.getMethod();
if (httpMethod == HttpMethod.GET)
return Method.GET;
if (httpMethod == HttpMethod.POST)
return Method.POST;
if (httpMethod == HttpMethod.PUT)
return Method.PUT;
if (httpMethod == HttpMethod.DELETE)
return Method.DELETE;
if (httpMethod == HttpMethod.HEAD) {
return Method.HEAD;
}
if (httpMethod == HttpMethod.OPTIONS) {
return Method.OPTIONS;
}
return Method.GET;
}
@Override
public String uri() {
return request.getUri();
}
@Override
public boolean hasContent() {
return content.length() > 0;
}
@Override
public BytesReference content() {
return content;
}
/**
* Returns the remote address where this rest request channel is "connected to". The
* returned {@link SocketAddress} is supposed to be down-cast into more
* concrete type such as {@link java.net.InetSocketAddress} to retrieve
* the detailed information.
*/
@Override
public SocketAddress getRemoteAddress() {
return channel.getRemoteAddress();
}
/**
* Returns the local address where this request channel is bound to. The returned
* {@link SocketAddress} is supposed to be down-cast into more concrete
* type such as {@link java.net.InetSocketAddress} to retrieve the detailed
* information.
*/
@Override
public SocketAddress getLocalAddress() {
return channel.getLocalAddress();
}
public Channel getChannel() {
return channel;
}
@Override
public String header(String name) {
return request.headers().get(name);
}
@Override
public Iterable<Map.Entry<String, String>> headers() {
return request.headers().entries();
}
}

69
modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java
View File

@ -1,69 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.http.netty3.pipelining.OrderedUpstreamMessageEvent;
import org.jboss.netty.channel.ChannelHandler;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
import org.jboss.netty.handler.codec.http.HttpRequest;
@ChannelHandler.Sharable
public class Netty3HttpRequestHandler extends SimpleChannelUpstreamHandler {
private final Netty3HttpServerTransport serverTransport;
private final boolean httpPipeliningEnabled;
private final boolean detailedErrorsEnabled;
private final ThreadContext threadContext;
public Netty3HttpRequestHandler(Netty3HttpServerTransport serverTransport, boolean detailedErrorsEnabled, ThreadContext threadContext) {
this.serverTransport = serverTransport;
this.httpPipeliningEnabled = serverTransport.pipelining;
this.detailedErrorsEnabled = detailedErrorsEnabled;
this.threadContext = threadContext;
}
@Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
HttpRequest request;
OrderedUpstreamMessageEvent oue = null;
if (this.httpPipeliningEnabled && e instanceof OrderedUpstreamMessageEvent) {
oue = (OrderedUpstreamMessageEvent) e;
request = (HttpRequest) oue.getMessage();
} else {
request = (HttpRequest) e.getMessage();
}
// the netty HTTP handling always copy over the buffer to its own buffer, either in NioWorker internally
// when reading, or using a cumulation buffer
Netty3HttpRequest httpRequest = new Netty3HttpRequest(request, e.getChannel());
Netty3HttpChannel channel = new Netty3HttpChannel(serverTransport, httpRequest, oue, detailedErrorsEnabled, threadContext);
serverTransport.dispatchRequest(httpRequest, channel);
super.messageReceived(ctx, e);
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
serverTransport.exceptionCaught(ctx, e);
}
}

573
modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java
View File

@ -1,573 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import com.carrotsearch.hppc.IntHashSet;
import com.carrotsearch.hppc.IntSet;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.BoundTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.transport.NetworkExceptionHelper;
import org.elasticsearch.common.transport.PortsRange;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.http.BindHttpException;
import org.elasticsearch.http.HttpInfo;
import org.elasticsearch.http.HttpServerAdapter;
import org.elasticsearch.http.HttpServerTransport;
import org.elasticsearch.http.HttpStats;
import org.elasticsearch.http.netty3.cors.Netty3CorsConfig;
import org.elasticsearch.http.netty3.cors.Netty3CorsConfigBuilder;
import org.elasticsearch.http.netty3.cors.Netty3CorsHandler;
import org.elasticsearch.http.netty3.pipelining.HttpPipeliningHandler;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.rest.RestChannel;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.RestUtils;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.BindTransportException;
import org.elasticsearch.transport.netty3.Netty3OpenChannelsHandler;
import org.elasticsearch.transport.netty3.Netty3Utils;
import org.jboss.netty.bootstrap.ServerBootstrap;
import org.jboss.netty.channel.AdaptiveReceiveBufferSizePredictorFactory;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.FixedReceiveBufferSizePredictorFactory;
import org.jboss.netty.channel.ReceiveBufferSizePredictorFactory;
import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
import org.jboss.netty.channel.socket.oio.OioServerSocketChannelFactory;
import org.jboss.netty.handler.codec.http.HttpChunkAggregator;
import org.jboss.netty.handler.codec.http.HttpContentCompressor;
import org.jboss.netty.handler.codec.http.HttpContentDecompressor;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpRequestDecoder;
import org.jboss.netty.handler.timeout.ReadTimeoutException;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.elasticsearch.common.settings.Setting.boolSetting;
import static org.elasticsearch.common.settings.Setting.byteSizeSetting;
import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_BIND_HOST;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION_LEVEL;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PORT;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_PORT;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_RESET_COOKIES;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_PIPELINING;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS;
import static org.elasticsearch.http.netty3.cors.Netty3CorsHandler.ANY_ORIGIN;
public class Netty3HttpServerTransport extends AbstractLifecycleComponent implements HttpServerTransport {
static {
Netty3Utils.setup();
}
public static Setting<ByteSizeValue> SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY =
Setting.byteSizeSetting("http.netty.max_cumulation_buffer_capacity", new ByteSizeValue(-1),
Property.NodeScope, Property.Shared);
public static Setting<Integer> SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS =
Setting.intSetting("http.netty.max_composite_buffer_components", -1, Property.NodeScope, Property.Shared);
public static final Setting<Integer> SETTING_HTTP_WORKER_COUNT = new Setting<>("http.netty.worker_count",
(s) -> Integer.toString(EsExecutors.numberOfProcessors(s) * 2),
(s) -> Setting.parseInt(s, 1, "http.netty.worker_count"), Property.NodeScope, Property.Shared);
public static final Setting<Boolean> SETTING_HTTP_TCP_NO_DELAY =
boolSetting("http.tcp_no_delay", NetworkService.TcpSettings.TCP_NO_DELAY, Property.NodeScope, Property.Shared);
public static final Setting<Boolean> SETTING_HTTP_TCP_KEEP_ALIVE =
boolSetting("http.tcp.keep_alive", NetworkService.TcpSettings.TCP_KEEP_ALIVE, Property.NodeScope, Property.Shared);
public static final Setting<Boolean> SETTING_HTTP_TCP_BLOCKING_SERVER =
boolSetting("http.tcp.blocking_server", NetworkService.TcpSettings.TCP_BLOCKING_SERVER, Property.NodeScope, Property.Shared);
public static final Setting<Boolean> SETTING_HTTP_TCP_REUSE_ADDRESS =
boolSetting("http.tcp.reuse_address", NetworkService.TcpSettings.TCP_REUSE_ADDRESS, Property.NodeScope, Property.Shared);
public static final Setting<ByteSizeValue> SETTING_HTTP_TCP_SEND_BUFFER_SIZE =
Setting.byteSizeSetting("http.tcp.send_buffer_size", NetworkService.TcpSettings.TCP_SEND_BUFFER_SIZE,
Property.NodeScope, Property.Shared);
public static final Setting<ByteSizeValue> SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE =
Setting.byteSizeSetting("http.tcp.receive_buffer_size", NetworkService.TcpSettings.TCP_RECEIVE_BUFFER_SIZE,
Property.NodeScope, Property.Shared);
public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE =
Setting.byteSizeSetting("transport.netty.receive_predictor_size",
settings -> {
long defaultReceiverPredictor = 512 * 1024;
if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().getBytes() > 0) {
// we can guess a better default...
long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().getBytes()) / SETTING_HTTP_WORKER_COUNT.get
(settings));
defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024));
}
return new ByteSizeValue(defaultReceiverPredictor).toString();
}, Property.NodeScope);
public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN =
byteSizeSetting("http.netty.receive_predictor_min", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope);
public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX =
byteSizeSetting("http.netty.receive_predictor_max", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope);
protected final NetworkService networkService;
protected final BigArrays bigArrays;
protected final ByteSizeValue maxContentLength;
protected final ByteSizeValue maxInitialLineLength;
protected final ByteSizeValue maxHeaderSize;
protected final ByteSizeValue maxChunkSize;
protected final int workerCount;
protected final boolean blockingServer;
protected final boolean pipelining;
protected final int pipeliningMaxEvents;
protected final boolean compression;
protected final int compressionLevel;
protected final boolean resetCookies;
protected final PortsRange port;
protected final String bindHosts[];
protected final String publishHosts[];
protected final boolean detailedErrorsEnabled;
protected final ThreadPool threadPool;
protected final boolean tcpNoDelay;
protected final boolean tcpKeepAlive;
protected final boolean reuseAddress;
protected final ByteSizeValue tcpSendBufferSize;
protected final ByteSizeValue tcpReceiveBufferSize;
protected final ReceiveBufferSizePredictorFactory receiveBufferSizePredictorFactory;
protected final ByteSizeValue maxCumulationBufferCapacity;
protected final int maxCompositeBufferComponents;
protected volatile ServerBootstrap serverBootstrap;
protected volatile BoundTransportAddress boundAddress;
protected volatile List<Channel> serverChannels = new ArrayList<>();
// package private for testing
Netty3OpenChannelsHandler serverOpenChannels;
protected volatile HttpServerAdapter httpServerAdapter;
private final Netty3CorsConfig corsConfig;
public Netty3HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool) {
super(settings);
this.networkService = networkService;
this.bigArrays = bigArrays;
this.threadPool = threadPool;
ByteSizeValue maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings);
this.maxChunkSize = SETTING_HTTP_MAX_CHUNK_SIZE.get(settings);
this.maxHeaderSize = SETTING_HTTP_MAX_HEADER_SIZE.get(settings);
this.maxInitialLineLength = SETTING_HTTP_MAX_INITIAL_LINE_LENGTH.get(settings);
this.resetCookies = SETTING_HTTP_RESET_COOKIES.get(settings);
this.maxCumulationBufferCapacity = SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY.get(settings);
this.maxCompositeBufferComponents = SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS.get(settings);
this.workerCount = SETTING_HTTP_WORKER_COUNT.get(settings);
this.blockingServer = SETTING_HTTP_TCP_BLOCKING_SERVER.get(settings);
this.port = SETTING_HTTP_PORT.get(settings);
this.bindHosts = SETTING_HTTP_BIND_HOST.get(settings).toArray(Strings.EMPTY_ARRAY);
this.publishHosts = SETTING_HTTP_PUBLISH_HOST.get(settings).toArray(Strings.EMPTY_ARRAY);
this.tcpNoDelay = SETTING_HTTP_TCP_NO_DELAY.get(settings);
this.tcpKeepAlive = SETTING_HTTP_TCP_KEEP_ALIVE.get(settings);
this.reuseAddress = SETTING_HTTP_TCP_REUSE_ADDRESS.get(settings);
this.tcpSendBufferSize = SETTING_HTTP_TCP_SEND_BUFFER_SIZE.get(settings);
this.tcpReceiveBufferSize = SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE.get(settings);
this.detailedErrorsEnabled = SETTING_HTTP_DETAILED_ERRORS_ENABLED.get(settings);
// See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one
ByteSizeValue receivePredictorMin = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN.get(settings);
ByteSizeValue receivePredictorMax = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX.get(settings);
if (receivePredictorMax.getBytes() == receivePredictorMin.getBytes()) {
receiveBufferSizePredictorFactory = new FixedReceiveBufferSizePredictorFactory((int) receivePredictorMax.getBytes());
} else {
receiveBufferSizePredictorFactory = new AdaptiveReceiveBufferSizePredictorFactory(
(int) receivePredictorMin.getBytes(), (int) receivePredictorMin.getBytes(), (int) receivePredictorMax.getBytes());
}
this.compression = SETTING_HTTP_COMPRESSION.get(settings);
this.compressionLevel = SETTING_HTTP_COMPRESSION_LEVEL.get(settings);
this.pipelining = SETTING_PIPELINING.get(settings);
this.pipeliningMaxEvents = SETTING_PIPELINING_MAX_EVENTS.get(settings);
this.corsConfig = buildCorsConfig(settings);
// validate max content length
if (maxContentLength.getBytes() > Integer.MAX_VALUE) {
logger.warn("maxContentLength[{}] set to high value, resetting it to [100mb]", maxContentLength);
maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB);
}
this.maxContentLength = maxContentLength;
logger.debug("using max_chunk_size[{}], max_header_size[{}], max_initial_line_length[{}], max_content_length[{}], " +
"receive_predictor[{}->{}], pipelining[{}], pipelining_max_events[{}]", maxChunkSize, maxHeaderSize, maxInitialLineLength,
this.maxContentLength, receivePredictorMin, receivePredictorMax, pipelining, pipeliningMaxEvents);
}
public Settings settings() {
return this.settings;
}
@Override
public void httpServerAdapter(HttpServerAdapter httpServerAdapter) {
this.httpServerAdapter = httpServerAdapter;
}
@Override
protected void doStart() {
boolean success = false;
try {
this.serverOpenChannels = new Netty3OpenChannelsHandler(logger);
if (blockingServer) {
serverBootstrap = new ServerBootstrap(new OioServerSocketChannelFactory(
Executors.newCachedThreadPool(daemonThreadFactory(settings, HTTP_SERVER_BOSS_THREAD_NAME_PREFIX)),
Executors.newCachedThreadPool(daemonThreadFactory(settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX))
));
} else {
serverBootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(
Executors.newCachedThreadPool(daemonThreadFactory(settings, HTTP_SERVER_BOSS_THREAD_NAME_PREFIX)),
Executors.newCachedThreadPool(daemonThreadFactory(settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX)),
workerCount));
}
serverBootstrap.setPipelineFactory(configureServerChannelPipelineFactory());
serverBootstrap.setOption("child.tcpNoDelay", tcpNoDelay);
serverBootstrap.setOption("child.keepAlive", tcpKeepAlive);
if (tcpSendBufferSize.getBytes() > 0) {
serverBootstrap.setOption("child.sendBufferSize", tcpSendBufferSize.getBytes());
}
if (tcpReceiveBufferSize.getBytes() > 0) {
serverBootstrap.setOption("child.receiveBufferSize", tcpReceiveBufferSize.getBytes());
}
serverBootstrap.setOption("receiveBufferSizePredictorFactory", receiveBufferSizePredictorFactory);
serverBootstrap.setOption("child.receiveBufferSizePredictorFactory", receiveBufferSizePredictorFactory);
serverBootstrap.setOption("reuseAddress", reuseAddress);
serverBootstrap.setOption("child.reuseAddress", reuseAddress);
this.boundAddress = createBoundHttpAddress();
success = true;
} finally {
if (success == false) {
doStop(); // otherwise we leak threads since we never moved to started
}
}
}
private BoundTransportAddress createBoundHttpAddress() {
// Bind and start to accept incoming connections.
InetAddress hostAddresses[];
try {
hostAddresses = networkService.resolveBindHostAddresses(bindHosts);
} catch (IOException e) {
throw new BindHttpException("Failed to resolve host [" + Arrays.toString(bindHosts) + "]", e);
}
List<TransportAddress> boundAddresses = new ArrayList<>(hostAddresses.length);
for (InetAddress address : hostAddresses) {
boundAddresses.add(bindAddress(address));
}
final InetAddress publishInetAddress;
try {
publishInetAddress = networkService.resolvePublishHostAddresses(publishHosts);
} catch (Exception e) {
throw new BindTransportException("Failed to resolve publish address", e);
}
final int publishPort = resolvePublishPort(settings, boundAddresses, publishInetAddress);
final InetSocketAddress publishAddress = new InetSocketAddress(publishInetAddress, publishPort);
return new BoundTransportAddress(boundAddresses.toArray(new TransportAddress[0]), new TransportAddress(publishAddress));
}
// package private for tests
static int resolvePublishPort(Settings settings, List<TransportAddress> boundAddresses, InetAddress publishInetAddress) {
int publishPort = SETTING_HTTP_PUBLISH_PORT.get(settings);
if (publishPort < 0) {
for (TransportAddress boundAddress : boundAddresses) {
InetAddress boundInetAddress = boundAddress.address().getAddress();
if (boundInetAddress.isAnyLocalAddress() || boundInetAddress.equals(publishInetAddress)) {
publishPort = boundAddress.getPort();
break;
}
}
}
// if no matching boundAddress found, check if there is a unique port for all bound addresses
if (publishPort < 0) {
final IntSet ports = new IntHashSet();
for (TransportAddress boundAddress : boundAddresses) {
ports.add(boundAddress.getPort());
}
if (ports.size() == 1) {
publishPort = ports.iterator().next().value;
}
}
if (publishPort < 0) {
throw new BindHttpException("Failed to auto-resolve http publish port, multiple bound addresses " + boundAddresses +
" with distinct ports and none of them matched the publish address (" + publishInetAddress + "). " +
"Please specify a unique port by setting " + SETTING_HTTP_PORT.getKey() + " or " + SETTING_HTTP_PUBLISH_PORT.getKey());
}
return publishPort;
}
private Netty3CorsConfig buildCorsConfig(Settings settings) {
if (SETTING_CORS_ENABLED.get(settings) == false) {
return Netty3CorsConfigBuilder.forOrigins().disable().build();
}
String origin = SETTING_CORS_ALLOW_ORIGIN.get(settings);
final Netty3CorsConfigBuilder builder;
if (Strings.isNullOrEmpty(origin)) {
builder = Netty3CorsConfigBuilder.forOrigins();
} else if (origin.equals(ANY_ORIGIN)) {
builder = Netty3CorsConfigBuilder.forAnyOrigin();
} else {
Pattern p = RestUtils.checkCorsSettingForRegex(origin);
if (p == null) {
builder = Netty3CorsConfigBuilder.forOrigins(RestUtils.corsSettingAsArray(origin));
} else {
builder = Netty3CorsConfigBuilder.forPattern(p);
}
}
if (SETTING_CORS_ALLOW_CREDENTIALS.get(settings)) {
builder.allowCredentials();
}
Set<String> strMethods = Strings.splitStringByCommaToSet(SETTING_CORS_ALLOW_METHODS.get(settings));
return builder.allowedRequestMethods(strMethods.stream().map(HttpMethod::valueOf).collect(Collectors.toSet()))
.maxAge(SETTING_CORS_MAX_AGE.get(settings))
.allowedRequestHeaders(Strings.splitStringByCommaToSet(SETTING_CORS_ALLOW_HEADERS.get(settings)))
.shortCircuit()
.build();
}
private TransportAddress bindAddress(final InetAddress hostAddress) {
final AtomicReference<Exception> lastException = new AtomicReference<>();
final AtomicReference<InetSocketAddress> boundSocket = new AtomicReference<>();
boolean success = port.iterate(portNumber -> {
try {
synchronized (serverChannels) {
Channel channel = serverBootstrap.bind(new InetSocketAddress(hostAddress, portNumber));
serverChannels.add(channel);
boundSocket.set((InetSocketAddress) channel.getLocalAddress());
}
} catch (Exception e) {
lastException.set(e);
return false;
}
return true;
});
if (!success) {
throw new BindHttpException("Failed to bind to [" + port.getPortRangeString() + "]", lastException.get());
}
if (logger.isDebugEnabled()) {
logger.debug("Bound http to address {{}}", NetworkAddress.format(boundSocket.get()));
}
return new TransportAddress(boundSocket.get());
}
@Override
protected void doStop() {
synchronized (serverChannels) {
if (serverChannels != null) {
for (Channel channel : serverChannels) {
channel.close().awaitUninterruptibly();
}
serverChannels = null;
}
}
if (serverOpenChannels != null) {
serverOpenChannels.close();
serverOpenChannels = null;
}
if (serverBootstrap != null) {
serverBootstrap.releaseExternalResources();
serverBootstrap = null;
}
}
@Override
protected void doClose() {
}
@Override
public BoundTransportAddress boundAddress() {
return this.boundAddress;
}
@Override
public HttpInfo info() {
BoundTransportAddress boundTransportAddress = boundAddress();
if (boundTransportAddress == null) {
return null;
}
return new HttpInfo(boundTransportAddress, maxContentLength.getBytes());
}
@Override
public HttpStats stats() {
Netty3OpenChannelsHandler channels = serverOpenChannels;
return new HttpStats(channels == null ? 0 : channels.numberOfOpenChannels(), channels == null ? 0 : channels.totalChannels());
}
public Netty3CorsConfig getCorsConfig() {
return corsConfig;
}
protected void dispatchRequest(RestRequest request, RestChannel channel) {
httpServerAdapter.dispatchRequest(request, channel, threadPool.getThreadContext());
}
protected void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
if (e.getCause() instanceof ReadTimeoutException) {
if (logger.isTraceEnabled()) {
logger.trace("Connection timeout [{}]", ctx.getChannel().getRemoteAddress());
}
ctx.getChannel().close();
} else {
if (!lifecycle.started()) {
// ignore
return;
}
if (!NetworkExceptionHelper.isCloseConnectionException(e.getCause())) {
logger.warn(
(Supplier<?>) () -> new ParameterizedMessage(
"Caught exception while handling client http traffic, closing connection {}",
ctx.getChannel()),
e.getCause());
ctx.getChannel().close();
} else {
logger.debug(
(Supplier<?>) () -> new ParameterizedMessage(
"Caught exception while handling client http traffic, closing connection {}",
ctx.getChannel()),
e.getCause());
ctx.getChannel().close();
}
}
}
public ChannelPipelineFactory configureServerChannelPipelineFactory() {
return new HttpChannelPipelineFactory(this, detailedErrorsEnabled, threadPool.getThreadContext());
}
protected static class HttpChannelPipelineFactory implements ChannelPipelineFactory {
protected final Netty3HttpServerTransport transport;
protected final Netty3HttpRequestHandler requestHandler;
public HttpChannelPipelineFactory(Netty3HttpServerTransport transport, boolean detailedErrorsEnabled, ThreadContext threadContext) {
this.transport = transport;
this.requestHandler = new Netty3HttpRequestHandler(transport, detailedErrorsEnabled, threadContext);
}
@Override
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline pipeline = Channels.pipeline();
pipeline.addLast("openChannels", transport.serverOpenChannels);
HttpRequestDecoder requestDecoder = new HttpRequestDecoder(
(int) transport.maxInitialLineLength.getBytes(),
(int) transport.maxHeaderSize.getBytes(),
(int) transport.maxChunkSize.getBytes()
);
if (transport.maxCumulationBufferCapacity.getBytes() >= 0) {
if (transport.maxCumulationBufferCapacity.getBytes() > Integer.MAX_VALUE) {
requestDecoder.setMaxCumulationBufferCapacity(Integer.MAX_VALUE);
} else {
requestDecoder.setMaxCumulationBufferCapacity((int) transport.maxCumulationBufferCapacity.getBytes());
}
}
if (transport.maxCompositeBufferComponents != -1) {
requestDecoder.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents);
}
pipeline.addLast("decoder", requestDecoder);
pipeline.addLast("decoder_compress", new HttpContentDecompressor());
HttpChunkAggregator httpChunkAggregator = new HttpChunkAggregator((int) transport.maxContentLength.getBytes());
if (transport.maxCompositeBufferComponents != -1) {
httpChunkAggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents);
}
pipeline.addLast("aggregator", httpChunkAggregator);
pipeline.addLast("encoder", new ESNetty3HttpResponseEncoder());
if (transport.compression) {
pipeline.addLast("encoder_compress", new HttpContentCompressor(transport.compressionLevel));
}
if (SETTING_CORS_ENABLED.get(transport.settings())) {
pipeline.addLast("cors", new Netty3CorsHandler(transport.getCorsConfig()));
}
if (transport.pipelining) {
pipeline.addLast("pipelining", new HttpPipeliningHandler(transport.pipeliningMaxEvents));
}
pipeline.addLast("handler", requestHandler);
return pipeline;
}
}
}

233
modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/cors/Netty3CorsConfig.java
View File

@ -1,233 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3.cors;
import org.jboss.netty.handler.codec.http.DefaultHttpHeaders;
import org.jboss.netty.handler.codec.http.HttpHeaders;
import org.jboss.netty.handler.codec.http.HttpMethod;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.regex.Pattern;
/**
* Configuration for Cross-Origin Resource Sharing (CORS).
*
* This class was lifted from the Netty project:
* https://github.com/netty/netty
*/
public final class Netty3CorsConfig {
private final Optional<Set<String>> origins;
private final Optional<Pattern> pattern;
private final boolean anyOrigin;
private final boolean enabled;
private final boolean allowCredentials;
private final long maxAge;
private final Set<HttpMethod> allowedRequestMethods;
private final Set<String> allowedRequestHeaders;
private final boolean allowNullOrigin;
private final Map<CharSequence, Callable<?>> preflightHeaders;
private final boolean shortCircuit;
Netty3CorsConfig(final Netty3CorsConfigBuilder builder) {
origins = builder.origins.map(s -> new LinkedHashSet<>(s));
pattern = builder.pattern;
anyOrigin = builder.anyOrigin;
enabled = builder.enabled;
allowCredentials = builder.allowCredentials;
maxAge = builder.maxAge;
allowedRequestMethods = builder.requestMethods;
allowedRequestHeaders = builder.requestHeaders;
allowNullOrigin = builder.allowNullOrigin;
preflightHeaders = builder.preflightHeaders;
shortCircuit = builder.shortCircuit;
}
/**
* Determines if support for CORS is enabled.
*
* @return {@code true} if support for CORS is enabled, false otherwise.
*/
public boolean isCorsSupportEnabled() {
return enabled;
}
/**
* Determines whether a wildcard origin, '*', is supported.
*
* @return {@code boolean} true if any origin is allowed.
*/
public boolean isAnyOriginSupported() {
return anyOrigin;
}
/**
* Returns the set of allowed origins.
*
* @return {@code Set} the allowed origins.
*/
public Optional<Set<String>> origins() {
return origins;
}
/**
* Returns whether the input origin is allowed by this configuration.
*
* @return {@code true} if the origin is allowed, otherwise {@code false}
*/
public boolean isOriginAllowed(final String origin) {
if (origins.isPresent()) {
return origins.get().contains(origin);
} else if (pattern.isPresent()) {
return pattern.get().matcher(origin).matches();
}
return false;
}
/**
* Web browsers may set the 'Origin' request header to 'null' if a resource is loaded
* from the local file system.
*
* If isNullOriginAllowed is true then the server will response with the wildcard for the
* the CORS response header 'Access-Control-Allow-Origin'.
*
* @return {@code true} if a 'null' origin should be supported.
*/
public boolean isNullOriginAllowed() {
return allowNullOrigin;
}
/**
* Determines if cookies are supported for CORS requests.
*
* By default cookies are not included in CORS requests but if isCredentialsAllowed returns
* true cookies will be added to CORS requests. Setting this value to true will set the
* CORS 'Access-Control-Allow-Credentials' response header to true.
*
* Please note that cookie support needs to be enabled on the client side as well.
* The client needs to opt-in to send cookies by calling:
* <pre>
* xhr.withCredentials = true;
* </pre>
* The default value for 'withCredentials' is false in which case no cookies are sent.
* Setting this to true will included cookies in cross origin requests.
*
* @return {@code true} if cookies are supported.
*/
public boolean isCredentialsAllowed() {
return allowCredentials;
}
/**
* Gets the maxAge setting.
*
* When making a preflight request the client has to perform two request with can be inefficient.
* This setting will set the CORS 'Access-Control-Max-Age' response header and enables the
* caching of the preflight response for the specified time. During this time no preflight
* request will be made.
*
* @return {@code long} the time in seconds that a preflight request may be cached.
*/
public long maxAge() {
return maxAge;
}
/**
* Returns the allowed set of Request Methods. The Http methods that should be returned in the
* CORS 'Access-Control-Request-Method' response header.
*
* @return {@code Set} of {@link HttpMethod}s that represent the allowed Request Methods.
*/
public Set<HttpMethod> allowedRequestMethods() {
return Collections.unmodifiableSet(allowedRequestMethods);
}
/**
* Returns the allowed set of Request Headers.
*
* The header names returned from this method will be used to set the CORS
* 'Access-Control-Allow-Headers' response header.
*
* @return {@code Set<String>} of strings that represent the allowed Request Headers.
*/
public Set<String> allowedRequestHeaders() {
return Collections.unmodifiableSet(allowedRequestHeaders);
}
/**
* Returns HTTP response headers that should be added to a CORS preflight response.
*
* @return {@link HttpHeaders} the HTTP response headers to be added.
*/
public HttpHeaders preflightResponseHeaders() {
if (preflightHeaders.isEmpty()) {
return HttpHeaders.EMPTY_HEADERS;
}
final HttpHeaders preflightHeaders = new DefaultHttpHeaders();
for (Map.Entry<CharSequence, Callable<?>> entry : this.preflightHeaders.entrySet()) {
final Object value = getValue(entry.getValue());
if (value instanceof Iterable) {
preflightHeaders.add(entry.getKey().toString(), (Iterable<?>) value);
} else {
preflightHeaders.add(entry.getKey().toString(), value);
}
}
return preflightHeaders;
}
/**
* Determines whether a CORS request should be rejected if it's invalid before being
* further processing.
*
* CORS headers are set after a request is processed. This may not always be desired
* and this setting will check that the Origin is valid and if it is not valid no
* further processing will take place, and a error will be returned to the calling client.
*
* @return {@code true} if a CORS request should short-circuit upon receiving an invalid Origin header.
*/
public boolean isShortCircuit() {
return shortCircuit;
}
private static <T> T getValue(final Callable<T> callable) {
try {
return callable.call();
} catch (final Exception e) {
throw new IllegalStateException("Could not generate value for callable [" + callable + ']', e);
}
}
@Override
public String toString() {
return "CorsConfig[enabled=" + enabled +
", origins=" + origins +
", anyOrigin=" + anyOrigin +
", isCredentialsAllowed=" + allowCredentials +
", maxAge=" + maxAge +
", allowedRequestMethods=" + allowedRequestMethods +
", allowedRequestHeaders=" + allowedRequestHeaders +
", preflightHeaders=" + preflightHeaders + ']';
}
}

356
modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/cors/Netty3CorsConfigBuilder.java
View File

@ -1,356 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3.cors;
import org.jboss.netty.handler.codec.http.HttpMethod;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.regex.Pattern;
/**
* Builder used to configure and build a {@link Netty3CorsConfig} instance.
*
* This class was lifted from the Netty project:
* https://github.com/netty/netty
*/
public final class Netty3CorsConfigBuilder {
/**
* Creates a Builder instance with it's origin set to '*'.
*
* @return Builder to support method chaining.
*/
public static Netty3CorsConfigBuilder forAnyOrigin() {
return new Netty3CorsConfigBuilder();
}
/**
* Creates a {@link Netty3CorsConfigBuilder} instance with the specified origin.
*
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
public static Netty3CorsConfigBuilder forOrigin(final String origin) {
if ("*".equals(origin)) {
return new Netty3CorsConfigBuilder();
}
return new Netty3CorsConfigBuilder(origin);
}
/**
* Create a {@link Netty3CorsConfigBuilder} instance with the specified pattern origin.
*
* @param pattern the regular expression pattern to match incoming origins on.
* @return {@link Netty3CorsConfigBuilder} with the configured origin pattern.
*/
public static Netty3CorsConfigBuilder forPattern(final Pattern pattern) {
if (pattern == null) {
throw new IllegalArgumentException("CORS pattern cannot be null");
}
return new Netty3CorsConfigBuilder(pattern);
}
/**
* Creates a {@link Netty3CorsConfigBuilder} instance with the specified origins.
*
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
public static Netty3CorsConfigBuilder forOrigins(final String... origins) {
return new Netty3CorsConfigBuilder(origins);
}
Optional<Set<String>> origins;
Optional<Pattern> pattern;
final boolean anyOrigin;
boolean allowNullOrigin;
boolean enabled = true;
boolean allowCredentials;
long maxAge;
final Set<HttpMethod> requestMethods = new HashSet<>();
final Set<String> requestHeaders = new HashSet<>();
final Map<CharSequence, Callable<?>> preflightHeaders = new HashMap<>();
private boolean noPreflightHeaders;
boolean shortCircuit;
/**
* Creates a new Builder instance with the origin passed in.
*
* @param origins the origin to be used for this builder.
*/
Netty3CorsConfigBuilder(final String... origins) {
this.origins = Optional.of(new LinkedHashSet<>(Arrays.asList(origins)));
pattern = Optional.empty();
anyOrigin = false;
}
/**
* Creates a new Builder instance allowing any origin, "*" which is the
* wildcard origin.
*
*/
Netty3CorsConfigBuilder() {
anyOrigin = true;
origins = Optional.empty();
pattern = Optional.empty();
}
/**
* Creates a new Builder instance allowing any origin that matches the pattern.
*
* @param pattern the pattern to match against for incoming origins.
*/
Netty3CorsConfigBuilder(final Pattern pattern) {
this.pattern = Optional.of(pattern);
origins = Optional.empty();
anyOrigin = false;
}
/**
* Web browsers may set the 'Origin' request header to 'null' if a resource is loaded
* from the local file system. Calling this method will enable a successful CORS response
* with a wildcard for the CORS response header 'Access-Control-Allow-Origin'.
*
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
Netty3CorsConfigBuilder allowNullOrigin() {
allowNullOrigin = true;
return this;
}
/**
* Disables CORS support.
*
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
public Netty3CorsConfigBuilder disable() {
enabled = false;
return this;
}
/**
* By default cookies are not included in CORS requests, but this method will enable cookies to
* be added to CORS requests. Calling this method will set the CORS 'Access-Control-Allow-Credentials'
* response header to true.
*
* Please note, that cookie support needs to be enabled on the client side as well.
* The client needs to opt-in to send cookies by calling:
* <pre>
* xhr.withCredentials = true;
* </pre>
* The default value for 'withCredentials' is false in which case no cookies are sent.
* Setting this to true will included cookies in cross origin requests.
*
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
public Netty3CorsConfigBuilder allowCredentials() {
allowCredentials = true;
return this;
}
/**
* When making a preflight request the client has to perform two request with can be inefficient.
* This setting will set the CORS 'Access-Control-Max-Age' response header and enables the
* caching of the preflight response for the specified time. During this time no preflight
* request will be made.
*
* @param max the maximum time, in seconds, that the preflight response may be cached.
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
public Netty3CorsConfigBuilder maxAge(final long max) {
maxAge = max;
return this;
}
/**
* Specifies the allowed set of HTTP Request Methods that should be returned in the
* CORS 'Access-Control-Request-Method' response header.
*
* @param methods the {@link HttpMethod}s that should be allowed.
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
public Netty3CorsConfigBuilder allowedRequestMethods(final Set<HttpMethod> methods) {
requestMethods.addAll(methods);
return this;
}
/**
* Specifies the if headers that should be returned in the CORS 'Access-Control-Allow-Headers'
* response header.
*
* If a client specifies headers on the request, for example by calling:
* <pre>
* xhr.setRequestHeader('My-Custom-Header', "SomeValue");
* </pre>
* the server will receive the above header name in the 'Access-Control-Request-Headers' of the
* preflight request. The server will then decide if it allows this header to be sent for the
* real request (remember that a preflight is not the real request but a request asking the server
* if it allow a request).
*
* @param headers the headers to be added to the preflight 'Access-Control-Allow-Headers' response header.
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
public Netty3CorsConfigBuilder allowedRequestHeaders(final Set<String> headers) {
requestHeaders.addAll(headers);
return this;
}
/**
* Returns HTTP response headers that should be added to a CORS preflight response.
*
* An intermediary like a load balancer might require that a CORS preflight request
* have certain headers set. This enables such headers to be added.
*
* @param name the name of the HTTP header.
* @param values the values for the HTTP header.
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
public Netty3CorsConfigBuilder preflightResponseHeader(final CharSequence name, final Object... values) {
if (values.length == 1) {
preflightHeaders.put(name, new ConstantValueGenerator(values[0]));
} else {
preflightResponseHeader(name, Arrays.asList(values));
}
return this;
}
/**
* Returns HTTP response headers that should be added to a CORS preflight response.
*
* An intermediary like a load balancer might require that a CORS preflight request
* have certain headers set. This enables such headers to be added.
*
* @param name the name of the HTTP header.
* @param value the values for the HTTP header.
* @param <T> the type of values that the Iterable contains.
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
public <T> Netty3CorsConfigBuilder preflightResponseHeader(final CharSequence name, final Iterable<T> value) {
preflightHeaders.put(name, new ConstantValueGenerator(value));
return this;
}
/**
* Returns HTTP response headers that should be added to a CORS preflight response.
*
* An intermediary like a load balancer might require that a CORS preflight request
* have certain headers set. This enables such headers to be added.
*
* Some values must be dynamically created when the HTTP response is created, for
* example the 'Date' response header. This can be accomplished by using a Callable
* which will have its 'call' method invoked when the HTTP response is created.
*
* @param name the name of the HTTP header.
* @param valueGenerator a Callable which will be invoked at HTTP response creation.
* @param <T> the type of the value that the Callable can return.
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
public <T> Netty3CorsConfigBuilder preflightResponseHeader(final CharSequence name, final Callable<T> valueGenerator) {
preflightHeaders.put(name, valueGenerator);
return this;
}
/**
* Specifies that no preflight response headers should be added to a preflight response.
*
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
public Netty3CorsConfigBuilder noPreflightResponseHeaders() {
noPreflightHeaders = true;
return this;
}
/**
* Specifies that a CORS request should be rejected if it's invalid before being
* further processing.
*
* CORS headers are set after a request is processed. This may not always be desired
* and this setting will check that the Origin is valid and if it is not valid no
* further processing will take place, and a error will be returned to the calling client.
*
* @return {@link Netty3CorsConfigBuilder} to support method chaining.
*/
public Netty3CorsConfigBuilder shortCircuit() {
shortCircuit = true;
return this;
}
/**
* Builds a {@link Netty3CorsConfig} with settings specified by previous method calls.
*
* @return {@link Netty3CorsConfig} the configured CorsConfig instance.
*/
public Netty3CorsConfig build() {
if (preflightHeaders.isEmpty() && !noPreflightHeaders) {
preflightHeaders.put("date", DateValueGenerator.INSTANCE);
preflightHeaders.put("content-length", new ConstantValueGenerator("0"));
}
return new Netty3CorsConfig(this);
}
/**
* This class is used for preflight HTTP response values that do not need to be
* generated, but instead the value is "static" in that the same value will be returned
* for each call.
*/
private static final class ConstantValueGenerator implements Callable<Object> {
private final Object value;
/**
* Sole constructor.
*
* @param value the value that will be returned when the call method is invoked.
*/
private ConstantValueGenerator(final Object value) {
if (value == null) {
throw new IllegalArgumentException("value must not be null");
}
this.value = value;
}
@Override
public Object call() {
return value;
}
}
/**
* This callable is used for the DATE preflight HTTP response HTTP header.
* It's value must be generated when the response is generated, hence will be
* different for every call.
*/
private static final class DateValueGenerator implements Callable<Date> {
static final DateValueGenerator INSTANCE = new DateValueGenerator();
@Override
public Date call() throws Exception {
return new Date();
}
}
}

244
modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/cors/Netty3CorsHandler.java
View File

@ -1,244 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3.cors;
import org.elasticsearch.common.Strings;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
import org.jboss.netty.handler.codec.http.HttpHeaders;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.jboss.netty.handler.codec.http.HttpResponse;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_CREDENTIALS;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_HEADERS;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_METHODS;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_MAX_AGE;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.HOST;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ORIGIN;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.VARY;
import static org.jboss.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK;
/**
* Handles <a href="http://www.w3.org/TR/cors/">Cross Origin Resource Sharing</a> (CORS) requests.
* <p>
* This handler can be configured using a {@link Netty3CorsConfig}, please
* refer to this class for details about the configuration options available.
*
* This code was borrowed from Netty 4 and refactored to work for Elasticsearch's Netty 3 setup.
*/
public class Netty3CorsHandler extends SimpleChannelUpstreamHandler {
public static final String ANY_ORIGIN = "*";
private static Pattern SCHEME_PATTERN = Pattern.compile("^https?://");
private final Netty3CorsConfig config;
private HttpRequest request;
/**
* Creates a new instance with the specified {@link Netty3CorsConfig}.
*/
public Netty3CorsHandler(final Netty3CorsConfig config) {
if (config == null) {
throw new IllegalArgumentException("Config cannot be null");
}
this.config = config;
}
@Override
public void messageReceived(final ChannelHandlerContext ctx, final MessageEvent e) throws Exception {
if (config.isCorsSupportEnabled() && e.getMessage() instanceof HttpRequest) {
request = (HttpRequest) e.getMessage();
if (isPreflightRequest(request)) {
handlePreflight(ctx, request);
return;
}
if (config.isShortCircuit() && !validateOrigin()) {
forbidden(ctx, request);
return;
}
}
super.messageReceived(ctx, e);
}
public static void setCorsResponseHeaders(HttpRequest request, HttpResponse resp, Netty3CorsConfig config) {
if (!config.isCorsSupportEnabled()) {
return;
}
String originHeader = request.headers().get(ORIGIN);
if (!Strings.isNullOrEmpty(originHeader)) {
final String originHeaderVal;
if (config.isAnyOriginSupported()) {
originHeaderVal = ANY_ORIGIN;
} else if (config.isOriginAllowed(originHeader) || isSameOrigin(originHeader, request.headers().get(HOST))) {
originHeaderVal = originHeader;
} else {
originHeaderVal = null;
}
if (originHeaderVal != null) {
resp.headers().add(ACCESS_CONTROL_ALLOW_ORIGIN, originHeaderVal);
}
}
if (config.isCredentialsAllowed()) {
resp.headers().add(ACCESS_CONTROL_ALLOW_CREDENTIALS, "true");
}
}
private void handlePreflight(final ChannelHandlerContext ctx, final HttpRequest request) {
final HttpResponse response = new DefaultHttpResponse(request.getProtocolVersion(), OK);
if (setOrigin(response)) {
setAllowMethods(response);
setAllowHeaders(response);
setAllowCredentials(response);
setMaxAge(response);
setPreflightHeaders(response);
ctx.getChannel().write(response).addListener(ChannelFutureListener.CLOSE);
} else {
forbidden(ctx, request);
}
}
private static void forbidden(final ChannelHandlerContext ctx, final HttpRequest request) {
ctx.getChannel().write(new DefaultHttpResponse(request.getProtocolVersion(), FORBIDDEN))
.addListener(ChannelFutureListener.CLOSE);
}
private static boolean isSameOrigin(final String origin, final String host) {
if (Strings.isNullOrEmpty(host) == false) {
// strip protocol from origin
final String originDomain = SCHEME_PATTERN.matcher(origin).replaceFirst("");
if (host.equals(originDomain)) {
return true;
}
}
return false;
}
/**
* This is a non CORS specification feature which enables the setting of preflight
* response headers that might be required by intermediaries.
*
* @param response the HttpResponse to which the preflight response headers should be added.
*/
private void setPreflightHeaders(final HttpResponse response) {
response.headers().add(config.preflightResponseHeaders());
}
private boolean setOrigin(final HttpResponse response) {
final String origin = request.headers().get(ORIGIN);
if (!Strings.isNullOrEmpty(origin)) {
if ("null".equals(origin) && config.isNullOriginAllowed()) {
setAnyOrigin(response);
return true;
}
if (config.isAnyOriginSupported()) {
if (config.isCredentialsAllowed()) {
echoRequestOrigin(response);
setVaryHeader(response);
} else {
setAnyOrigin(response);
}
return true;
}
if (config.isOriginAllowed(origin)) {
setOrigin(response, origin);
setVaryHeader(response);
return true;
}
}
return false;
}
private boolean validateOrigin() {
if (config.isAnyOriginSupported()) {
return true;
}
final String origin = request.headers().get(ORIGIN);
if (Strings.isNullOrEmpty(origin)) {
// Not a CORS request so we cannot validate it. It may be a non CORS request.
return true;
}
if ("null".equals(origin) && config.isNullOriginAllowed()) {
return true;
}
// if the origin is the same as the host of the request, then allow
if (isSameOrigin(origin, request.headers().get(HOST))) {
return true;
}
return config.isOriginAllowed(origin);
}
private void echoRequestOrigin(final HttpResponse response) {
setOrigin(response, request.headers().get(ORIGIN));
}
private static void setVaryHeader(final HttpResponse response) {
response.headers().set(VARY, ORIGIN);
}
private static void setAnyOrigin(final HttpResponse response) {
setOrigin(response, ANY_ORIGIN);
}
private static void setOrigin(final HttpResponse response, final String origin) {
response.headers().set(ACCESS_CONTROL_ALLOW_ORIGIN, origin);
}
private void setAllowCredentials(final HttpResponse response) {
if (config.isCredentialsAllowed()
&& !response.headers().get(ACCESS_CONTROL_ALLOW_ORIGIN).equals(ANY_ORIGIN)) {
response.headers().set(ACCESS_CONTROL_ALLOW_CREDENTIALS, "true");
}
}
private static boolean isPreflightRequest(final HttpRequest request) {
final HttpHeaders headers = request.headers();
return request.getMethod().equals(HttpMethod.OPTIONS) &&
headers.contains(HttpHeaders.Names.ORIGIN) &&
headers.contains(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD);
}
private void setAllowMethods(final HttpResponse response) {
response.headers().set(ACCESS_CONTROL_ALLOW_METHODS, config.allowedRequestMethods().stream()
.map(m -> m.getName().trim())
.collect(Collectors.toList()));
}
private void setAllowHeaders(final HttpResponse response) {
response.headers().set(ACCESS_CONTROL_ALLOW_HEADERS, config.allowedRequestHeaders());
}
private void setMaxAge(final HttpResponse response) {
response.headers().set(ACCESS_CONTROL_MAX_AGE, config.maxAge());
}
}

134
modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/pipelining/HttpPipeliningHandler.java
View File

@ -1,134 +0,0 @@
package org.elasticsearch.http.netty3.pipelining;
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
// this file is from netty-http-pipelining, under apache 2.0 license
// see github.com/typesafehub/netty-http-pipelining
import org.jboss.netty.channel.ChannelEvent;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
import org.jboss.netty.handler.codec.http.HttpRequest;
import java.util.Comparator;
import java.util.PriorityQueue;
import java.util.Queue;
/**
* Implements HTTP pipelining ordering, ensuring that responses are completely served in the same order as their
* corresponding requests. NOTE: A side effect of using this handler is that upstream HttpRequest objects will
* cause the original message event to be effectively transformed into an OrderedUpstreamMessageEvent. Conversely
* OrderedDownstreamChannelEvent objects are expected to be received for the correlating response objects.
*
* @author Christopher Hunt
*/
public class HttpPipeliningHandler extends SimpleChannelHandler {
public static final int INITIAL_EVENTS_HELD = 3;
private final int maxEventsHeld;
private int sequence;
private int nextRequiredSequence;
private int nextRequiredSubsequence;
private final Queue<OrderedDownstreamChannelEvent> holdingQueue;
/**
* @param maxEventsHeld the maximum number of channel events that will be retained prior to aborting the channel
* connection. This is required as events cannot queue up indefinitely; we would run out of
* memory if this was the case.
*/
public HttpPipeliningHandler(final int maxEventsHeld) {
this.maxEventsHeld = maxEventsHeld;
holdingQueue = new PriorityQueue<>(INITIAL_EVENTS_HELD, new Comparator<OrderedDownstreamChannelEvent>() {
@Override
public int compare(OrderedDownstreamChannelEvent o1, OrderedDownstreamChannelEvent o2) {
final int delta = o1.getOrderedUpstreamMessageEvent().getSequence() - o2.getOrderedUpstreamMessageEvent().getSequence();
if (delta == 0) {
return o1.getSubsequence() - o2.getSubsequence();
} else {
return delta;
}
}
});
}
public int getMaxEventsHeld() {
return maxEventsHeld;
}
@Override
public void messageReceived(final ChannelHandlerContext ctx, final MessageEvent e) {
final Object msg = e.getMessage();
if (msg instanceof HttpRequest) {
ctx.sendUpstream(new OrderedUpstreamMessageEvent(sequence++, e.getChannel(), msg, e.getRemoteAddress()));
} else {
ctx.sendUpstream(e);
}
}
@Override
public void handleDownstream(ChannelHandlerContext ctx, ChannelEvent e)
throws Exception {
if (e instanceof OrderedDownstreamChannelEvent) {
boolean channelShouldClose = false;
synchronized (holdingQueue) {
if (holdingQueue.size() < maxEventsHeld) {
final OrderedDownstreamChannelEvent currentEvent = (OrderedDownstreamChannelEvent) e;
holdingQueue.add(currentEvent);
while (!holdingQueue.isEmpty()) {
final OrderedDownstreamChannelEvent nextEvent = holdingQueue.peek();
if (nextEvent.getOrderedUpstreamMessageEvent().getSequence() != nextRequiredSequence |
nextEvent.getSubsequence() != nextRequiredSubsequence) {
break;
}
holdingQueue.remove();
ctx.sendDownstream(nextEvent.getChannelEvent());
if (nextEvent.isLast()) {
++nextRequiredSequence;
nextRequiredSubsequence = 0;
} else {
++nextRequiredSubsequence;
}
}
} else {
channelShouldClose = true;
}
}
if (channelShouldClose) {
Channels.close(e.getChannel());
}
} else {
super.handleDownstream(ctx, e);
}
}
}

103
modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/pipelining/OrderedDownstreamChannelEvent.java
View File

@ -1,103 +0,0 @@
package org.elasticsearch.http.netty3.pipelining;
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
// this file is from netty-http-pipelining, under apache 2.0 license
// see github.com/typesafehub/netty-http-pipelining
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelEvent;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.DownstreamMessageEvent;
/**
* Permits downstream channel events to be ordered and signalled as to whether more are to come for a given sequence.
*
* @author Christopher Hunt
*/
public class OrderedDownstreamChannelEvent implements ChannelEvent {
final ChannelEvent ce;
final OrderedUpstreamMessageEvent oue;
final int subsequence;
final boolean last;
/**
* Construct a downstream channel event for all types of events.
*
* @param oue the OrderedUpstreamMessageEvent that this response is associated with
* @param subsequence the sequence within the sequence
* @param last when set to true this indicates that there are no more responses to be received for the
* original OrderedUpstreamMessageEvent
*/
public OrderedDownstreamChannelEvent(final OrderedUpstreamMessageEvent oue, final int subsequence, boolean last,
final ChannelEvent ce) {
this.oue = oue;
this.ce = ce;
this.subsequence = subsequence;
this.last = last;
}
/**
* Convenience constructor signifying that this downstream message event is the last one for the given sequence,
* and that there is only one response.
*/
public OrderedDownstreamChannelEvent(final OrderedUpstreamMessageEvent oe,
final Object message) {
this(oe, 0, true, message);
}
/**
* Convenience constructor for passing message events.
*/
public OrderedDownstreamChannelEvent(final OrderedUpstreamMessageEvent oue, final int subsequence, boolean last,
final Object message) {
this(oue, subsequence, last, new DownstreamMessageEvent(oue.getChannel(), Channels.future(oue.getChannel()),
message, oue.getRemoteAddress()));
}
public OrderedUpstreamMessageEvent getOrderedUpstreamMessageEvent() {
return oue;
}
public int getSubsequence() {
return subsequence;
}
public boolean isLast() {
return last;
}
@Override
public Channel getChannel() {
return ce.getChannel();
}
@Override
public ChannelFuture getFuture() {
return ce.getFuture();
}
public ChannelEvent getChannelEvent() {
return ce;
}
}

47
modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/pipelining/OrderedUpstreamMessageEvent.java
View File

@ -1,47 +0,0 @@
package org.elasticsearch.http.netty3.pipelining;
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
// this file is from netty-http-pipelining, under apache 2.0 license
// see github.com/typesafehub/netty-http-pipelining
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.UpstreamMessageEvent;
import java.net.SocketAddress;
/**
* Permits upstream message events to be ordered.
*
* @author Christopher Hunt
*/
public class OrderedUpstreamMessageEvent extends UpstreamMessageEvent {
final int sequence;
public OrderedUpstreamMessageEvent(final int sequence, final Channel channel, final Object msg, final SocketAddress remoteAddress) {
super(channel, msg, remoteAddress);
this.sequence = sequence;
}
public int getSequence() {
return sequence;
}
}

83
modules/transport-netty3/src/main/java/org/elasticsearch/transport/Netty3Plugin.java
View File

@ -1,83 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.http.HttpServerTransport;
import org.elasticsearch.http.netty3.Netty3HttpServerTransport;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.plugins.NetworkPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.netty3.Netty3Transport;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
public class Netty3Plugin extends Plugin implements NetworkPlugin {
public static final String NETTY_TRANSPORT_NAME = "netty3";
public static final String NETTY_HTTP_TRANSPORT_NAME = "netty3";
@Override
public List<Setting<?>> getSettings() {
return Arrays.asList(
Netty3HttpServerTransport.SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY,
Netty3HttpServerTransport.SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS,
Netty3HttpServerTransport.SETTING_HTTP_WORKER_COUNT,
Netty3HttpServerTransport.SETTING_HTTP_TCP_NO_DELAY,
Netty3HttpServerTransport.SETTING_HTTP_TCP_KEEP_ALIVE,
Netty3HttpServerTransport.SETTING_HTTP_TCP_BLOCKING_SERVER,
Netty3HttpServerTransport.SETTING_HTTP_TCP_REUSE_ADDRESS,
Netty3HttpServerTransport.SETTING_HTTP_TCP_SEND_BUFFER_SIZE,
Netty3HttpServerTransport.SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE,
Netty3Transport.WORKER_COUNT,
Netty3Transport.NETTY_MAX_CUMULATION_BUFFER_CAPACITY,
Netty3Transport.NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS,
Netty3Transport.NETTY_RECEIVE_PREDICTOR_SIZE,
Netty3Transport.NETTY_RECEIVE_PREDICTOR_MIN,
Netty3Transport.NETTY_RECEIVE_PREDICTOR_MAX,
Netty3Transport.NETTY_BOSS_COUNT
);
}
@Override
public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
CircuitBreakerService circuitBreakerService,
NamedWriteableRegistry namedWriteableRegistry, NetworkService networkService) {
return Collections.singletonMap(NETTY_TRANSPORT_NAME, () -> new Netty3Transport(settings, threadPool, networkService, bigArrays,
namedWriteableRegistry, circuitBreakerService));
}
@Override
public Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
CircuitBreakerService circuitBreakerService,
NamedWriteableRegistry namedWriteableRegistry,
NetworkService networkService) {
return Collections.singletonMap(NETTY_HTTP_TRANSPORT_NAME, () -> new Netty3HttpServerTransport(settings, networkService,
bigArrays, threadPool));
}
}

91
modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/ChannelBufferBytesReference.java
View File

@ -1,91 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.StreamInput;
import org.jboss.netty.buffer.ChannelBuffer;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
final class ChannelBufferBytesReference extends BytesReference {
private final ChannelBuffer buffer;
private final int length;
private final int offset;
ChannelBufferBytesReference(ChannelBuffer buffer, int length) {
this.buffer = buffer;
this.length = length;
this.offset = buffer.readerIndex();
assert length <= buffer.readableBytes() : "length[" + length +"] > " + buffer.readableBytes();
}
@Override
public byte get(int index) {
return buffer.getByte(offset + index);
}
@Override
public int length() {
return length;
}
@Override
public BytesReference slice(int from, int length) {
return new ChannelBufferBytesReference(buffer.slice(offset + from, length), length);
}
@Override
public StreamInput streamInput() {
return new ChannelBufferStreamInput(buffer.duplicate(), length);
}
@Override
public void writeTo(OutputStream os) throws IOException {
buffer.getBytes(offset, os, length);
}
ChannelBuffer toChannelBuffer() {
return buffer.duplicate();
}
@Override
public String utf8ToString() {
return buffer.toString(offset, length, StandardCharsets.UTF_8);
}
@Override
public BytesRef toBytesRef() {
if (buffer.hasArray()) {
return new BytesRef(buffer.array(), buffer.arrayOffset() + offset, length);
}
final byte[] copy = new byte[length];
buffer.getBytes(offset, copy);
return new BytesRef(copy);
}
@Override
public long ramBytesUsed() {
return buffer.capacity();
}
}

141
modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/ChannelBufferStreamInput.java
View File

@ -1,141 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.StreamInput;
import org.jboss.netty.buffer.ChannelBuffer;
import java.io.IOException;
/**
* A Netty {@link org.jboss.netty.buffer.ChannelBuffer} based {@link org.elasticsearch.common.io.stream.StreamInput}.
*/
class ChannelBufferStreamInput extends StreamInput {
private final ChannelBuffer buffer;
private final int startIndex;
private final int endIndex;
public ChannelBufferStreamInput(ChannelBuffer buffer, int length) {
if (length > buffer.readableBytes()) {
throw new IndexOutOfBoundsException();
}
this.buffer = buffer;
startIndex = buffer.readerIndex();
endIndex = startIndex + length;
buffer.markReaderIndex();
}
@Override
public BytesReference readBytesReference(int length) throws IOException {
BytesReference ref = Netty3Utils.toBytesReference(buffer.slice(buffer.readerIndex(), length));
buffer.skipBytes(length);
return ref;
}
@Override
public BytesRef readBytesRef(int length) throws IOException {
if (!buffer.hasArray()) {
return super.readBytesRef(length);
}
BytesRef bytesRef = new BytesRef(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), length);
buffer.skipBytes(length);
return bytesRef;
}
@Override
public int available() throws IOException {
return endIndex - buffer.readerIndex();
}
@Override
public void mark(int readlimit) {
buffer.markReaderIndex();
}
@Override
public boolean markSupported() {
return true;
}
@Override
public int read() throws IOException {
if (available() == 0) {
return -1;
}
return buffer.readByte() & 0xff;
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (len == 0) {
return 0;
}
int available = available();
if (available == 0) {
return -1;
}
len = Math.min(available, len);
buffer.readBytes(b, off, len);
return len;
}
@Override
public void reset() throws IOException {
buffer.resetReaderIndex();
}
@Override
public long skip(long n) throws IOException {
if (n > Integer.MAX_VALUE) {
return skipBytes(Integer.MAX_VALUE);
} else {
return skipBytes((int) n);
}
}
public int skipBytes(int n) throws IOException {
int nBytes = Math.min(available(), n);
buffer.skipBytes(nBytes);
return nBytes;
}
@Override
public byte readByte() throws IOException {
return buffer.readByte();
}
@Override
public void readBytes(byte[] b, int offset, int len) throws IOException {
int read = read(b, offset, len);
if (read < len) {
throw new IndexOutOfBoundsException();
}
}
@Override
public void close() throws IOException {
// nothing to do here
}
}

94
modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3InternalESLogger.java
View File

@ -1,94 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.SuppressLoggerChecks;
import org.jboss.netty.logging.AbstractInternalLogger;
@SuppressLoggerChecks(reason = "safely delegates to logger")
final class Netty3InternalESLogger extends AbstractInternalLogger {
private final Logger logger;
Netty3InternalESLogger(Logger logger) {
this.logger = logger;
}
@Override
public boolean isDebugEnabled() {
return logger.isDebugEnabled();
}
@Override
public boolean isInfoEnabled() {
return logger.isInfoEnabled();
}
@Override
public boolean isWarnEnabled() {
return logger.isWarnEnabled();
}
@Override
public boolean isErrorEnabled() {
return logger.isErrorEnabled();
}
@Override
public void debug(String msg) {
logger.debug(msg);
}
@Override
public void debug(String msg, Throwable cause) {
logger.debug(msg, cause);
}
@Override
public void info(String msg) {
logger.info(msg);
}
@Override
public void info(String msg, Throwable cause) {
logger.info(msg, cause);
}
@Override
public void warn(String msg) {
logger.warn(msg);
}
@Override
public void warn(String msg, Throwable cause) {
logger.warn(msg, cause);
}
@Override
public void error(String msg) {
logger.error(msg);
}
@Override
public void error(String msg, Throwable cause) {
logger.error(msg, cause);
}
}

84
modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java
View File

@ -1,84 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.transport.TcpHeader;
import org.elasticsearch.transport.TransportServiceAdapter;
import org.elasticsearch.transport.Transports;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
import org.jboss.netty.channel.WriteCompletionEvent;
import java.net.InetSocketAddress;
/**
* A handler (must be the last one!) that does size based frame decoding and forwards the actual message
* to the relevant action.
*/
class Netty3MessageChannelHandler extends SimpleChannelUpstreamHandler {
protected final TransportServiceAdapter transportServiceAdapter;
protected final Netty3Transport transport;
protected final String profileName;
Netty3MessageChannelHandler(Netty3Transport transport, String profileName) {
this.transportServiceAdapter = transport.transportServiceAdapter();
this.transport = transport;
this.profileName = profileName;
}
@Override
public void writeComplete(ChannelHandlerContext ctx, WriteCompletionEvent e) throws Exception {
transportServiceAdapter.addBytesSent(e.getWrittenAmount());
super.writeComplete(ctx, e);
}
@Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
Transports.assertTransportThread();
Object m = e.getMessage();
if (!(m instanceof ChannelBuffer)) {
ctx.sendUpstream(e);
return;
}
final ChannelBuffer buffer = (ChannelBuffer) m;
final int remainingMessageSize = buffer.getInt(buffer.readerIndex() - TcpHeader.MESSAGE_LENGTH_SIZE);
final int expectedReaderIndex = buffer.readerIndex() + remainingMessageSize;
InetSocketAddress remoteAddress = (InetSocketAddress) ctx.getChannel().getRemoteAddress();
try {
// netty always copies a buffer, either in NioWorker in its read handler, where it copies to a fresh
// buffer, or in the cumulation buffer, which is cleaned each time so it could be bigger than the actual size
BytesReference reference = Netty3Utils.toBytesReference(buffer, remainingMessageSize);
transport.messageReceived(reference, ctx.getChannel(), profileName, remoteAddress, remainingMessageSize);
} finally {
// Set the expected position of the buffer, no matter what happened
buffer.readerIndex(expectedReaderIndex);
}
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
transport.exceptionCaught(ctx, e);
}
}

98
modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3OpenChannelsHandler.java
View File

@ -1,98 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelEvent;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.ChannelHandler;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelState;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.ChannelUpstreamHandler;
import java.util.Set;
@ChannelHandler.Sharable
public class Netty3OpenChannelsHandler implements ChannelUpstreamHandler, Releasable {
final Set<Channel> openChannels = ConcurrentCollections.newConcurrentSet();
final CounterMetric openChannelsMetric = new CounterMetric();
final CounterMetric totalChannelsMetric = new CounterMetric();
final Logger logger;
public Netty3OpenChannelsHandler(Logger logger) {
this.logger = logger;
}
final ChannelFutureListener remover = new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
boolean removed = openChannels.remove(future.getChannel());
if (removed) {
openChannelsMetric.dec();
}
if (logger.isTraceEnabled()) {
logger.trace("channel closed: {}", future.getChannel());
}
}
};
@Override
public void handleUpstream(ChannelHandlerContext ctx, ChannelEvent e) throws Exception {
if (e instanceof ChannelStateEvent) {
ChannelStateEvent evt = (ChannelStateEvent) e;
// OPEN is also sent to when closing channel, but with FALSE on it to indicate it closes
if (evt.getState() == ChannelState.OPEN && Boolean.TRUE.equals(evt.getValue())) {
if (logger.isTraceEnabled()) {
logger.trace("channel opened: {}", ctx.getChannel());
}
boolean added = openChannels.add(ctx.getChannel());
if (added) {
openChannelsMetric.inc();
totalChannelsMetric.inc();
ctx.getChannel().getCloseFuture().addListener(remover);
}
}
}
ctx.sendUpstream(e);
}
public long numberOfOpenChannels() {
return openChannelsMetric.count();
}
public long totalChannels() {
return totalChannelsMetric.count();
}
@Override
public void close() {
for (Channel channel : openChannels) {
channel.close().awaitUninterruptibly();
}
}
}

45
modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3SizeHeaderFrameDecoder.java
View File

@ -1,45 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.elasticsearch.transport.TcpHeader;
import org.elasticsearch.transport.TcpTransport;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.handler.codec.frame.FrameDecoder;
import org.jboss.netty.handler.codec.frame.TooLongFrameException;
final class Netty3SizeHeaderFrameDecoder extends FrameDecoder {
@Override
protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer) throws Exception {
try {
boolean continueProcessing = TcpTransport.validateMessageHeader(Netty3Utils.toBytesReference(buffer));
buffer.skipBytes(TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE);
return continueProcessing ? buffer : null;
} catch (IllegalArgumentException ex) {
throw new TooLongFrameException(ex.getMessage(), ex);
} catch (IllegalStateException ex) {
return null;
}
}
}

555
modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java
View File

@ -1,555 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.network.NetworkService.TcpSettings;
import org.elasticsearch.common.network.NetworkUtils;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.ConnectTransportException;
import org.elasticsearch.transport.TcpTransport;
import org.elasticsearch.transport.TransportServiceAdapter;
import org.elasticsearch.transport.TransportSettings;
import org.jboss.netty.bootstrap.ClientBootstrap;
import org.jboss.netty.bootstrap.ServerBootstrap;
import org.jboss.netty.channel.AdaptiveReceiveBufferSizePredictorFactory;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.FixedReceiveBufferSizePredictorFactory;
import org.jboss.netty.channel.ReceiveBufferSizePredictorFactory;
import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
import org.jboss.netty.channel.socket.nio.NioWorkerPool;
import org.jboss.netty.channel.socket.oio.OioClientSocketChannelFactory;
import org.jboss.netty.channel.socket.oio.OioServerSocketChannelFactory;
import org.jboss.netty.util.HashedWheelTimer;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import static org.elasticsearch.common.settings.Setting.byteSizeSetting;
import static org.elasticsearch.common.settings.Setting.intSetting;
import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory;
/**
* There are 4 types of connections per node, low/med/high/ping. Low if for batch oriented APIs (like recovery or
* batch) with high payload that will cause regular request. (like search or single index) to take
* longer. Med is for the typical search / single doc index. And High for things like cluster state. Ping is reserved for
* sending out ping requests to other nodes.
*/
public class Netty3Transport extends TcpTransport<Channel> {
static {
Netty3Utils.setup();
}
public static final Setting<Integer> WORKER_COUNT =
new Setting<>("transport.netty.worker_count",
(s) -> Integer.toString(EsExecutors.numberOfProcessors(s) * 2),
(s) -> Setting.parseInt(s, 1, "transport.netty.worker_count"), Property.NodeScope, Property.Shared);
public static final Setting<ByteSizeValue> NETTY_MAX_CUMULATION_BUFFER_CAPACITY =
Setting.byteSizeSetting(
"transport.netty.max_cumulation_buffer_capacity",
new ByteSizeValue(-1),
Property.NodeScope,
Property.Shared);
public static final Setting<Integer> NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS =
Setting.intSetting("transport.netty.max_composite_buffer_components", -1, -1, Property.NodeScope, Property.Shared);
// See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one
public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_SIZE = Setting.byteSizeSetting(
"transport.netty.receive_predictor_size",
settings -> {
long defaultReceiverPredictor = 512 * 1024;
if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().getBytes() > 0) {
// we can guess a better default...
long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().getBytes()) / WORKER_COUNT.get(settings));
defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024));
}
return new ByteSizeValue(defaultReceiverPredictor).toString();
}, Property.NodeScope,
Property.Shared);
public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_MIN =
byteSizeSetting("transport.netty.receive_predictor_min", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared);
public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_MAX =
byteSizeSetting("transport.netty.receive_predictor_max", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared);
public static final Setting<Integer> NETTY_BOSS_COUNT =
intSetting("transport.netty.boss_count", 1, 1, Property.NodeScope, Property.Shared);
protected final ByteSizeValue maxCumulationBufferCapacity;
protected final int maxCompositeBufferComponents;
protected final ReceiveBufferSizePredictorFactory receiveBufferSizePredictorFactory;
protected final int workerCount;
protected final ByteSizeValue receivePredictorMin;
protected final ByteSizeValue receivePredictorMax;
// package private for testing
volatile Netty3OpenChannelsHandler serverOpenChannels;
protected volatile ClientBootstrap clientBootstrap;
protected final Map<String, ServerBootstrap> serverBootstraps = newConcurrentMap();
public Netty3Transport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays,
NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService) {
super("netty3", settings, threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, networkService);
this.workerCount = WORKER_COUNT.get(settings);
this.maxCumulationBufferCapacity = NETTY_MAX_CUMULATION_BUFFER_CAPACITY.get(settings);
this.maxCompositeBufferComponents = NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS.get(settings);
// See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one
this.receivePredictorMin = NETTY_RECEIVE_PREDICTOR_MIN.get(settings);
this.receivePredictorMax = NETTY_RECEIVE_PREDICTOR_MAX.get(settings);
if (receivePredictorMax.getBytes() == receivePredictorMin.getBytes()) {
receiveBufferSizePredictorFactory = new FixedReceiveBufferSizePredictorFactory((int) receivePredictorMax.getBytes());
} else {
receiveBufferSizePredictorFactory = new AdaptiveReceiveBufferSizePredictorFactory((int) receivePredictorMin.getBytes(),
(int) receivePredictorMin.getBytes(), (int) receivePredictorMax.getBytes());
}
}
TransportServiceAdapter transportServiceAdapter() {
return transportServiceAdapter;
}
@Override
protected void doStart() {
boolean success = false;
try {
clientBootstrap = createClientBootstrap();
if (NetworkService.NETWORK_SERVER.get(settings)) {
final Netty3OpenChannelsHandler openChannels = new Netty3OpenChannelsHandler(logger);
this.serverOpenChannels = openChannels;
// loop through all profiles and start them up, special handling for default one
for (Map.Entry<String, Settings> entry : buildProfileSettings().entrySet()) {
// merge fallback settings with default settings with profile settings so we have complete settings with default values
final Settings settings = Settings.builder()
.put(createFallbackSettings())
.put(entry.getValue()).build();
createServerBootstrap(entry.getKey(), settings);
bindServer(entry.getKey(), settings);
}
}
super.doStart();
success = true;
} finally {
if (success == false) {
doStop();
}
}
}
private ClientBootstrap createClientBootstrap() {
// this doPrivileged is for SelectorUtil.java that tries to set "sun.nio.ch.bugLevel"
if (blockingClient) {
clientBootstrap = new ClientBootstrap(new OioClientSocketChannelFactory(
Executors.newCachedThreadPool(daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX))));
} else {
int bossCount = NETTY_BOSS_COUNT.get(settings);
clientBootstrap = new ClientBootstrap(
new NioClientSocketChannelFactory(
Executors.newCachedThreadPool(daemonThreadFactory(settings, TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX)),
bossCount,
new NioWorkerPool(Executors.newCachedThreadPool(
daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX)), workerCount),
new HashedWheelTimer(daemonThreadFactory(settings, "transport_client_timer"))));
}
clientBootstrap.setPipelineFactory(configureClientChannelPipelineFactory());
clientBootstrap.setOption("connectTimeoutMillis", connectTimeout.millis());
boolean tcpNoDelay = TCP_NO_DELAY.get(settings);
clientBootstrap.setOption("tcpNoDelay", tcpNoDelay);
boolean tcpKeepAlive = TCP_KEEP_ALIVE.get(settings);
clientBootstrap.setOption("keepAlive", tcpKeepAlive);
ByteSizeValue tcpSendBufferSize = TCP_SEND_BUFFER_SIZE.get(settings);
if (tcpSendBufferSize.getBytes() > 0) {
clientBootstrap.setOption("sendBufferSize", tcpSendBufferSize.getBytes());
}
ByteSizeValue tcpReceiveBufferSize = TCP_RECEIVE_BUFFER_SIZE.get(settings);
if (tcpReceiveBufferSize.getBytes() > 0) {
clientBootstrap.setOption("receiveBufferSize", tcpReceiveBufferSize.getBytes());
}
clientBootstrap.setOption("receiveBufferSizePredictorFactory", receiveBufferSizePredictorFactory);
boolean reuseAddress = TCP_REUSE_ADDRESS.get(settings);
clientBootstrap.setOption("reuseAddress", reuseAddress);
return clientBootstrap;
}
private Settings createFallbackSettings() {
Settings.Builder fallbackSettingsBuilder = Settings.builder();
List<String> fallbackBindHost = TransportSettings.BIND_HOST.get(settings);
if (fallbackBindHost.isEmpty() == false) {
fallbackSettingsBuilder.putArray("bind_host", fallbackBindHost);
}
List<String> fallbackPublishHost = TransportSettings.PUBLISH_HOST.get(settings);
if (fallbackPublishHost.isEmpty() == false) {
fallbackSettingsBuilder.putArray("publish_host", fallbackPublishHost);
}
boolean fallbackTcpNoDelay = settings.getAsBoolean("transport.netty.tcp_no_delay", TcpSettings.TCP_NO_DELAY.get(settings));
fallbackSettingsBuilder.put("tcp_no_delay", fallbackTcpNoDelay);
boolean fallbackTcpKeepAlive = settings.getAsBoolean("transport.netty.tcp_keep_alive", TcpSettings.TCP_KEEP_ALIVE.get(settings));
fallbackSettingsBuilder.put("tcp_keep_alive", fallbackTcpKeepAlive);
boolean fallbackReuseAddress = settings.getAsBoolean("transport.netty.reuse_address", TcpSettings.TCP_REUSE_ADDRESS.get(settings));
fallbackSettingsBuilder.put("reuse_address", fallbackReuseAddress);
ByteSizeValue fallbackTcpSendBufferSize = settings.getAsBytesSize("transport.netty.tcp_send_buffer_size",
TCP_SEND_BUFFER_SIZE.get(settings));
if (fallbackTcpSendBufferSize.getBytes() >= 0) {
fallbackSettingsBuilder.put("tcp_send_buffer_size", fallbackTcpSendBufferSize);
}
ByteSizeValue fallbackTcpBufferSize = settings.getAsBytesSize("transport.netty.tcp_receive_buffer_size",
TCP_RECEIVE_BUFFER_SIZE.get(settings));
if (fallbackTcpBufferSize.getBytes() >= 0) {
fallbackSettingsBuilder.put("tcp_receive_buffer_size", fallbackTcpBufferSize);
}
return fallbackSettingsBuilder.build();
}
private void createServerBootstrap(String name, Settings settings) {
boolean blockingServer = TCP_BLOCKING_SERVER.get(settings);
String port = settings.get("port");
String bindHost = settings.get("bind_host");
String publishHost = settings.get("publish_host");
String tcpNoDelay = settings.get("tcp_no_delay");
String tcpKeepAlive = settings.get("tcp_keep_alive");
boolean reuseAddress = settings.getAsBoolean("reuse_address", NetworkUtils.defaultReuseAddress());
ByteSizeValue tcpSendBufferSize = TCP_SEND_BUFFER_SIZE.getDefault(settings);
ByteSizeValue tcpReceiveBufferSize = TCP_RECEIVE_BUFFER_SIZE.getDefault(settings);
if (logger.isDebugEnabled()) {
logger.debug("using profile[{}], worker_count[{}], port[{}], bind_host[{}], publish_host[{}], compress[{}], "
+ "connect_timeout[{}], connections_per_node[{}/{}/{}/{}/{}], receive_predictor[{}->{}]",
name, workerCount, port, bindHost, publishHost, compress, connectTimeout, connectionsPerNodeRecovery,
connectionsPerNodeBulk, connectionsPerNodeReg, connectionsPerNodeState, connectionsPerNodePing, receivePredictorMin,
receivePredictorMax);
}
final ThreadFactory bossFactory = daemonThreadFactory(this.settings, TRANSPORT_SERVER_BOSS_THREAD_NAME_PREFIX, name);
final ThreadFactory workerFactory = daemonThreadFactory(this.settings, TRANSPORT_SERVER_WORKER_THREAD_NAME_PREFIX, name);
final ServerBootstrap serverBootstrap;
if (blockingServer) {
serverBootstrap = new ServerBootstrap(new OioServerSocketChannelFactory(
Executors.newCachedThreadPool(bossFactory),
Executors.newCachedThreadPool(workerFactory)
));
} else {
serverBootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(
Executors.newCachedThreadPool(bossFactory),
Executors.newCachedThreadPool(workerFactory),
workerCount));
}
serverBootstrap.setPipelineFactory(configureServerChannelPipelineFactory(name, settings));
if (!"default".equals(tcpNoDelay)) {
serverBootstrap.setOption("child.tcpNoDelay", Booleans.parseBoolean(tcpNoDelay, null));
}
if (!"default".equals(tcpKeepAlive)) {
serverBootstrap.setOption("child.keepAlive", Booleans.parseBoolean(tcpKeepAlive, null));
}
if (tcpSendBufferSize != null && tcpSendBufferSize.getBytes() > 0) {
serverBootstrap.setOption("child.sendBufferSize", tcpSendBufferSize.getBytes());
}
if (tcpReceiveBufferSize != null && tcpReceiveBufferSize.getBytes() > 0) {
serverBootstrap.setOption("child.receiveBufferSize", tcpReceiveBufferSize.getBytes());
}
serverBootstrap.setOption("receiveBufferSizePredictorFactory", receiveBufferSizePredictorFactory);
serverBootstrap.setOption("child.receiveBufferSizePredictorFactory", receiveBufferSizePredictorFactory);
serverBootstrap.setOption("reuseAddress", reuseAddress);
serverBootstrap.setOption("child.reuseAddress", reuseAddress);
serverBootstraps.put(name, serverBootstrap);
}
protected final void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
onException(
ctx.getChannel(),
e.getCause() == null || e.getCause() instanceof Exception ?
(Exception)e.getCause() : new ElasticsearchException(e.getCause()));
}
@Override
public long serverOpen() {
Netty3OpenChannelsHandler channels = serverOpenChannels;
return channels == null ? 0 : channels.numberOfOpenChannels();
}
protected NodeChannels connectToChannelsLight(DiscoveryNode node) {
InetSocketAddress address = node.getAddress().address();
ChannelFuture connect = clientBootstrap.connect(address);
connect.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
if (!connect.isSuccess()) {
throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connect.getCause());
}
Channel[] channels = new Channel[1];
channels[0] = connect.getChannel();
channels[0].getCloseFuture().addListener(new ChannelCloseListener(node));
return new NodeChannels(channels, channels, channels, channels, channels);
}
protected NodeChannels connectToChannels(DiscoveryNode node) throws IOException {
final NodeChannels nodeChannels = new NodeChannels(new Channel[connectionsPerNodeRecovery], new Channel[connectionsPerNodeBulk],
new Channel[connectionsPerNodeReg], new Channel[connectionsPerNodeState],
new Channel[connectionsPerNodePing]);
boolean success = false;
try {
int numConnections = connectionsPerNodeBulk + connectionsPerNodePing + connectionsPerNodeRecovery + connectionsPerNodeReg
+ connectionsPerNodeState;
ArrayList<ChannelFuture> connections = new ArrayList<>();
InetSocketAddress address = node.getAddress().address();
for (int i = 0; i < numConnections; i++) {
connections.add(clientBootstrap.connect(address));
}
final Iterator<ChannelFuture> iterator = connections.iterator();
try {
for (Channel[] channels : nodeChannels.getChannelArrays()) {
for (int i = 0; i < channels.length; i++) {
assert iterator.hasNext();
ChannelFuture future = iterator.next();
future.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
if (!future.isSuccess()) {
throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", future.getCause());
}
channels[i] = future.getChannel();
channels[i].getCloseFuture().addListener(new ChannelCloseListener(node));
}
}
if (nodeChannels.recovery.length == 0) {
if (nodeChannels.bulk.length > 0) {
nodeChannels.recovery = nodeChannels.bulk;
} else {
nodeChannels.recovery = nodeChannels.reg;
}
}
if (nodeChannels.bulk.length == 0) {
nodeChannels.bulk = nodeChannels.reg;
}
} catch (RuntimeException e) {
for (ChannelFuture future : Collections.unmodifiableList(connections)) {
future.cancel();
if (future.getChannel() != null && future.getChannel().isOpen()) {
try {
future.getChannel().close();
} catch (Exception e1) {
// ignore
}
}
}
throw e;
}
success = true;
} finally {
if (success == false) {
nodeChannels.close();
}
}
return nodeChannels;
}
public ChannelPipelineFactory configureClientChannelPipelineFactory() {
return new ClientChannelPipelineFactory(this);
}
protected static class ClientChannelPipelineFactory implements ChannelPipelineFactory {
protected final Netty3Transport nettyTransport;
public ClientChannelPipelineFactory(Netty3Transport nettyTransport) {
this.nettyTransport = nettyTransport;
}
@Override
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline channelPipeline = Channels.pipeline();
Netty3SizeHeaderFrameDecoder sizeHeader = new Netty3SizeHeaderFrameDecoder();
if (nettyTransport.maxCumulationBufferCapacity.getBytes() >= 0) {
if (nettyTransport.maxCumulationBufferCapacity.getBytes() > Integer.MAX_VALUE) {
sizeHeader.setMaxCumulationBufferCapacity(Integer.MAX_VALUE);
} else {
sizeHeader.setMaxCumulationBufferCapacity((int) nettyTransport.maxCumulationBufferCapacity.getBytes());
}
}
if (nettyTransport.maxCompositeBufferComponents != -1) {
sizeHeader.setMaxCumulationBufferComponents(nettyTransport.maxCompositeBufferComponents);
}
channelPipeline.addLast("size", sizeHeader);
// using a dot as a prefix means, this cannot come from any settings parsed
channelPipeline.addLast("dispatcher", new Netty3MessageChannelHandler(nettyTransport, ".client"));
return channelPipeline;
}
}
public ChannelPipelineFactory configureServerChannelPipelineFactory(String name, Settings settings) {
return new ServerChannelPipelineFactory(this, name, settings);
}
protected static class ServerChannelPipelineFactory implements ChannelPipelineFactory {
protected final Netty3Transport nettyTransport;
protected final String name;
protected final Settings settings;
public ServerChannelPipelineFactory(Netty3Transport nettyTransport, String name, Settings settings) {
this.nettyTransport = nettyTransport;
this.name = name;
this.settings = settings;
}
@Override
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline channelPipeline = Channels.pipeline();
channelPipeline.addLast("openChannels", nettyTransport.serverOpenChannels);
Netty3SizeHeaderFrameDecoder sizeHeader = new Netty3SizeHeaderFrameDecoder();
if (nettyTransport.maxCumulationBufferCapacity.getBytes() > 0) {
if (nettyTransport.maxCumulationBufferCapacity.getBytes() > Integer.MAX_VALUE) {
sizeHeader.setMaxCumulationBufferCapacity(Integer.MAX_VALUE);
} else {
sizeHeader.setMaxCumulationBufferCapacity((int) nettyTransport.maxCumulationBufferCapacity.getBytes());
}
}
if (nettyTransport.maxCompositeBufferComponents != -1) {
sizeHeader.setMaxCumulationBufferComponents(nettyTransport.maxCompositeBufferComponents);
}
channelPipeline.addLast("size", sizeHeader);
channelPipeline.addLast("dispatcher", new Netty3MessageChannelHandler(nettyTransport, name));
return channelPipeline;
}
}
protected class ChannelCloseListener implements ChannelFutureListener {
private final DiscoveryNode node;
private ChannelCloseListener(DiscoveryNode node) {
this.node = node;
}
@Override
public void operationComplete(final ChannelFuture future) throws Exception {
NodeChannels nodeChannels = connectedNodes.get(node);
if (nodeChannels != null && nodeChannels.hasChannel(future.getChannel())) {
threadPool.generic().execute(() -> {
disconnectFromNode(node, future.getChannel(), "channel closed event");
});
}
}
}
@Override
protected void sendMessage(Channel channel, BytesReference reference, Runnable sendListener) {
final ChannelFuture future = channel.write(Netty3Utils.toChannelBuffer(reference));
future.addListener(future1 -> sendListener.run());
}
@Override
protected void closeChannels(List<Channel> channels) {
List<ChannelFuture> futures = new ArrayList<>();
for (Channel channel : channels) {
try {
if (channel != null && channel.isOpen()) {
futures.add(channel.close());
}
} catch (Exception e) {
logger.trace("failed to close channel", e);
}
}
for (ChannelFuture future : futures) {
future.awaitUninterruptibly();
}
}
@Override
protected InetSocketAddress getLocalAddress(Channel channel) {
return (InetSocketAddress) channel.getLocalAddress();
}
@Override
protected Channel bind(String name, InetSocketAddress address) {
return serverBootstraps.get(name).bind(address);
}
ScheduledPing getPing() {
return scheduledPing;
}
@Override
protected boolean isOpen(Channel channel) {
return channel.isOpen();
}
@Override
protected void stopInternal() {
Releasables.close(serverOpenChannels, () ->{
for (Map.Entry<String, ServerBootstrap> entry : serverBootstraps.entrySet()) {
String name = entry.getKey();
ServerBootstrap serverBootstrap = entry.getValue();
try {
serverBootstrap.releaseExternalResources();
} catch (Exception e) {
logger.debug((Supplier<?>) () -> new ParameterizedMessage("Error closing serverBootstrap for profile [{}]", name), e);
}
}
serverBootstraps.clear();
if (clientBootstrap != null) {
clientBootstrap.releaseExternalResources();
clientBootstrap = null;
}
});
}
}

171
modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Utils.java
View File

@ -1,171 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.logging.Loggers;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.logging.InternalLogger;
import org.jboss.netty.logging.InternalLoggerFactory;
import org.jboss.netty.util.ThreadNameDeterminer;
import org.jboss.netty.util.ThreadRenamingRunnable;
import java.io.IOException;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.ArrayList;
public class Netty3Utils {
/**
* Here we go....
* <p>
* When using the socket or file channel API to write or read using heap ByteBuffer, the sun.nio
* package will convert it to a direct buffer before doing the actual operation. The direct buffer is
* cached on an array of buffers under the nio.ch.Util$BufferCache on a thread local.
* <p>
* In netty specifically, if we send a single ChannelBuffer that is bigger than
* SocketSendBufferPool#DEFAULT_PREALLOCATION_SIZE (64kb), it will just convert the ChannelBuffer
* to a ByteBuffer and send it. The problem is, that then same size DirectByteBuffer will be
* allocated (or reused) and kept around on a thread local in the sun.nio BufferCache. If very
* large buffer is sent, imagine a 10mb one, then a 10mb direct buffer will be allocated as an
* entry within the thread local buffers.
* <p>
* In ES, we try and page the buffers allocated, all serialized data uses {@link org.elasticsearch.common.bytes.PagedBytesReference}
* typically generated from {@link org.elasticsearch.common.io.stream.BytesStreamOutput}. When sending it over
* to netty, it creates a {@link org.jboss.netty.buffer.CompositeChannelBuffer} that wraps the relevant pages.
* <p>
* The idea with the usage of composite channel buffer is that a single large buffer will not be sent over
* to the sun.nio layer. But, this will only happen if the composite channel buffer is created with a gathering
* flag set to true. In such a case, the GatheringSendBuffer is used in netty, resulting in calling the sun.nio
* layer with a ByteBuffer array.
* <p>
* This, potentially would have been as disastrous if the sun.nio layer would have tried to still copy over
* all of it to a direct buffer. But, the write(ByteBuffer[]) API (see sun.nio.ch.IOUtil), goes one buffer
* at a time, and gets a temporary direct buffer from the BufferCache, up to a limit of IOUtil#IOV_MAX (which
* is 1024 on most OSes). This means that there will be a max of 1024 direct buffer per thread.
* <p>
* This is still less than optimal to be honest, since it means that if not all data was written successfully
* (1024 paged buffers), then the rest of the data will need to be copied over again to the direct buffer
* and re-transmitted, but its much better than trying to send the full large buffer over and over again.
* <p>
* In ES, we use by default, in our paged data structures, a page of 16kb, so this is not so terrible.
* <p>
* Note, on the read size of netty, it uses a single direct buffer that is defined in both the transport
* and http configuration (based on the direct memory available), and the upstream handlers (SizeHeaderFrameDecoder,
* or more specifically the FrameDecoder base class) makes sure to use a cumulation buffer and not copy it
* over all the time.
* <p>
* TODO: potentially, a more complete solution would be to write a netty channel handler that is the last
* in the pipeline, and if the buffer is composite, verifies that its a gathering one with reasonable
* sized pages, and if its a single one, makes sure that it gets sliced and wrapped in a composite
* buffer.
*/
public static final boolean DEFAULT_GATHERING = true;
private static EsThreadNameDeterminer ES_THREAD_NAME_DETERMINER = new EsThreadNameDeterminer();
public static class EsThreadNameDeterminer implements ThreadNameDeterminer {
@Override
public String determineThreadName(String currentThreadName, String proposedThreadName) throws Exception {
// we control the thread name with a context, so use both
return currentThreadName + "{" + proposedThreadName + "}";
}
}
static {
InternalLoggerFactory.setDefaultFactory(new InternalLoggerFactory() {
@Override
public InternalLogger newInstance(String name) {
return new Netty3InternalESLogger(Loggers.getLogger(name));
}
});
ThreadRenamingRunnable.setThreadNameDeterminer(ES_THREAD_NAME_DETERMINER);
// Netty 3 SelectorUtil wants to set this; however, it does not execute the property write
// in a privileged block so we just do what Netty wants to do here
final String key = "sun.nio.ch.bugLevel";
final String buglevel = System.getProperty(key);
if (buglevel == null) {
try {
AccessController.doPrivileged(new PrivilegedAction<Void>() {
@Override
@SuppressForbidden(reason = "to use System#setProperty to set sun.nio.ch.bugLevel")
public Void run() {
System.setProperty(key, "");
return null;
}
});
} catch (final SecurityException e) {
Loggers
.getLogger(Netty3Utils.class)
.debug((Supplier<?>) () -> new ParameterizedMessage("Unable to get/set System Property: {}", key), e);
}
}
}
public static void setup() {
}
/**
* Turns the given BytesReference into a ChannelBuffer. Note: the returned ChannelBuffer will reference the internal
* pages of the BytesReference. Don't free the bytes of reference before the ChannelBuffer goes out of scope.
*/
public static ChannelBuffer toChannelBuffer(BytesReference reference) {
if (reference.length() == 0) {
return ChannelBuffers.EMPTY_BUFFER;
}
if (reference instanceof ChannelBufferBytesReference) {
return ((ChannelBufferBytesReference) reference).toChannelBuffer();
} else {
final BytesRefIterator iterator = reference.iterator();
BytesRef slice;
final ArrayList<ChannelBuffer> buffers = new ArrayList<>();
try {
while ((slice = iterator.next()) != null) {
buffers.add(ChannelBuffers.wrappedBuffer(slice.bytes, slice.offset, slice.length));
}
return ChannelBuffers.wrappedBuffer(DEFAULT_GATHERING, buffers.toArray(new ChannelBuffer[buffers.size()]));
} catch (IOException ex) {
throw new AssertionError("no IO happens here", ex);
}
}
}
/**
* Wraps the given ChannelBuffer with a BytesReference
*/
public static BytesReference toBytesReference(ChannelBuffer channelBuffer) {
return toBytesReference(channelBuffer, channelBuffer.readableBytes());
}
/**
* Wraps the given ChannelBuffer with a BytesReference of a given size
*/
public static BytesReference toBytesReference(ChannelBuffer channelBuffer, int size) {
return new ChannelBufferBytesReference(channelBuffer, size);
}
}

24
modules/transport-netty3/src/main/plugin-metadata/plugin-security.policy
View File

@ -1,24 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
grant {
// Netty SelectorUtil wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
// the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
};

72
modules/transport-netty3/src/test/java/org/elasticsearch/ESNetty3IntegTestCase.java
View File

@ -1,72 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.transport.Netty3Plugin;
import org.elasticsearch.transport.netty3.Netty3Transport;
import java.util.Arrays;
import java.util.Collection;
public abstract class ESNetty3IntegTestCase extends ESIntegTestCase {
@Override
protected boolean ignoreExternalCluster() {
return true;
}
@Override
protected boolean addMockTransportService() {
return false;
}
@Override
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal));
// randomize netty settings
if (randomBoolean()) {
builder.put(Netty3Transport.WORKER_COUNT.getKey(), random().nextInt(3) + 1);
}
builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME);
builder.put(NetworkModule.HTTP_TYPE_KEY, Netty3Plugin.NETTY_HTTP_TRANSPORT_NAME);
return builder.build();
}
@Override
protected Settings transportClientSettings() {
Settings.Builder builder = Settings.builder().put(super.transportClientSettings());
builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME);
return builder.build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Arrays.asList(Netty3Plugin.class);
}
@Override
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
return Arrays.asList(Netty3Plugin.class);
}
}

45
modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3ClientYamlTestSuiteIT.java
View File

@ -1,45 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import org.apache.lucene.util.TimeUnits;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
import java.io.IOException;
// TODO: remove timeout after address slow test issue
@TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
public class Netty3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
public Netty3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
return ESClientYamlSuiteTestCase.createParameters();
}
}

452
modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpChannelTests.java
View File

@ -1,452 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.MockBigArrays;
import org.elasticsearch.http.HttpTransportSettings;
import org.elasticsearch.http.netty3.cors.Netty3CorsHandler;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelConfig;
import org.jboss.netty.channel.ChannelFactory;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.handler.codec.http.DefaultHttpHeaders;
import org.jboss.netty.handler.codec.http.HttpHeaders;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.jboss.netty.handler.codec.http.HttpResponse;
import org.jboss.netty.handler.codec.http.HttpVersion;
import org.junit.After;
import org.junit.Before;
import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
public class Netty3HttpChannelTests extends ESTestCase {
private NetworkService networkService;
private ThreadPool threadPool;
private MockBigArrays bigArrays;
private Netty3HttpServerTransport httpServerTransport;
@Before
public void setup() throws Exception {
networkService = new NetworkService(Settings.EMPTY, Collections.emptyList());
threadPool = new TestThreadPool("test");
bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
}
@After
public void shutdown() throws Exception {
if (threadPool != null) {
threadPool.shutdownNow();
}
if (httpServerTransport != null) {
httpServerTransport.close();
}
}
public void testCorsEnabledWithoutAllowOrigins() {
// Set up a HTTP transport with only the CORS enabled setting
Settings settings = Settings.builder()
.put(HttpTransportSettings.SETTING_CORS_ENABLED.getKey(), true)
.build();
HttpResponse response = execRequestWithCors(settings, "remote-host", "request-host");
// inspect response and validate
assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), nullValue());
}
public void testCorsEnabledWithAllowOrigins() {
final String originValue = "remote-host";
// create a http transport with CORS enabled and allow origin configured
Settings settings = Settings.builder()
.put(SETTING_CORS_ENABLED.getKey(), true)
.put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue)
.build();
HttpResponse response = execRequestWithCors(settings, originValue, "request-host");
// inspect response and validate
assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
String allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
}
public void testCorsAllowOriginWithSameHost() {
String originValue = "remote-host";
String host = "remote-host";
// create a http transport with CORS enabled
Settings settings = Settings.builder()
.put(SETTING_CORS_ENABLED.getKey(), true)
.build();
HttpResponse response = execRequestWithCors(settings, originValue, host);
// inspect response and validate
assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
String allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
originValue = "http://" + originValue;
response = execRequestWithCors(settings, originValue, host);
assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
originValue = originValue + ":5555";
host = host + ":5555";
response = execRequestWithCors(settings, originValue, host);
assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
originValue = originValue.replace("http", "https");
response = execRequestWithCors(settings, originValue, host);
assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
}
public void testThatStringLiteralWorksOnMatch() {
final String originValue = "remote-host";
Settings settings = Settings.builder()
.put(SETTING_CORS_ENABLED.getKey(), true)
.put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue)
.put(SETTING_CORS_ALLOW_METHODS.getKey(), "get, options, post")
.put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
.build();
HttpResponse response = execRequestWithCors(settings, originValue, "request-host");
// inspect response and validate
assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
String allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_CREDENTIALS), equalTo("true"));
}
public void testThatAnyOriginWorks() {
final String originValue = Netty3CorsHandler.ANY_ORIGIN;
Settings settings = Settings.builder()
.put(SETTING_CORS_ENABLED.getKey(), true)
.put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue)
.build();
HttpResponse response = execRequestWithCors(settings, originValue, "request-host");
// inspect response and validate
assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
String allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
assertThat(allowedOrigins, is(originValue));
assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_CREDENTIALS), nullValue());
}
public void testHeadersSet() {
Settings settings = Settings.builder().build();
httpServerTransport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool);
HttpRequest httpRequest = new TestHttpRequest();
httpRequest.headers().add(HttpHeaders.Names.ORIGIN, "remote");
WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel();
Netty3HttpRequest request = new Netty3HttpRequest(httpRequest, writeCapturingChannel);
// send a response
Netty3HttpChannel channel =
new Netty3HttpChannel(httpServerTransport, request, null, randomBoolean(), threadPool.getThreadContext());
TestReponse resp = new TestReponse();
final String customHeader = "custom-header";
final String customHeaderValue = "xyz";
resp.addHeader(customHeader, customHeaderValue);
channel.sendResponse(resp);
// inspect what was written
List<Object> writtenObjects = writeCapturingChannel.getWrittenObjects();
assertThat(writtenObjects.size(), is(1));
HttpResponse response = (HttpResponse) writtenObjects.get(0);
assertThat(response.headers().get("non-existent-header"), nullValue());
assertThat(response.headers().get(customHeader), equalTo(customHeaderValue));
assertThat(response.headers().get(HttpHeaders.Names.CONTENT_LENGTH), equalTo(Integer.toString(resp.content().length())));
assertThat(response.headers().get(HttpHeaders.Names.CONTENT_TYPE), equalTo(resp.contentType()));
}
private HttpResponse execRequestWithCors(final Settings settings, final String originValue, final String host) {
// construct request and send it over the transport layer
httpServerTransport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool);
HttpRequest httpRequest = new TestHttpRequest();
httpRequest.headers().add(HttpHeaders.Names.ORIGIN, originValue);
httpRequest.headers().add(HttpHeaders.Names.HOST, host);
WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel();
Netty3HttpRequest request = new Netty3HttpRequest(httpRequest, writeCapturingChannel);
Netty3HttpChannel channel =
new Netty3HttpChannel(httpServerTransport, request, null, randomBoolean(), threadPool.getThreadContext());
channel.sendResponse(new TestReponse());
// get the response
List<Object> writtenObjects = writeCapturingChannel.getWrittenObjects();
assertThat(writtenObjects.size(), is(1));
return (HttpResponse) writtenObjects.get(0);
}
private static class WriteCapturingChannel implements Channel {
private List<Object> writtenObjects = new ArrayList<>();
@Override
public Integer getId() {
return null;
}
@Override
public ChannelFactory getFactory() {
return null;
}
@Override
public Channel getParent() {
return null;
}
@Override
public ChannelConfig getConfig() {
return null;
}
@Override
public ChannelPipeline getPipeline() {
return null;
}
@Override
public boolean isOpen() {
return false;
}
@Override
public boolean isBound() {
return false;
}
@Override
public boolean isConnected() {
return false;
}
@Override
public SocketAddress getLocalAddress() {
return null;
}
@Override
public SocketAddress getRemoteAddress() {
return null;
}
@Override
public ChannelFuture write(Object message) {
writtenObjects.add(message);
return null;
}
@Override
public ChannelFuture write(Object message, SocketAddress remoteAddress) {
writtenObjects.add(message);
return null;
}
@Override
public ChannelFuture bind(SocketAddress localAddress) {
return null;
}
@Override
public ChannelFuture connect(SocketAddress remoteAddress) {
return null;
}
@Override
public ChannelFuture disconnect() {
return null;
}
@Override
public ChannelFuture unbind() {
return null;
}
@Override
public ChannelFuture close() {
return null;
}
@Override
public ChannelFuture getCloseFuture() {
return null;
}
@Override
public int getInterestOps() {
return 0;
}
@Override
public boolean isReadable() {
return false;
}
@Override
public boolean isWritable() {
return false;
}
@Override
public ChannelFuture setInterestOps(int interestOps) {
return null;
}
@Override
public ChannelFuture setReadable(boolean readable) {
return null;
}
@Override
public boolean getUserDefinedWritability(int index) {
return false;
}
@Override
public void setUserDefinedWritability(int index, boolean isWritable) {
}
@Override
public Object getAttachment() {
return null;
}
@Override
public void setAttachment(Object attachment) {
}
@Override
public int compareTo(Channel o) {
return 0;
}
public List<Object> getWrittenObjects() {
return writtenObjects;
}
}
private static class TestHttpRequest implements HttpRequest {
private HttpHeaders headers = new DefaultHttpHeaders();
private ChannelBuffer content = ChannelBuffers.EMPTY_BUFFER;
@Override
public HttpMethod getMethod() {
return null;
}
@Override
public void setMethod(HttpMethod method) {
}
@Override
public String getUri() {
return "";
}
@Override
public void setUri(String uri) {
}
@Override
public HttpVersion getProtocolVersion() {
return HttpVersion.HTTP_1_1;
}
@Override
public void setProtocolVersion(HttpVersion version) {
}
@Override
public HttpHeaders headers() {
return headers;
}
@Override
public ChannelBuffer getContent() {
return content;
}
@Override
public void setContent(ChannelBuffer content) {
this.content = content;
}
@Override
public boolean isChunked() {
return false;
}
@Override
public void setChunked(boolean chunked) {
}
}
private static class TestReponse extends RestResponse {
@Override
public String contentType() {
return "text";
}
@Override
public BytesReference content() {
return BytesArray.EMPTY;
}
@Override
public RestStatus status() {
return RestStatus.OK;
}
}
}

193
modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpClient.java
View File

@ -1,193 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.jboss.netty.bootstrap.ClientBootstrap;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
import org.jboss.netty.handler.codec.http.DefaultHttpRequest;
import org.jboss.netty.handler.codec.http.HttpChunkAggregator;
import org.jboss.netty.handler.codec.http.HttpClientCodec;
import org.jboss.netty.handler.codec.http.HttpHeaders;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.jboss.netty.handler.codec.http.HttpResponse;
import java.io.Closeable;
import java.net.SocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.HOST;
import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1;
/**
* Tiny helper to send http requests over netty3.
*/
public class Netty3HttpClient implements Closeable {
public static Collection<String> returnHttpResponseBodies(Collection<HttpResponse> responses) {
List<String> list = new ArrayList<>(responses.size());
for (HttpResponse response : responses) {
list.add(response.getContent().toString(StandardCharsets.UTF_8));
}
return list;
}
public static Collection<String> returnOpaqueIds(Collection<HttpResponse> responses) {
List<String> list = new ArrayList<>(responses.size());
for (HttpResponse response : responses) {
list.add(response.headers().get("X-Opaque-Id"));
}
return list;
}
private final ClientBootstrap clientBootstrap;
public Netty3HttpClient() {
clientBootstrap = new ClientBootstrap(new NioClientSocketChannelFactory());;
}
public Collection<HttpResponse> get(SocketAddress remoteAddress, String... uris) throws InterruptedException {
Collection<HttpRequest> requests = new ArrayList<>(uris.length);
for (int i = 0; i < uris.length; i++) {
final HttpRequest httpRequest = new DefaultHttpRequest(HTTP_1_1, HttpMethod.GET, uris[i]);
httpRequest.headers().add(HOST, "localhost");
httpRequest.headers().add("X-Opaque-ID", String.valueOf(i));
requests.add(httpRequest);
}
return sendRequests(remoteAddress, requests);
}
@SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods.
public final Collection<HttpResponse> post(SocketAddress remoteAddress, Tuple<String, CharSequence>... urisAndBodies)
throws InterruptedException {
return processRequestsWithBody(HttpMethod.POST, remoteAddress, urisAndBodies);
}
@SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods.
public final Collection<HttpResponse> put(SocketAddress remoteAddress, Tuple<String, CharSequence>... urisAndBodies)
throws InterruptedException {
return processRequestsWithBody(HttpMethod.PUT, remoteAddress, urisAndBodies);
}
@SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods.
final Collection<HttpResponse> processRequestsWithBody(HttpMethod method, SocketAddress remoteAddress, Tuple<String,
CharSequence>... urisAndBodies) throws InterruptedException {
Collection<HttpRequest> requests = new ArrayList<>(urisAndBodies.length);
for (Tuple<String, CharSequence> uriAndBody : urisAndBodies) {
ChannelBuffer content = ChannelBuffers.copiedBuffer(uriAndBody.v2(), StandardCharsets.UTF_8);
HttpRequest request = new DefaultHttpRequest(HTTP_1_1, method, uriAndBody.v1());
request.headers().add(HOST, "localhost");
request.headers().add(HttpHeaders.Names.CONTENT_LENGTH, content.readableBytes());
request.setContent(content);
requests.add(request);
}
return sendRequests(remoteAddress, requests);
}
private synchronized Collection<HttpResponse> sendRequests(SocketAddress remoteAddress, Collection<HttpRequest> requests)
throws InterruptedException {
final CountDownLatch latch = new CountDownLatch(requests.size());
final Collection<HttpResponse> content = Collections.synchronizedList(new ArrayList<>(requests.size()));
clientBootstrap.setPipelineFactory(new CountDownLatchPipelineFactory(latch, content));
ChannelFuture channelFuture = null;
try {
channelFuture = clientBootstrap.connect(remoteAddress);
channelFuture.await(1000);
for (HttpRequest request : requests) {
channelFuture.getChannel().write(request);
}
latch.await();
} finally {
if (channelFuture != null) {
channelFuture.getChannel().close();
}
}
return content;
}
@Override
public void close() {
clientBootstrap.shutdown();
clientBootstrap.releaseExternalResources();
}
/**
* helper factory which adds returned data to a list and uses a count down latch to decide when done
*/
public static class CountDownLatchPipelineFactory implements ChannelPipelineFactory {
private final CountDownLatch latch;
private final Collection<HttpResponse> content;
public CountDownLatchPipelineFactory(CountDownLatch latch, Collection<HttpResponse> content) {
this.latch = latch;
this.content = content;
}
@Override
public ChannelPipeline getPipeline() throws Exception {
final int maxBytes = new ByteSizeValue(100, ByteSizeUnit.MB).bytesAsInt();
return Channels.pipeline(
new HttpClientCodec(),
new HttpChunkAggregator(maxBytes),
new SimpleChannelUpstreamHandler() {
@Override
public void messageReceived(final ChannelHandlerContext ctx, final MessageEvent e) {
final Object message = e.getMessage();
if (message instanceof HttpResponse) {
HttpResponse response = (HttpResponse) message;
content.add(response);
}
latch.countDown();
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
super.exceptionCaught(ctx, e);
latch.countDown();
}
});
}
}
}

91
modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpPublishPortTests.java
View File

@ -1,91 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import org.elasticsearch.common.network.NetworkUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.http.BindHttpException;
import org.elasticsearch.http.HttpTransportSettings;
import org.elasticsearch.test.ESTestCase;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import static java.net.InetAddress.getByName;
import static java.util.Arrays.asList;
import static org.elasticsearch.http.netty3.Netty3HttpServerTransport.resolvePublishPort;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
public class Netty3HttpPublishPortTests extends ESTestCase {
public void testHttpPublishPort() throws Exception {
int boundPort = randomIntBetween(9000, 9100);
int otherBoundPort = randomIntBetween(9200, 9300);
int publishPort = resolvePublishPort(Settings.builder().put(HttpTransportSettings.SETTING_HTTP_PUBLISH_PORT.getKey(), 9080).build(),
randomAddresses(), getByName("127.0.0.2"));
assertThat("Publish port should be explicitly set to 9080", publishPort, equalTo(9080));
publishPort = resolvePublishPort(Settings.EMPTY, asList(address("127.0.0.1", boundPort), address("127.0.0.2", otherBoundPort)),
getByName("127.0.0.1"));
assertThat("Publish port should be derived from matched address", publishPort, equalTo(boundPort));
publishPort = resolvePublishPort(Settings.EMPTY, asList(address("127.0.0.1", boundPort), address("127.0.0.2", boundPort)),
getByName("127.0.0.3"));
assertThat("Publish port should be derived from unique port of bound addresses", publishPort, equalTo(boundPort));
try {
resolvePublishPort(Settings.EMPTY, asList(address("127.0.0.1", boundPort), address("127.0.0.2", otherBoundPort)),
getByName("127.0.0.3"));
fail("Expected BindHttpException as publish_port not specified and non-unique port of bound addresses");
} catch (BindHttpException e) {
assertThat(e.getMessage(), containsString("Failed to auto-resolve http publish port"));
}
publishPort = resolvePublishPort(Settings.EMPTY, asList(address("0.0.0.0", boundPort), address("127.0.0.2", otherBoundPort)),
getByName("127.0.0.1"));
assertThat("Publish port should be derived from matching wildcard address", publishPort, equalTo(boundPort));
if (NetworkUtils.SUPPORTS_V6) {
publishPort = resolvePublishPort(Settings.EMPTY, asList(address("0.0.0.0", boundPort), address("127.0.0.2", otherBoundPort)),
getByName("::1"));
assertThat("Publish port should be derived from matching wildcard address", publishPort, equalTo(boundPort));
}
}
private TransportAddress address(String host, int port) throws UnknownHostException {
return new TransportAddress(getByName(host), port);
}
private TransportAddress randomAddress() throws UnknownHostException {
return address("127.0.0." + randomIntBetween(1, 100), randomIntBetween(9200, 9300));
}
private List<TransportAddress> randomAddresses() throws UnknownHostException {
List<TransportAddress> addresses = new ArrayList<>();
for (int i = 0; i < randomIntBetween(1, 5); i++) {
addresses.add(randomAddress());
}
return addresses;
}
}

126
modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpRequestSizeLimitIT.java
View File

@ -1,126 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import org.elasticsearch.ESNetty3IntegTestCase;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.http.HttpServerTransport;
import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.jboss.netty.handler.codec.http.HttpResponse;
import org.jboss.netty.handler.codec.http.HttpResponseStatus;
import java.util.Collection;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.hasSize;
/**
* This test checks that in-flight requests are limited on HTTP level and that requests that are excluded from limiting can pass.
*
* As the same setting is also used to limit in-flight requests on transport level, we avoid transport messages by forcing
* a single node "cluster". We also force test infrastructure to use the node client instead of the transport client for the same reason.
*/
@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numClientNodes = 0, numDataNodes = 1, transportClientRatio = 0)
public class Netty3HttpRequestSizeLimitIT extends ESNetty3IntegTestCase {
private static final ByteSizeValue LIMIT = new ByteSizeValue(2, ByteSizeUnit.KB);
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(NetworkModule.HTTP_ENABLED.getKey(), true)
.put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), LIMIT)
.build();
}
public void testLimitsInFlightRequests() throws Exception {
ensureGreen();
// we use the limit size as a (very) rough indication on how many requests we should sent to hit the limit
int numRequests = LIMIT.bytesAsInt() / 100;
StringBuilder bulkRequest = new StringBuilder();
for (int i = 0; i < numRequests; i++) {
bulkRequest.append("{\"index\": {}}");
bulkRequest.append(System.lineSeparator());
bulkRequest.append("{ \"field\" : \"value\" }");
bulkRequest.append(System.lineSeparator());
}
@SuppressWarnings("unchecked")
Tuple<String, CharSequence>[] requests = new Tuple[150];
for (int i = 0; i < requests.length; i++) {
requests[i] = Tuple.tuple("/index/type/_bulk", bulkRequest);
}
HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class);
TransportAddress transportAddress = (TransportAddress) randomFrom(httpServerTransport.boundAddress
().boundAddresses());
try (Netty3HttpClient nettyHttpClient = new Netty3HttpClient()) {
Collection<HttpResponse> singleResponse = nettyHttpClient.post(transportAddress.address(), requests[0]);
assertThat(singleResponse, hasSize(1));
assertAtLeastOnceExpectedStatus(singleResponse, HttpResponseStatus.OK);
Collection<HttpResponse> multipleResponses = nettyHttpClient.post(transportAddress.address(), requests);
assertThat(multipleResponses, hasSize(requests.length));
assertAtLeastOnceExpectedStatus(multipleResponses, HttpResponseStatus.SERVICE_UNAVAILABLE);
}
}
public void testDoesNotLimitExcludedRequests() throws Exception {
ensureGreen();
@SuppressWarnings("unchecked")
Tuple<String, CharSequence>[] requestUris = new Tuple[1500];
for (int i = 0; i < requestUris.length; i++) {
requestUris[i] = Tuple.tuple("/_cluster/settings",
"{ \"transient\": {\"indices.ttl.interval\": \"40s\" } }");
}
HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class);
TransportAddress transportAddress = (TransportAddress) randomFrom(httpServerTransport.boundAddress
().boundAddresses());
try (Netty3HttpClient nettyHttpClient = new Netty3HttpClient()) {
Collection<HttpResponse> responses = nettyHttpClient.put(transportAddress.address(), requestUris);
assertThat(responses, hasSize(requestUris.length));
assertAllInExpectedStatus(responses, HttpResponseStatus.OK);
}
}
private void assertAtLeastOnceExpectedStatus(Collection<HttpResponse> responses, HttpResponseStatus expectedStatus) {
long countExpectedStatus = responses.stream().filter(r -> r.getStatus().equals(expectedStatus)).count();
assertThat("Expected at least one request with status [" + expectedStatus + "]", countExpectedStatus, greaterThan(0L));
}
private void assertAllInExpectedStatus(Collection<HttpResponse> responses, HttpResponseStatus expectedStatus) {
long countUnexpectedStatus = responses.stream().filter(r -> r.getStatus().equals(expectedStatus) == false).count();
assertThat("Expected all requests with status [" + expectedStatus + "] but [" + countUnexpectedStatus +
"] requests had a different one", countUnexpectedStatus, equalTo(0L));
}
}

265
modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerPipeliningTests.java
View File

@ -1,265 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.util.MockBigArrays;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.http.netty3.Netty3HttpServerTransport.HttpChannelPipelineFactory;
import org.elasticsearch.http.netty3.pipelining.OrderedDownstreamChannelEvent;
import org.elasticsearch.http.netty3.pipelining.OrderedUpstreamMessageEvent;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.jboss.netty.handler.codec.http.HttpResponse;
import org.junit.After;
import org.junit.Before;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static org.elasticsearch.http.netty3.Netty3HttpClient.returnHttpResponseBodies;
import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.hasSize;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK;
import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1;
/**
* This test just tests, if he pipelining works in general with out any connection the elasticsearch handler
*/
public class Netty3HttpServerPipeliningTests extends ESTestCase {
private NetworkService networkService;
private ThreadPool threadPool;
private MockBigArrays bigArrays;
private CustomNetty3HttpServerTransport httpServerTransport;
@Before
public void setup() throws Exception {
networkService = new NetworkService(Settings.EMPTY, Collections.emptyList());
threadPool = new TestThreadPool("test");
bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
}
@After
public void shutdown() throws Exception {
if (threadPool != null) {
threadPool.shutdownNow();
}
if (httpServerTransport != null) {
httpServerTransport.close();
}
}
public void testThatHttpPipeliningWorksWhenEnabled() throws Exception {
Settings settings = Settings.builder()
.put("http.pipelining", true)
.put("http.port", "0")
.build();
httpServerTransport = new CustomNetty3HttpServerTransport(settings);
httpServerTransport.start();
TransportAddress transportAddress = randomFrom(httpServerTransport.boundAddress().boundAddresses());
final int numberOfRequests = randomIntBetween(4, 16);
final List<String> requests = new ArrayList<>(numberOfRequests);
for (int i = 0; i < numberOfRequests; i++) {
if (rarely()) {
requests.add("/slow/" + i);
} else {
requests.add("/" + i);
}
}
try (Netty3HttpClient nettyHttpClient = new Netty3HttpClient()) {
Collection<HttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests.toArray(new String[]{}));
Collection<String> responseBodies = returnHttpResponseBodies(responses);
assertThat(responseBodies, contains(requests.toArray()));
}
}
public void testThatHttpPipeliningCanBeDisabled() throws Exception {
Settings settings = Settings.builder()
.put("http.pipelining", false)
.put("http.port", "0")
.build();
httpServerTransport = new CustomNetty3HttpServerTransport(settings);
httpServerTransport.start();
TransportAddress transportAddress = randomFrom(httpServerTransport.boundAddress().boundAddresses());
final int numberOfRequests = randomIntBetween(4, 16);
final Set<Integer> slowIds = new HashSet<>();
final List<String> requests = new ArrayList<>(numberOfRequests);
for (int i = 0; i < numberOfRequests; i++) {
if (rarely()) {
requests.add("/slow/" + i);
slowIds.add(i);
} else {
requests.add("/" + i);
}
}
try (Netty3HttpClient nettyHttpClient = new Netty3HttpClient()) {
Collection<HttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests.toArray(new String[]{}));
List<String> responseBodies = new ArrayList<>(returnHttpResponseBodies(responses));
// we cannot be sure about the order of the responses, but the slow ones should come last
assertThat(responseBodies, hasSize(numberOfRequests));
for (int i = 0; i < numberOfRequests - slowIds.size(); i++) {
assertThat(responseBodies.get(i), matches("/\\d+"));
}
final Set<Integer> ids = new HashSet<>();
for (int i = 0; i < slowIds.size(); i++) {
final String response = responseBodies.get(numberOfRequests - slowIds.size() + i);
assertThat(response, matches("/slow/\\d+"));
assertTrue(ids.add(Integer.parseInt(response.split("/")[2])));
}
assertThat(ids, equalTo(slowIds));
}
}
class CustomNetty3HttpServerTransport extends Netty3HttpServerTransport {
private final ExecutorService executorService;
public CustomNetty3HttpServerTransport(Settings settings) {
super(settings, Netty3HttpServerPipeliningTests.this.networkService,
Netty3HttpServerPipeliningTests.this.bigArrays, Netty3HttpServerPipeliningTests.this.threadPool
);
this.executorService = Executors.newFixedThreadPool(5);
}
@Override
public ChannelPipelineFactory configureServerChannelPipelineFactory() {
return new CustomHttpChannelPipelineFactory(this, executorService, Netty3HttpServerPipeliningTests.this.threadPool
.getThreadContext());
}
@Override
public void stop() {
executorService.shutdownNow();
super.stop();
}
}
private class CustomHttpChannelPipelineFactory extends HttpChannelPipelineFactory {
private final ExecutorService executorService;
public CustomHttpChannelPipelineFactory(Netty3HttpServerTransport transport, ExecutorService executorService,
ThreadContext threadContext) {
super(transport, randomBoolean(), threadContext);
this.executorService = executorService;
}
@Override
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline pipeline = super.getPipeline();
pipeline.replace("handler", "handler", new PossiblySlowUpstreamHandler(executorService));
return pipeline;
}
}
class PossiblySlowUpstreamHandler extends SimpleChannelUpstreamHandler {
private final ExecutorService executorService;
public PossiblySlowUpstreamHandler(ExecutorService executorService) {
this.executorService = executorService;
}
@Override
public void messageReceived(final ChannelHandlerContext ctx, final MessageEvent e) throws Exception {
executorService.submit(new PossiblySlowRunnable(ctx, e));
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
logger.info("Caught exception", e.getCause());
e.getChannel().close();
}
}
class PossiblySlowRunnable implements Runnable {
private ChannelHandlerContext ctx;
private MessageEvent e;
public PossiblySlowRunnable(ChannelHandlerContext ctx, MessageEvent e) {
this.ctx = ctx;
this.e = e;
}
@Override
public void run() {
HttpRequest request;
OrderedUpstreamMessageEvent oue = null;
if (e instanceof OrderedUpstreamMessageEvent) {
oue = (OrderedUpstreamMessageEvent) e;
request = (HttpRequest) oue.getMessage();
} else {
request = (HttpRequest) e.getMessage();
}
ChannelBuffer buffer = ChannelBuffers.copiedBuffer(request.getUri(), StandardCharsets.UTF_8);
DefaultHttpResponse httpResponse = new DefaultHttpResponse(HTTP_1_1, OK);
httpResponse.headers().add(CONTENT_LENGTH, buffer.readableBytes());
httpResponse.setContent(buffer);
final boolean slow = request.getUri().matches("/slow/\\d+");
if (slow) {
try {
Thread.sleep(scaledRandomIntBetween(500, 1000));
} catch (InterruptedException e1) {
throw new RuntimeException(e1);
}
} else {
assert request.getUri().matches("/\\d+");
}
if (oue != null) {
ctx.sendDownstream(new OrderedDownstreamChannelEvent(oue, 0, true, httpResponse));
} else {
ctx.getChannel().write(httpResponse);
}
}
}
}

121
modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerTransportTests.java
View File

@ -1,121 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.util.MockBigArrays;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.http.BindHttpException;
import org.elasticsearch.http.netty3.cors.Netty3CorsConfig;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.junit.After;
import org.junit.Before;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
import static org.hamcrest.Matchers.equalTo;
/**
* Tests for the {@link Netty3HttpServerTransport} class.
*/
public class Netty3HttpServerTransportTests extends ESTestCase {
private NetworkService networkService;
private ThreadPool threadPool;
private MockBigArrays bigArrays;
@Before
public void setup() throws Exception {
networkService = new NetworkService(Settings.EMPTY, Collections.emptyList());
threadPool = new TestThreadPool("test");
bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
}
@After
public void shutdown() throws Exception {
if (threadPool != null) {
threadPool.shutdownNow();
}
threadPool = null;
networkService = null;
bigArrays = null;
}
public void testCorsConfig() {
final Set<String> methods = new HashSet<>(Arrays.asList("get", "options", "post"));
final Set<String> headers = new HashSet<>(Arrays.asList("Content-Type", "Content-Length"));
final Settings settings = Settings.builder()
.put(SETTING_CORS_ENABLED.getKey(), true)
.put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*")
.put(SETTING_CORS_ALLOW_METHODS.getKey(), Strings.collectionToCommaDelimitedString(methods))
.put(SETTING_CORS_ALLOW_HEADERS.getKey(), Strings.collectionToCommaDelimitedString(headers))
.put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
.build();
final Netty3HttpServerTransport transport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool);
final Netty3CorsConfig corsConfig = transport.getCorsConfig();
assertThat(corsConfig.isAnyOriginSupported(), equalTo(true));
assertThat(corsConfig.allowedRequestHeaders(), equalTo(headers));
assertThat(corsConfig.allowedRequestMethods().stream().map(HttpMethod::getName).collect(Collectors.toSet()), equalTo(methods));
transport.close();
}
public void testCorsConfigDefaults() {
final Set<String> headers = Sets.newHashSet("X-Requested-With", "Content-Type", "Content-Length");
final Set<String> methods = Sets.newHashSet("OPTIONS", "HEAD", "GET", "POST", "PUT", "DELETE");
final Settings settings = Settings.builder()
.put(SETTING_CORS_ENABLED.getKey(), true)
.put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*")
.put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
.build();
final Netty3HttpServerTransport transport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool);
final Netty3CorsConfig corsConfig = transport.getCorsConfig();
assertThat(corsConfig.allowedRequestHeaders(), equalTo(headers));
assertThat(corsConfig.allowedRequestMethods().stream().map(HttpMethod::getName).collect(Collectors.toSet()), equalTo(methods));
transport.close();
}
public void testBindUnavailableAddress() {
try (Netty3HttpServerTransport transport = new Netty3HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool)) {
transport.start();
TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
Settings settings = Settings.builder().put("http.port", remoteAddress.getPort()).build();
try (Netty3HttpServerTransport otherTransport = new Netty3HttpServerTransport(settings, networkService, bigArrays,
threadPool)) {
BindHttpException bindHttpException = expectThrows(BindHttpException.class, () -> otherTransport.start());
assertEquals("Failed to bind to [" + remoteAddress.getPort() + "]", bindHttpException.getMessage());
}
}
}
}

75
modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3PipeliningDisabledIT.java
View File

@ -1,75 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import org.elasticsearch.ESNetty3IntegTestCase;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.http.HttpServerTransport;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.jboss.netty.handler.codec.http.HttpResponse;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import static org.elasticsearch.http.netty3.Netty3HttpClient.returnOpaqueIds;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.hasSize;
@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
public class Netty3PipeliningDisabledIT extends ESNetty3IntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(NetworkModule.HTTP_ENABLED.getKey(), true)
.put("http.pipelining", false)
.build();
}
public void testThatNettyHttpServerDoesNotSupportPipelining() throws Exception {
ensureGreen();
String[] requests = new String[] {"/", "/_nodes/stats", "/", "/_cluster/state", "/", "/_nodes", "/"};
HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class);
TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses();
TransportAddress transportAddress = (TransportAddress) randomFrom(boundAddresses);
try (Netty3HttpClient nettyHttpClient = new Netty3HttpClient()) {
Collection<HttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests);
assertThat(responses, hasSize(requests.length));
List<String> opaqueIds = new ArrayList<>(returnOpaqueIds(responses));
assertResponsesOutOfOrder(opaqueIds);
}
}
/**
* checks if all responses are there, but also tests that they are out of order because pipelining is disabled
*/
private void assertResponsesOutOfOrder(List<String> opaqueIds) {
String message = String.format(Locale.ROOT, "Expected returned http message ids to be in any order of: %s", opaqueIds);
assertThat(message, opaqueIds, containsInAnyOrder("0", "1", "2", "3", "4", "5", "6"));
}
}

64
modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3PipeliningEnabledIT.java
View File

@ -1,64 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.transport.Netty3Plugin;
import org.jboss.netty.handler.codec.http.HttpResponse;
import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Collection;
import java.util.Locale;
import static org.elasticsearch.http.netty3.Netty3HttpClient.returnOpaqueIds;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
public class Netty3PipeliningEnabledIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
return Arrays.asList(Netty3Plugin.class);
}
public void testThatNettyHttpServerSupportsPipelining() throws Exception {
String[] requests = new String[]{"/", "/_nodes/stats", "/", "/_cluster/state", "/"};
InetSocketAddress inetSocketAddress = randomFrom(cluster().httpAddresses());
try (Netty3HttpClient nettyHttpClient = new Netty3HttpClient()) {
Collection<HttpResponse> responses = nettyHttpClient.get(inetSocketAddress, requests);
assertThat(responses, hasSize(5));
Collection<String> opaqueIds = returnOpaqueIds(responses);
assertOpaqueIdsInOrder(opaqueIds);
}
}
private void assertOpaqueIdsInOrder(Collection<String> opaqueIds) {
// check if opaque ids are monotonically increasing
int i = 0;
String msg = String.format(Locale.ROOT, "Expected list of opaque ids to be monotonically increasing, got [%s]", opaqueIds);
for (String opaqueId : opaqueIds) {
assertThat(msg, opaqueId, is(String.valueOf(i++)));
}
}
}

232
modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/pipelining/HttpPipeliningHandlerTests.java
View File

@ -1,232 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.http.netty3.pipelining;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.test.ESTestCase;
import org.jboss.netty.bootstrap.ClientBootstrap;
import org.jboss.netty.bootstrap.ServerBootstrap;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
import org.jboss.netty.handler.codec.http.DefaultHttpChunk;
import org.jboss.netty.handler.codec.http.DefaultHttpRequest;
import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
import org.jboss.netty.handler.codec.http.HttpChunk;
import org.jboss.netty.handler.codec.http.HttpClientCodec;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.jboss.netty.handler.codec.http.HttpRequestDecoder;
import org.jboss.netty.handler.codec.http.HttpResponse;
import org.jboss.netty.handler.codec.http.HttpResponseEncoder;
import org.jboss.netty.util.HashedWheelTimer;
import org.jboss.netty.util.Timeout;
import org.jboss.netty.util.TimerTask;
import org.junit.After;
import org.junit.Before;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.jboss.netty.buffer.ChannelBuffers.EMPTY_BUFFER;
import static org.jboss.netty.buffer.ChannelBuffers.copiedBuffer;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.HOST;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.TRANSFER_ENCODING;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Values.CHUNKED;
import static org.jboss.netty.handler.codec.http.HttpHeaders.Values.KEEP_ALIVE;
import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK;
import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1;
import static org.jboss.netty.util.CharsetUtil.UTF_8;
public class HttpPipeliningHandlerTests extends ESTestCase {
private static final long RESPONSE_TIMEOUT = 10000L;
private static final long CONNECTION_TIMEOUT = 10000L;
private static final String CONTENT_TYPE_TEXT = "text/plain; charset=UTF-8";
private static final String PATH1 = "/1";
private static final String PATH2 = "/2";
private static final String SOME_RESPONSE_TEXT = "some response for ";
private ClientBootstrap clientBootstrap;
private ServerBootstrap serverBootstrap;
private CountDownLatch responsesIn;
private final List<String> responses = new ArrayList<>(2);
private HashedWheelTimer timer;
private InetSocketAddress boundAddress;
@Before
public void startBootstraps() {
clientBootstrap = new ClientBootstrap(new NioClientSocketChannelFactory());
clientBootstrap.setPipelineFactory(new ChannelPipelineFactory() {
@Override
public ChannelPipeline getPipeline() throws Exception {
return Channels.pipeline(
new HttpClientCodec(),
new ClientHandler()
);
}
});
serverBootstrap = new ServerBootstrap(new NioServerSocketChannelFactory());
serverBootstrap.setPipelineFactory(new ChannelPipelineFactory() {
@Override
public ChannelPipeline getPipeline() throws Exception {
return Channels.pipeline(
new HttpRequestDecoder(),
new HttpResponseEncoder(),
new HttpPipeliningHandler(10000),
new ServerHandler()
);
}
});
Channel channel = serverBootstrap.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0));
boundAddress = (InetSocketAddress) channel.getLocalAddress();
timer = new HashedWheelTimer();
}
@After
public void releaseResources() {
timer.stop();
serverBootstrap.shutdown();
serverBootstrap.releaseExternalResources();
clientBootstrap.shutdown();
clientBootstrap.releaseExternalResources();
}
public void testShouldReturnMessagesInOrder() throws InterruptedException {
responsesIn = new CountDownLatch(1);
responses.clear();
final ChannelFuture connectionFuture = clientBootstrap.connect(boundAddress);
assertTrue(connectionFuture.await(CONNECTION_TIMEOUT));
final Channel clientChannel = connectionFuture.getChannel();
// NetworkAddress.format makes a proper HOST header.
final HttpRequest request1 = new DefaultHttpRequest(
HTTP_1_1, HttpMethod.GET, PATH1);
request1.headers().add(HOST, NetworkAddress.format(boundAddress));
final HttpRequest request2 = new DefaultHttpRequest(
HTTP_1_1, HttpMethod.GET, PATH2);
request2.headers().add(HOST, NetworkAddress.format(boundAddress));
clientChannel.write(request1);
clientChannel.write(request2);
responsesIn.await(RESPONSE_TIMEOUT, MILLISECONDS);
assertTrue(responses.contains(SOME_RESPONSE_TEXT + PATH1));
assertTrue(responses.contains(SOME_RESPONSE_TEXT + PATH2));
}
public class ClientHandler extends SimpleChannelUpstreamHandler {
@Override
public void messageReceived(final ChannelHandlerContext ctx, final MessageEvent e) {
final Object message = e.getMessage();
if (message instanceof HttpChunk) {
final HttpChunk response = (HttpChunk) e.getMessage();
if (!response.isLast()) {
final String content = response.getContent().toString(UTF_8);
responses.add(content);
if (content.equals(SOME_RESPONSE_TEXT + PATH2)) {
responsesIn.countDown();
}
}
}
}
}
public class ServerHandler extends SimpleChannelUpstreamHandler {
private final AtomicBoolean sendFinalChunk = new AtomicBoolean(false);
@Override
public void messageReceived(final ChannelHandlerContext ctx, final MessageEvent e) throws InterruptedException {
final HttpRequest request = (HttpRequest) e.getMessage();
final OrderedUpstreamMessageEvent oue = (OrderedUpstreamMessageEvent) e;
final String uri = request.getUri();
final HttpResponse initialChunk = new DefaultHttpResponse(HTTP_1_1, OK);
initialChunk.headers().add(CONTENT_TYPE, CONTENT_TYPE_TEXT);
initialChunk.headers().add(CONNECTION, KEEP_ALIVE);
initialChunk.headers().add(TRANSFER_ENCODING, CHUNKED);
ctx.sendDownstream(new OrderedDownstreamChannelEvent(oue, 0, false, initialChunk));
timer.newTimeout(new ChunkWriter(ctx, e, uri, oue, 1), 0, MILLISECONDS);
}
private class ChunkWriter implements TimerTask {
private final ChannelHandlerContext ctx;
private final MessageEvent e;
private final String uri;
private final OrderedUpstreamMessageEvent oue;
private final int subSequence;
public ChunkWriter(final ChannelHandlerContext ctx, final MessageEvent e, final String uri,
final OrderedUpstreamMessageEvent oue, final int subSequence) {
this.ctx = ctx;
this.e = e;
this.uri = uri;
this.oue = oue;
this.subSequence = subSequence;
}
@Override
public void run(final Timeout timeout) {
if (sendFinalChunk.get() && subSequence > 1) {
final HttpChunk finalChunk = new DefaultHttpChunk(EMPTY_BUFFER);
ctx.sendDownstream(new OrderedDownstreamChannelEvent(oue, subSequence, true, finalChunk));
} else {
final HttpChunk chunk = new DefaultHttpChunk(copiedBuffer(SOME_RESPONSE_TEXT + uri, UTF_8));
ctx.sendDownstream(new OrderedDownstreamChannelEvent(oue, subSequence, false, chunk));
timer.newTimeout(new ChunkWriter(ctx, e, uri, oue, subSequence + 1), 0, MILLISECONDS);
if (uri.equals(PATH2)) {
sendFinalChunk.set(true);
}
}
}
}
}
}

23
modules/transport-netty3/src/test/java/org/elasticsearch/rest/Netty3HeadBodyIsEmptyIT.java
View File

@ -1,23 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.rest;
public class Netty3HeadBodyIsEmptyIT extends HeadBodyIsEmptyIntegTestCase {
}

110
modules/transport-netty3/src/test/java/org/elasticsearch/transport/Netty3SizeHeaderFrameDecoderTests.java
View File

@ -1,110 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.MockBigArrays;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.netty3.Netty3Transport;
import org.junit.After;
import org.junit.Before;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import static org.hamcrest.Matchers.is;
/**
* This test checks, if a HTTP look-alike request (starting with a HTTP method and a space)
* actually returns text response instead of just dropping the connection
*/
public class Netty3SizeHeaderFrameDecoderTests extends ESTestCase {
private final Settings settings = Settings.builder()
.put("node.name", "Netty3SizeHeaderFrameDecoderTests")
.put(TransportSettings.BIND_HOST.getKey(), "127.0.0.1")
.put(TransportSettings.PORT.getKey(), "0")
.build();
private ThreadPool threadPool;
private Netty3Transport nettyTransport;
private int port;
private InetAddress host;
@Before
public void startThreadPool() {
threadPool = new ThreadPool(settings);
NetworkService networkService = new NetworkService(settings, Collections.emptyList());
BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
nettyTransport = new Netty3Transport(settings, threadPool, networkService, bigArrays,
new NamedWriteableRegistry(Collections.emptyList()), new NoneCircuitBreakerService());
nettyTransport.start();
TransportService transportService = new TransportService(settings, nettyTransport, threadPool,
TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
nettyTransport.transportServiceAdapter(transportService.createAdapter());
TransportAddress[] boundAddresses = nettyTransport.boundAddress().boundAddresses();
TransportAddress transportAddress = (TransportAddress) randomFrom(boundAddresses);
port = transportAddress.address().getPort();
host = transportAddress.address().getAddress();
}
@After
public void terminateThreadPool() throws InterruptedException {
nettyTransport.stop();
terminate(threadPool);
threadPool = null;
}
public void testThatTextMessageIsReturnedOnHTTPLikeRequest() throws Exception {
String randomMethod = randomFrom("GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH");
String data = randomMethod + " / HTTP/1.1";
try (Socket socket = new Socket(host, port)) {
socket.getOutputStream().write(data.getBytes(StandardCharsets.UTF_8));
socket.getOutputStream().flush();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))) {
assertThat(reader.readLine(), is("This is not a HTTP port"));
}
}
}
public void testThatNothingIsReturnedForOtherInvalidPackets() throws Exception {
try (Socket socket = new Socket(host, port)) {
socket.getOutputStream().write("FOOBAR".getBytes(StandardCharsets.UTF_8));
socket.getOutputStream().flush();
// end of stream
assertThat(socket.getInputStream().read(), is(-1));
}
}
}

78
modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/ChannelBufferBytesReferenceTests.java
View File

@ -1,78 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import java.io.IOException;
public class ChannelBufferBytesReferenceTests extends AbstractBytesReferenceTestCase {
@Override
protected BytesReference newBytesReference(int length) throws IOException {
ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
for (int i = 0; i < length; i++) {
out.writeByte((byte) random().nextInt(1 << 8));
}
assertEquals(out.size(), length);
BytesReference ref = out.bytes();
assertEquals(ref.length(), length);
BytesRef bytesRef = ref.toBytesRef();
final ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset, bytesRef.length);
return Netty3Utils.toBytesReference(channelBuffer);
}
public void testSliceOnAdvancedBuffer() throws IOException {
BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE));
BytesRef bytesRef = bytesReference.toBytesRef();
ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset,
bytesRef.length);
int numBytesToRead = randomIntBetween(1, 5);
for (int i = 0; i < numBytesToRead; i++) {
channelBuffer.readByte();
}
BytesReference other = Netty3Utils.toBytesReference(channelBuffer);
BytesReference slice = bytesReference.slice(numBytesToRead, bytesReference.length() - numBytesToRead);
assertEquals(other, slice);
assertEquals(other.slice(3, 1), slice.slice(3, 1));
}
public void testImmutable() throws IOException {
BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE));
BytesRef bytesRef = BytesRef.deepCopyOf(bytesReference.toBytesRef());
ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset,
bytesRef.length);
ChannelBufferBytesReference channelBufferBytesReference = new ChannelBufferBytesReference(channelBuffer, bytesRef.length);
assertEquals(channelBufferBytesReference, bytesReference);
channelBuffer.readInt(); // this advances the index of the channel buffer
assertEquals(channelBufferBytesReference, bytesReference);
assertEquals(bytesRef, channelBufferBytesReference.toBytesRef());
BytesRef unicodeBytes = new BytesRef(randomUnicodeOfCodepointLength(100));
channelBuffer = ChannelBuffers.wrappedBuffer(unicodeBytes.bytes, unicodeBytes.offset, unicodeBytes.length);
channelBufferBytesReference = new ChannelBufferBytesReference(channelBuffer, unicodeBytes.length);
String utf8ToString = channelBufferBytesReference.utf8ToString();
channelBuffer.readInt(); // this advances the index of the channel buffer
assertEquals(utf8ToString, channelBufferBytesReference.utf8ToString());
}
}

151
modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3ScheduledPingTests.java
View File

@ -1,151 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.transport.MockTransportService;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponseHandler;
import org.elasticsearch.transport.TcpTransport;
import org.elasticsearch.transport.TransportChannel;
import org.elasticsearch.transport.TransportException;
import org.elasticsearch.transport.TransportRequest;
import org.elasticsearch.transport.TransportRequestHandler;
import org.elasticsearch.transport.TransportRequestOptions;
import org.elasticsearch.transport.TransportResponse;
import org.elasticsearch.transport.TransportResponseOptions;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.transport.TransportSettings;
import java.io.IOException;
import java.util.Collections;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
public class Netty3ScheduledPingTests extends ESTestCase {
public void testScheduledPing() throws Exception {
ThreadPool threadPool = new TestThreadPool(getClass().getName());
Settings settings = Settings.builder()
.put(TcpTransport.PING_SCHEDULE.getKey(), "5ms")
.put(TransportSettings.PORT.getKey(), 0)
.put("cluster.name", "test")
.build();
CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
NamedWriteableRegistry registry = new NamedWriteableRegistry(Collections.emptyList());
final Netty3Transport nettyA = new Netty3Transport(settings, threadPool, new NetworkService(settings, Collections.emptyList()),
BigArrays.NON_RECYCLING_INSTANCE, registry, circuitBreakerService);
MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
null);
serviceA.start();
serviceA.acceptIncomingRequests();
final Netty3Transport nettyB = new Netty3Transport(settings, threadPool, new NetworkService(settings, Collections.emptyList()),
BigArrays.NON_RECYCLING_INSTANCE, registry, circuitBreakerService);
MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
null);
serviceB.start();
serviceB.acceptIncomingRequests();
DiscoveryNode nodeA =
new DiscoveryNode("TS_A", "TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), Version.CURRENT);
DiscoveryNode nodeB =
new DiscoveryNode("TS_B", "TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), Version.CURRENT);
serviceA.connectToNode(nodeB);
serviceB.connectToNode(nodeA);
assertBusy(new Runnable() {
@Override
public void run() {
assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(100L));
assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(100L));
}
});
assertThat(nettyA.getPing().getFailedPings(), equalTo(0L));
assertThat(nettyB.getPing().getFailedPings(), equalTo(0L));
serviceA.registerRequestHandler("sayHello", TransportRequest.Empty::new, ThreadPool.Names.GENERIC,
new TransportRequestHandler<TransportRequest.Empty>() {
@Override
public void messageReceived(TransportRequest.Empty request, TransportChannel channel) {
try {
channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.EMPTY);
} catch (IOException e) {
logger.error("Unexpected failure", e);
fail(e.getMessage());
}
}
});
// send some messages while ping requests are going around
int rounds = scaledRandomIntBetween(100, 5000);
for (int i = 0; i < rounds; i++) {
serviceB.submitRequest(nodeA, "sayHello",
TransportRequest.Empty.INSTANCE, TransportRequestOptions.builder().withCompress(randomBoolean()).build(),
new TransportResponseHandler<TransportResponse.Empty>() {
@Override
public TransportResponse.Empty newInstance() {
return TransportResponse.Empty.INSTANCE;
}
@Override
public String executor() {
return ThreadPool.Names.GENERIC;
}
@Override
public void handleResponse(TransportResponse.Empty response) {
}
@Override
public void handleException(TransportException exp) {
logger.error("Unexpected failure", exp);
fail("got exception instead of a response: " + exp.getMessage());
}
}).txGet();
}
assertBusy(() -> {
assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(200L));
assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(200L));
});
assertThat(nettyA.getPing().getFailedPings(), equalTo(0L));
assertThat(nettyB.getPing().getFailedPings(), equalTo(0L));
Releasables.close(serviceA, serviceB);
terminate(threadPool);
}
}

132
modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportIT.java
View File

@ -1,132 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.elasticsearch.ESNetty3IntegTestCase;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.plugins.NetworkPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.Transport;
import org.elasticsearch.transport.TransportSettings;
import org.jboss.netty.channel.Channel;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.is;
@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
public class Netty3TransportIT extends ESNetty3IntegTestCase {
// static so we can use it in anonymous classes
private static String channelProfileName = null;
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder().put(super.nodeSettings(nodeOrdinal))
.put(NetworkModule.TRANSPORT_TYPE_KEY, "exception-throwing").build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
List<Class<? extends Plugin>> list = new ArrayList<>();
list.add(ExceptionThrowingNetty3Transport.TestPlugin.class);
list.addAll(super.nodePlugins());
return Collections.unmodifiableCollection(list);
}
public void testThatConnectionFailsAsIntended() throws Exception {
Client transportClient = internalCluster().transportClient();
ClusterHealthResponse clusterIndexHealths = transportClient.admin().cluster().prepareHealth().get();
assertThat(clusterIndexHealths.getStatus(), is(ClusterHealthStatus.GREEN));
try {
transportClient.filterWithHeader(Collections.singletonMap("ERROR", "MY MESSAGE")).admin().cluster().prepareHealth().get();
fail("Expected exception, but didn't happen");
} catch (ElasticsearchException e) {
assertThat(e.getMessage(), containsString("MY MESSAGE"));
assertThat(channelProfileName, is(TransportSettings.DEFAULT_PROFILE));
}
}
public static final class ExceptionThrowingNetty3Transport extends Netty3Transport {
public static class TestPlugin extends Plugin implements NetworkPlugin {
@Override
public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
CircuitBreakerService circuitBreakerService,
NamedWriteableRegistry namedWriteableRegistry,
NetworkService networkService) {
return Collections.singletonMap("exception-throwing", () ->
new ExceptionThrowingNetty3Transport(settings, threadPool, networkService, bigArrays,
namedWriteableRegistry, circuitBreakerService));
}
}
public ExceptionThrowingNetty3Transport(
Settings settings,
ThreadPool threadPool,
NetworkService networkService,
BigArrays bigArrays,
NamedWriteableRegistry namedWriteableRegistry,
CircuitBreakerService circuitBreakerService) {
super(settings, threadPool, networkService, bigArrays, namedWriteableRegistry, circuitBreakerService);
}
protected String handleRequest(Channel channel, String profileName,
StreamInput stream, long requestId, int messageLengthBytes, Version version,
InetSocketAddress remoteAddress) throws IOException {
String action = super.handleRequest(channel, profileName, stream, requestId, messageLengthBytes, version,
remoteAddress);
channelProfileName = TransportSettings.DEFAULT_PROFILE;
return action;
}
@Override
protected void validateRequest(StreamInput buffer, long requestId, String action)
throws IOException {
super.validateRequest(buffer, requestId, action);
String error = threadPool.getThreadContext().getHeader("ERROR");
if (error != null) {
throw new ElasticsearchException(error);
}
}
}
}

110
modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java
View File

@ -1,110 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.elasticsearch.ESNetty3IntegTestCase;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.BoundTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.env.Environment;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.elasticsearch.test.junit.annotations.Network;
import org.elasticsearch.transport.MockTransportClient;
import org.elasticsearch.transport.Netty3Plugin;
import java.net.InetAddress;
import java.util.Locale;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1, numClientNodes = 0)
public class Netty3TransportMultiPortIntegrationIT extends ESNetty3IntegTestCase {
private static int randomPort = -1;
private static String randomPortRange;
@Override
protected Settings nodeSettings(int nodeOrdinal) {
if (randomPort == -1) {
randomPort = randomIntBetween(49152, 65525);
randomPortRange = String.format(Locale.ROOT, "%s-%s", randomPort, randomPort+10);
}
Settings.Builder builder = Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put("network.host", "127.0.0.1")
.put("transport.profiles.client1.port", randomPortRange)
.put("transport.profiles.client1.publish_host", "127.0.0.7")
.put("transport.profiles.client1.publish_port", "4321")
.put("transport.profiles.client1.reuse_address", true);
return builder.build();
}
public void testThatTransportClientCanConnect() throws Exception {
Settings settings = Settings.builder()
.put("cluster.name", internalCluster().getClusterName())
.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME)
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build();
try (TransportClient transportClient = new MockTransportClient(settings, Netty3Plugin.class)) {
transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName("127.0.0.1"), randomPort));
ClusterHealthResponse response = transportClient.admin().cluster().prepareHealth().get();
assertThat(response.getStatus(), is(ClusterHealthStatus.GREEN));
}
}
@Network
public void testThatInfosAreExposed() throws Exception {
NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().clear().setTransport(true).get();
for (NodeInfo nodeInfo : response.getNodes()) {
assertThat(nodeInfo.getTransport().getProfileAddresses().keySet(), hasSize(1));
assertThat(nodeInfo.getTransport().getProfileAddresses(), hasKey("client1"));
BoundTransportAddress boundTransportAddress = nodeInfo.getTransport().getProfileAddresses().get("client1");
for (TransportAddress transportAddress : boundTransportAddress.boundAddresses()) {
assertThat(transportAddress, instanceOf(TransportAddress.class));
}
// bound addresses
for (TransportAddress transportAddress : boundTransportAddress.boundAddresses()) {
assertThat(transportAddress, instanceOf(TransportAddress.class));
assertThat(transportAddress.address().getPort(),
is(allOf(greaterThanOrEqualTo(randomPort), lessThanOrEqualTo(randomPort + 10))));
}
// publish address
assertThat(boundTransportAddress.publishAddress(), instanceOf(TransportAddress.class));
TransportAddress publishAddress = boundTransportAddress.publishAddress();
assertThat(NetworkAddress.format(publishAddress.address().getAddress()), is("127.0.0.7"));
assertThat(publishAddress.address().getPort(), is(4321));
}
}
}

147
modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortTests.java
View File

@ -1,147 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.elasticsearch.common.component.Lifecycle;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.network.NetworkUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.MockBigArrays;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TcpTransport;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.transport.TransportSettings;
import org.junit.Before;
import java.util.Collections;
import static org.hamcrest.Matchers.is;
public class Netty3TransportMultiPortTests extends ESTestCase {
private String host;
@Before
public void setup() {
if (NetworkUtils.SUPPORTS_V6 && randomBoolean()) {
host = "::1";
} else {
host = "127.0.0.1";
}
}
public void testThatNettyCanBindToMultiplePorts() throws Exception {
Settings settings = Settings.builder()
.put("network.host", host)
.put(TransportSettings.PORT.getKey(), 22) // will not actually bind to this
.put("transport.profiles.default.port", 0)
.put("transport.profiles.client1.port", 0)
.build();
ThreadPool threadPool = new TestThreadPool("tst");
try (TcpTransport<?> transport = startTransport(settings, threadPool)) {
assertEquals(1, transport.profileBoundAddresses().size());
assertEquals(1, transport.boundAddress().boundAddresses().length);
} finally {
terminate(threadPool);
}
}
public void testThatDefaultProfileInheritsFromStandardSettings() throws Exception {
Settings settings = Settings.builder()
.put("network.host", host)
.put(TransportSettings.PORT.getKey(), 0)
.put("transport.profiles.client1.port", 0)
.build();
ThreadPool threadPool = new TestThreadPool("tst");
try (TcpTransport<?> transport = startTransport(settings, threadPool)) {
assertEquals(1, transport.profileBoundAddresses().size());
assertEquals(1, transport.boundAddress().boundAddresses().length);
} finally {
terminate(threadPool);
}
}
public void testThatProfileWithoutPortSettingsFails() throws Exception {
Settings settings = Settings.builder()
.put("network.host", host)
.put(TransportSettings.PORT.getKey(), 0)
.put("transport.profiles.client1.whatever", "foo")
.build();
ThreadPool threadPool = new TestThreadPool("tst");
try (TcpTransport<?> transport = startTransport(settings, threadPool)) {
assertEquals(0, transport.profileBoundAddresses().size());
assertEquals(1, transport.boundAddress().boundAddresses().length);
} finally {
terminate(threadPool);
}
}
public void testThatDefaultProfilePortOverridesGeneralConfiguration() throws Exception {
Settings settings = Settings.builder()
.put("network.host", host)
.put(TransportSettings.PORT.getKey(), 22) // will not actually bind to this
.put("transport.profiles.default.port", 0)
.build();
ThreadPool threadPool = new TestThreadPool("tst");
try (TcpTransport<?> transport = startTransport(settings, threadPool)) {
assertEquals(0, transport.profileBoundAddresses().size());
assertEquals(1, transport.boundAddress().boundAddresses().length);
} finally {
terminate(threadPool);
}
}
public void testThatProfileWithoutValidNameIsIgnored() throws Exception {
Settings settings = Settings.builder()
.put("network.host", host)
.put(TransportSettings.PORT.getKey(), 0)
// mimics someone trying to define a profile for .local which is the profile for a node request to itself
.put("transport.profiles." + TransportService.DIRECT_RESPONSE_PROFILE + ".port", 22) // will not actually bind to this
.put("transport.profiles..port", 23) // will not actually bind to this
.build();
ThreadPool threadPool = new TestThreadPool("tst");
try (TcpTransport<?> transport = startTransport(settings, threadPool)) {
assertEquals(0, transport.profileBoundAddresses().size());
assertEquals(1, transport.boundAddress().boundAddresses().length);
} finally {
terminate(threadPool);
}
}
private TcpTransport<?> startTransport(Settings settings, ThreadPool threadPool) {
BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
TcpTransport<?> transport = new Netty3Transport(settings, threadPool, new NetworkService(settings, Collections.emptyList()),
bigArrays, new NamedWriteableRegistry(Collections.emptyList()), new NoneCircuitBreakerService());
transport.start();
assertThat(transport.lifecycleState(), is(Lifecycle.State.STARTED));
return transport;
}
}

79
modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportPublishAddressIT.java
View File

@ -1,79 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.elasticsearch.ESNetty3IntegTestCase;
import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.common.network.NetworkUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.BoundTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.test.ESIntegTestCase;
import java.net.Inet4Address;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.instanceOf;
/**
* Checks that Elasticsearch produces a sane publish_address when it binds to
* different ports on ipv4 and ipv6.
*/
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
public class Netty3TransportPublishAddressIT extends ESNetty3IntegTestCase {
public void testDifferentPorts() throws Exception {
if (!NetworkUtils.SUPPORTS_V6) {
return;
}
logger.info("--> starting a node on ipv4 only");
Settings ipv4Settings = Settings.builder().put("network.host", "127.0.0.1").build();
String ipv4OnlyNode = internalCluster().startNode(ipv4Settings); // should bind 127.0.0.1:XYZ
logger.info("--> starting a node on ipv4 and ipv6");
Settings bothSettings = Settings.builder().put("network.host", "_local_").build();
internalCluster().startNode(bothSettings); // should bind [::1]:XYZ and 127.0.0.1:XYZ+1
logger.info("--> waiting for the cluster to declare itself stable");
ensureStableCluster(2); // fails if port of publish address does not match corresponding bound address
logger.info("--> checking if boundAddress matching publishAddress has same port");
NodesInfoResponse nodesInfoResponse = client().admin().cluster().prepareNodesInfo().get();
for (NodeInfo nodeInfo : nodesInfoResponse.getNodes()) {
BoundTransportAddress boundTransportAddress = nodeInfo.getTransport().getAddress();
if (nodeInfo.getNode().getName().equals(ipv4OnlyNode)) {
assertThat(boundTransportAddress.boundAddresses().length, equalTo(1));
assertThat(boundTransportAddress.boundAddresses()[0].getPort(), equalTo(boundTransportAddress.publishAddress().getPort()));
} else {
assertThat(boundTransportAddress.boundAddresses().length, greaterThan(1));
for (TransportAddress boundAddress : boundTransportAddress.boundAddresses()) {
assertThat(boundAddress, instanceOf(TransportAddress.class));
TransportAddress inetBoundAddress = (TransportAddress) boundAddress;
if (inetBoundAddress.address().getAddress() instanceof Inet4Address) {
// IPv4 address is preferred publish address for _local_
assertThat(inetBoundAddress.getPort(), equalTo(boundTransportAddress.publishAddress().getPort()));
}
}
}
}
}
}

97
modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3UtilsTests.java
View File

@ -1,97 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.test.ESTestCase;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.buffer.CompositeChannelBuffer;
import java.io.IOException;
public class Netty3UtilsTests extends ESTestCase {
private static final int PAGE_SIZE = BigArrays.BYTE_PAGE_SIZE;
private final BigArrays bigarrays = new BigArrays(null, new NoneCircuitBreakerService(), false);
public void testToChannelBufferWithEmptyRef() throws IOException {
ChannelBuffer channelBuffer = Netty3Utils.toChannelBuffer(getRandomizedBytesReference(0));
assertSame(ChannelBuffers.EMPTY_BUFFER, channelBuffer);
}
public void testToChannelBufferWithSlice() throws IOException {
BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE));
int sliceOffset = randomIntBetween(0, ref.length());
int sliceLength = randomIntBetween(ref.length() - sliceOffset, ref.length() - sliceOffset);
BytesReference slice = ref.slice(sliceOffset, sliceLength);
ChannelBuffer channelBuffer = Netty3Utils.toChannelBuffer(slice);
BytesReference bytesReference = Netty3Utils.toBytesReference(channelBuffer);
assertArrayEquals(BytesReference.toBytes(slice), BytesReference.toBytes(bytesReference));
}
public void testToChannelBufferWithSliceAfter() throws IOException {
BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE));
int sliceOffset = randomIntBetween(0, ref.length());
int sliceLength = randomIntBetween(ref.length() - sliceOffset, ref.length() - sliceOffset);
ChannelBuffer channelBuffer = Netty3Utils.toChannelBuffer(ref);
BytesReference bytesReference = Netty3Utils.toBytesReference(channelBuffer);
assertArrayEquals(BytesReference.toBytes(ref.slice(sliceOffset, sliceLength)),
BytesReference.toBytes(bytesReference.slice(sliceOffset, sliceLength)));
}
public void testToChannelBuffer() throws IOException {
BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE));
ChannelBuffer channelBuffer = Netty3Utils.toChannelBuffer(ref);
BytesReference bytesReference = Netty3Utils.toBytesReference(channelBuffer);
if (ref instanceof ChannelBufferBytesReference) {
assertEquals(channelBuffer, ((ChannelBufferBytesReference) ref).toChannelBuffer());
} else if (AbstractBytesReferenceTestCase.getNumPages(ref) > 1) { // we gather the buffers into a channel buffer
assertTrue(channelBuffer instanceof CompositeChannelBuffer);
}
assertArrayEquals(BytesReference.toBytes(ref), BytesReference.toBytes(bytesReference));
}
private BytesReference getRandomizedBytesReference(int length) throws IOException {
// we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
for (int i = 0; i < length; i++) {
out.writeByte((byte) random().nextInt(1 << 8));
}
assertEquals(out.size(), length);
BytesReference ref = out.bytes();
assertEquals(ref.length(), length);
if (randomBoolean()) {
return new BytesArray(ref.toBytesRef());
} else if (randomBoolean()) {
BytesRef bytesRef = ref.toBytesRef();
return Netty3Utils.toBytesReference(ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset,
bytesRef.length));
} else {
return ref;
}
}
}

105
modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/SimpleNetty3TransportTests.java
View File

@ -1,105 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.netty3;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.node.Node;
import org.elasticsearch.test.transport.MockTransportService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.AbstractSimpleTransportTestCase;
import org.elasticsearch.transport.BindTransportException;
import org.elasticsearch.transport.ConnectTransportException;
import org.elasticsearch.transport.Transport;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.transport.TransportSettings;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Collections;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.Matchers.containsString;
public class SimpleNetty3TransportTests extends AbstractSimpleTransportTestCase {
public static MockTransportService nettyFromThreadPool(Settings settings, ThreadPool threadPool, final Version version,
ClusterSettings clusterSettings) {
NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
Transport transport = new Netty3Transport(settings, threadPool, new NetworkService(settings, Collections.emptyList()),
BigArrays.NON_RECYCLING_INSTANCE, namedWriteableRegistry, new NoneCircuitBreakerService()) {
@Override
protected Version getCurrentVersion() {
return version;
}
};
return new MockTransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
clusterSettings);
}
@Override
protected MockTransportService build(Settings settings, Version version, ClusterSettings clusterSettings) {
settings = Settings.builder().put(settings).put(TransportSettings.PORT.getKey(), "0").build();
MockTransportService transportService = nettyFromThreadPool(settings, threadPool, version, clusterSettings);
transportService.start();
return transportService;
}
public void testConnectException() throws UnknownHostException {
try {
serviceA.connectToNode(new DiscoveryNode("C", new TransportAddress(InetAddress.getByName("localhost"), 9876),
emptyMap(), emptySet(),Version.CURRENT));
fail("Expected ConnectTransportException");
} catch (ConnectTransportException e) {
assertThat(e.getMessage(), containsString("connect_timeout"));
assertThat(e.getMessage(), containsString("[127.0.0.1:9876]"));
}
}
public void testBindUnavailableAddress() {
// this is on a lower level since it needs access to the TransportService before it's started
int port = serviceA.boundAddress().publishAddress().getPort();
Settings settings = Settings.builder()
.put(Node.NODE_NAME_SETTING.getKey(), "foobar")
.put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
.put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
.put("transport.tcp.port", port)
.build();
ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
BindTransportException bindTransportException = expectThrows(BindTransportException.class, () -> {
MockTransportService transportService = nettyFromThreadPool(settings, threadPool, Version.CURRENT, clusterSettings);
try {
transportService.start();
} finally {
transportService.stop();
transportService.close();
}
});
assertEquals("Failed to bind to ["+ port + "]", bindTransportException.getMessage());
}
}

19
modules/transport-netty3/src/test/resources/rest-api-spec/test/10_basic.yaml
View File

@ -1,19 +0,0 @@
# Integration tests for Netty transport
#
"Netty loaded":
- do:
cluster.state: {}
# Get master node id
- set: { master_node: master }
- do:
nodes.info: {}
- match: { nodes.$master.modules.0.name: transport-netty3 }
- do:
cluster.stats: {}
- match: { nodes.network_types.transport_types.netty3: 2 }
- match: { nodes.network_types.http_types.netty3: 2 }

7
modules/transport-netty4/build.gradle
View File

@ -42,13 +42,6 @@ dependencies {
compile "io.netty:netty-transport:4.1.6.Final"
}
integTest {
includePackaged = true
cluster {
numNodes = 2
}
}
thirdPartyAudit.excludes = [
// classes are missing

6
modules/transport-netty4/src/test/resources/rest-api-spec/test/10_basic.yaml
View File

@ -10,10 +10,10 @@
- do:
nodes.info: {}
- match: { nodes.$master.modules.1.name: transport-netty4 }
- match: { nodes.$master.modules.0.name: transport-netty4 }
- do:
cluster.stats: {}
- match: { nodes.network_types.transport_types.netty4: 2 }
- match: { nodes.network_types.http_types.netty4: 2 }
- match: { nodes.network_types.transport_types.netty4: 1 }
- match: { nodes.network_types.http_types.netty4: 1 }

22
qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java
View File

@ -31,7 +31,6 @@ import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.env.Environment;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.MockTcpTransportPlugin;
import org.elasticsearch.transport.Netty3Plugin;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.junit.After;
import org.junit.AfterClass;
@ -49,7 +48,6 @@ import java.util.Locale;
import java.util.concurrent.atomic.AtomicInteger;
import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength;
import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween;
import static org.hamcrest.Matchers.notNullValue;
/**
@ -83,20 +81,11 @@ public abstract class ESSmokeClientTestCase extends LuceneTestCase {
.put("client.transport.ignore_cluster_name", true)
.put(Environment.PATH_HOME_SETTING.getKey(), tempDir);
final Collection<Class<? extends Plugin>> plugins;
switch (randomIntBetween(0, 2)) {
case 0:
builder.put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME);
plugins = Collections.singleton(MockTcpTransportPlugin.class);
break;
case 1:
plugins = Collections.emptyList();
builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME);
break;
case 2:
plugins = Collections.emptyList();
break;
default:
throw new AssertionError();
if (random().nextBoolean()) {
builder.put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME);
plugins = Collections.singleton(MockTcpTransportPlugin.class);
} else {
plugins = Collections.emptyList();
}
TransportClient client = new PreBuiltTransportClient(builder.build(), plugins).addTransportAddresses(transportAddresses);
@ -176,5 +165,4 @@ public abstract class ESSmokeClientTestCase extends LuceneTestCase {
}
}
}
}

1
qa/smoke-test-http/build.gradle
View File

@ -20,6 +20,5 @@
apply plugin: 'elasticsearch.rest-test'
dependencies {
testCompile project(path: ':modules:transport-netty3', configuration: 'runtime') // for http
testCompile project(path: ':modules:transport-netty4', configuration: 'runtime') // for http
}

4
qa/smoke-test-http/src/test/java/org/elasticsearch/http/CorsRegexIT.java
View File

@ -23,10 +23,8 @@ import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.jboss.netty.handler.codec.http.HttpHeaders;
import java.io.IOException;
@ -102,7 +100,7 @@ public class CorsRegexIT extends HttpSmokeTestCase {
String corsValue = "http://localhost:9200";
Response response = getRestClient().performRequest("OPTIONS", "/",
new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue),
new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));
new BasicHeader("Access-Control-Request-Method", "GET"));
assertResponseWithOriginheader(response, corsValue);
assertNotNull(response.getHeader("Access-Control-Allow-Methods"));
}

16
qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java
View File

@ -19,19 +19,15 @@
package org.elasticsearch.http;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.transport.MockTcpTransportPlugin;
import org.elasticsearch.transport.Netty3Plugin;
import org.elasticsearch.transport.Netty4Plugin;
import org.junit.BeforeClass;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
public abstract class HttpSmokeTestCase extends ESIntegTestCase {
@ -42,16 +38,14 @@ public abstract class HttpSmokeTestCase extends ESIntegTestCase {
@SuppressWarnings("unchecked")
@BeforeClass
public static void setUpTransport() {
nodeTransportTypeKey = getTypeKey(randomFrom(MockTcpTransportPlugin.class, Netty3Plugin.class, Netty4Plugin.class));
nodeHttpTypeKey = getTypeKey(randomFrom(Netty3Plugin.class, Netty4Plugin.class));
clientTypeKey = getTypeKey(randomFrom(MockTcpTransportPlugin.class, Netty3Plugin.class, Netty4Plugin.class));
nodeTransportTypeKey = getTypeKey(randomFrom(MockTcpTransportPlugin.class, Netty4Plugin.class));
nodeHttpTypeKey = getTypeKey(Netty4Plugin.class);
clientTypeKey = getTypeKey(randomFrom(MockTcpTransportPlugin.class,Netty4Plugin.class));
}
private static String getTypeKey(Class<? extends Plugin> clazz) {
if (clazz.equals(MockTcpTransportPlugin.class)) {
return MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME;
} else if (clazz.equals(Netty3Plugin.class)) {
return Netty3Plugin.NETTY_TRANSPORT_NAME;
} else {
assert clazz.equals(Netty4Plugin.class);
return Netty4Plugin.NETTY_TRANSPORT_NAME;
@ -69,12 +63,12 @@ public abstract class HttpSmokeTestCase extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Arrays.asList(MockTcpTransportPlugin.class, Netty3Plugin.class, Netty4Plugin.class);
return Arrays.asList(MockTcpTransportPlugin.class, Netty4Plugin.class);
}
@Override
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
return Arrays.asList(MockTcpTransportPlugin.class, Netty3Plugin.class, Netty4Plugin.class);
return Arrays.asList(MockTcpTransportPlugin.class, Netty4Plugin.class);
}
@Override

1
qa/smoke-test-multinode/build.gradle
View File

@ -2,6 +2,7 @@
apply plugin: 'elasticsearch.rest-test'
integTest {
includePackaged = true
cluster {
numNodes = 2
}

1
settings.gradle
View File

@ -27,7 +27,6 @@ List projects = [
'modules:lang-groovy',
'modules:lang-mustache',
'modules:lang-painless',
'modules:transport-netty3',
'modules:transport-netty4',
'modules:reindex',
'modules:percolator',