Fix remaining tests that either need access to the netty module or require explict configuration

Some tests still start http implicitly or miss configuring the transport clients correctly.
This commit fixes all remaining tests and adds a depdenceny to `transport-netty` from
`qa/smoke-test-http` and `modules/reindex` since they need an http server running on the nodes.

This also moves all required permissions for netty into it's module and out of core.

core/src/main/java/org/elasticsearch/common/network/NetworkModule.java

@@ -147,8 +147,9 @@ public class NetworkModule extends AbstractModule {
         bind(NetworkService.class).toInstance(networkService);
         bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
 
+        boolean nettyRegistered = transportTypes.getExtension(NETTY_TRANSPORT) != null;
         transportServiceTypes.bindType(binder(), settings, TRANSPORT_SERVICE_TYPE_KEY, NETTY_TRANSPORT);
-        String defaultTransport = DiscoveryNode.isLocalNode(settings) ? LOCAL_TRANSPORT : NETTY_TRANSPORT;
+        String defaultTransport = DiscoveryNode.isLocalNode(settings) || nettyRegistered == false ? LOCAL_TRANSPORT : NETTY_TRANSPORT;
         transportTypes.bindType(binder(), settings, TRANSPORT_TYPE_KEY, defaultTransport);
 
         if (transportClient == false) {

core/src/main/resources/org/elasticsearch/bootstrap/security.policy

@@ -83,14 +83,6 @@ grant {
   permission java.util.PropertyPermission "junit4.childvm.count", "write";
   permission java.util.PropertyPermission "junit4.childvm.id", "write";
 
-  // set by NettyTransport/NettyHttpServerTransport based on another parameter
-  // TODO: look into this and decide if users should simply set the actual sysprop?!
-  permission java.util.PropertyPermission "org.jboss.netty.epollBugWorkaround", "write";
-
-  // Netty SelectorUtil wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
-  // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
-  permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
-
   // needed by Settings
   permission java.lang.RuntimePermission "getenv.*";
 

core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java

@@ -108,7 +108,9 @@ public class NetworkModuleTests extends ModuleTestCase {
     }
 
     public void testRegisterTransportService() {
-        Settings settings = Settings.builder().put(NetworkModule.TRANSPORT_SERVICE_TYPE_KEY, "custom").build();
+        Settings settings = Settings.builder().put(NetworkModule.TRANSPORT_SERVICE_TYPE_KEY, "custom")
+            .put(NetworkModule.HTTP_ENABLED.getKey(), false)
+            .build();
         NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
         module.registerTransportService("custom", FakeTransportService.class);
         assertBinding(module, TransportService.class, FakeTransportService.class);
@@ -122,7 +124,9 @@ public class NetworkModuleTests extends ModuleTestCase {
     }
 
     public void testRegisterTransport() {
-        Settings settings = Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, "custom").build();
+        Settings settings = Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, "custom")
+            .put(NetworkModule.HTTP_ENABLED.getKey(), false)
+            .build();
         NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
         module.registerTransport("custom", FakeTransport.class);
         assertBinding(module, Transport.class, FakeTransport.class);

core/src/test/java/org/elasticsearch/tribe/TribeIT.java

@@ -33,6 +33,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
@@ -131,6 +132,7 @@ public class TribeIT extends ESIntegTestCase {
                 .put("tribe.t1.cluster.name", internalCluster().getClusterName())
                 .put("tribe.t2.cluster.name", cluster2.getClusterName())
                 .put("tribe.blocks.write", false)
+                .put(NetworkModule.HTTP_ENABLED.getKey(), false)
                 .put(settings)
                 .put(tribe1Defaults.build())
                 .put(tribe2Defaults.build())

distribution/build.gradle

@@ -96,7 +96,6 @@ project.rootProject.subprojects.findAll { it.path.startsWith(':modules:transport
   module.afterEvaluate({
     module.integTest.mustRunAfter(':distribution:integ-test-zip:integTest#stop')
   })
-  restTestExpansions['expected.modules.count'] += 1
 }
 
 

modules/reindex/build.gradle

@@ -42,6 +42,8 @@ dependencies {
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
+  // for http - testing reindex from remote
+  testCompile project(path: ':modules:transport-netty', configuration: 'runtime')
 }
 
 dependencyLicenses {

modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java

@@ -35,6 +35,7 @@ import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.NettyPlugin;
 import org.junit.After;
 import org.junit.Before;
 
@@ -59,7 +60,7 @@ public class RetryTests extends ESSingleNodeTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(ReindexPlugin.class);
+        return pluginList(ReindexPlugin.class, NettyPlugin.class); // we need netty here to http communication
     }
 
     /**

modules/transport-netty/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java

@@ -78,6 +78,8 @@ import org.jboss.netty.handler.timeout.ReadTimeoutException;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -284,19 +286,23 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent impleme
     @Override
     protected void doStart() {
         this.serverOpenChannels = new OpenChannelsHandler(logger);
-
+        // this doPrivileged is for SelectorUtil.java that tries to set "sun.nio.ch.bugLevel"
-        if (blockingServer) {
+        AccessController.doPrivileged((PrivilegedAction<Object>) () -> {
-            serverBootstrap = new ServerBootstrap(new OioServerSocketChannelFactory(
+                if (blockingServer) {
-                    Executors.newCachedThreadPool(daemonThreadFactory(settings, "http_server_boss")),
+                    serverBootstrap = new ServerBootstrap(new OioServerSocketChannelFactory(
-                    Executors.newCachedThreadPool(daemonThreadFactory(settings, "http_server_worker"))
+                        Executors.newCachedThreadPool(daemonThreadFactory(settings, "http_server_boss")),
-            ));
+                        Executors.newCachedThreadPool(daemonThreadFactory(settings, "http_server_worker"))
-        } else {
+                    ));
-            serverBootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(
+                } else {
-                    Executors.newCachedThreadPool(daemonThreadFactory(settings, "http_server_boss")),
+                    serverBootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(
-                    Executors.newCachedThreadPool(daemonThreadFactory(settings, "http_server_worker")),
+                        Executors.newCachedThreadPool(daemonThreadFactory(settings, "http_server_boss")),
-                    workerCount));
+                        Executors.newCachedThreadPool(daemonThreadFactory(settings, "http_server_worker")),
-        }
+                        workerCount));
-
+                }
+                return null;
+            });
+        assert System.getProperty("sun.nio.ch.bugLevel") != null :
+            "sun.nio.ch.bugLevel is null somebody pulls in SelectorUtil without doing stuff in a doPrivileged block?";
         serverBootstrap.setPipelineFactory(configureServerChannelPipelineFactory());
 
         serverBootstrap.setOption("child.tcpNoDelay", tcpNoDelay);

modules/transport-netty/src/main/java/org/elasticsearch/transport/NettyPlugin.java

@@ -21,7 +21,6 @@ package org.elasticsearch.transport;
 
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.http.netty.NettyHttpServerTransport;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.transport.netty.NettyTransport;
@@ -31,6 +30,9 @@ import java.util.List;
 
 public class NettyPlugin extends Plugin {
 
+    public static final String NETTY_TRANSPORT_NAME = "netty";
+    public static final String NETTY_HTTP_TRANSPORT_NAME = "netty";
+
     @Override
     public List<Setting<?>> getSettings() {
         return Arrays.asList(
@@ -55,8 +57,8 @@ public class NettyPlugin extends Plugin {
 
     public void onModule(NetworkModule networkModule) {
         if (networkModule.canRegisterHttpExtensions()) {
-            networkModule.registerHttpTransport("netty", NettyHttpServerTransport.class);
+            networkModule.registerHttpTransport(NETTY_HTTP_TRANSPORT_NAME, NettyHttpServerTransport.class);
         }
-        networkModule.registerTransport("netty", NettyTransport.class);
+        networkModule.registerTransport(NETTY_TRANSPORT_NAME, NettyTransport.class);
     }
 }

modules/transport-netty/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java

@@ -65,6 +65,8 @@ import org.jboss.netty.util.HashedWheelTimer;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
@@ -182,19 +184,25 @@ public class NettyTransport extends TcpTransport<Channel> {
     }
 
     private ClientBootstrap createClientBootstrap() {
-        if (blockingClient) {
+        // this doPrivileged is for SelectorUtil.java that tries to set "sun.nio.ch.bugLevel"
-            clientBootstrap = new ClientBootstrap(new OioClientSocketChannelFactory(
+        AccessController.doPrivileged((PrivilegedAction<Object>) () -> {
-                Executors.newCachedThreadPool(daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX))));
+                if (blockingClient) {
-        } else {
+                    clientBootstrap = new ClientBootstrap(new OioClientSocketChannelFactory(
-            int bossCount = NETTY_BOSS_COUNT.get(settings);
+                        Executors.newCachedThreadPool(daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX))));
-            clientBootstrap = new ClientBootstrap(
+                } else {
-                new NioClientSocketChannelFactory(
+                    int bossCount = NETTY_BOSS_COUNT.get(settings);
-                    Executors.newCachedThreadPool(daemonThreadFactory(settings, TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX)),
+                    clientBootstrap = new ClientBootstrap(
-                    bossCount,
+                        new NioClientSocketChannelFactory(
-                    new NioWorkerPool(Executors.newCachedThreadPool(
+                            Executors.newCachedThreadPool(daemonThreadFactory(settings, TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX)),
-                        daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX)), workerCount),
+                            bossCount,
-                    new HashedWheelTimer(daemonThreadFactory(settings, "transport_client_timer"))));
+                            new NioWorkerPool(Executors.newCachedThreadPool(
-        }
+                                daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX)), workerCount),
+                            new HashedWheelTimer(daemonThreadFactory(settings, "transport_client_timer"))));
+                }
+            return null;
+        });
+        assert System.getProperty("sun.nio.ch.bugLevel") != null :
+            "sun.nio.ch.bugLevel is null somebody pulls in SelectorUtil without doing stuff in a doPrivileged block?";
         clientBootstrap.setPipelineFactory(configureClientChannelPipelineFactory());
         clientBootstrap.setOption("connectTimeoutMillis", connectTimeout.millis());
 
@@ -280,18 +288,22 @@ public class NettyTransport extends TcpTransport<Channel> {
 
         final ThreadFactory bossFactory = daemonThreadFactory(this.settings, HTTP_SERVER_BOSS_THREAD_NAME_PREFIX, name);
         final ThreadFactory workerFactory = daemonThreadFactory(this.settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX, name);
-        ServerBootstrap serverBootstrap;
+        // this doPrivileged is for SelectorUtil.java that tries to set "sun.nio.ch.bugLevel"
-        if (blockingServer) {
+        ServerBootstrap serverBootstrap = AccessController.doPrivileged((PrivilegedAction<ServerBootstrap>) () -> {
-            serverBootstrap = new ServerBootstrap(new OioServerSocketChannelFactory(
+            if (blockingServer) {
-                Executors.newCachedThreadPool(bossFactory),
+                return new ServerBootstrap(new OioServerSocketChannelFactory(
-                Executors.newCachedThreadPool(workerFactory)
+                    Executors.newCachedThreadPool(bossFactory),
-            ));
+                    Executors.newCachedThreadPool(workerFactory)
-        } else {
+                ));
-            serverBootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(
+            } else {
-                Executors.newCachedThreadPool(bossFactory),
+                return new ServerBootstrap(new NioServerSocketChannelFactory(
-                Executors.newCachedThreadPool(workerFactory),
+                    Executors.newCachedThreadPool(bossFactory),
-                workerCount));
+                    Executors.newCachedThreadPool(workerFactory),
-        }
+                    workerCount));
+            }
+        });
+        assert System.getProperty("sun.nio.ch.bugLevel") != null :
+            "sun.nio.ch.bugLevel is null somebody pulls in SelectorUtil without doing stuff in a doPrivileged block?";
         serverBootstrap.setPipelineFactory(configureServerChannelPipelineFactory(name, settings));
         if (!"default".equals(tcpNoDelay)) {
             serverBootstrap.setOption("child.tcpNoDelay", Booleans.parseBoolean(tcpNoDelay, null));

modules/transport-netty/src/main/java/org/elasticsearch/transport/netty/NettyUtils.java

@@ -105,7 +105,6 @@ public class NettyUtils {
     }
 
     public static void setup() {
-
     }
 
     /**

modules/transport-netty/src/main/plugin-metadata/plugin-security.policy

@@ -18,5 +18,7 @@
  */
 
 grant {
- // TODO add netty specific permissions
+   // Netty SelectorUtil wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
+   // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
+   permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
 };

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java

@@ -22,6 +22,8 @@ package org.elasticsearch.discovery.ec2;
 import com.amazonaws.util.IOUtils;
 import com.sun.net.httpserver.Headers;
 import com.sun.net.httpserver.HttpServer;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
 import org.elasticsearch.cloud.aws.AwsEc2Service;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.FileSystemUtils;
@@ -30,7 +32,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.jboss.netty.handler.codec.http.QueryStringDecoder;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -49,7 +50,6 @@ import java.nio.file.Path;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 
@@ -101,9 +101,13 @@ public class Ec2DiscoveryClusterFormationTests extends ESIntegTestCase {
         httpServer.createContext("/", (s) -> {
             Headers headers = s.getResponseHeaders();
             headers.add("Content-Type", "text/xml; charset=UTF-8");
-            QueryStringDecoder decoder = new QueryStringDecoder("?" + IOUtils.toString(s.getRequestBody()));
+            String action = null;
-            Map<String, List<String>> queryParams = decoder.getParameters();
+            for (NameValuePair parse : URLEncodedUtils.parse(IOUtils.toString(s.getRequestBody()), StandardCharsets.UTF_8)) {
-            String action = queryParams.get("Action").get(0);
+                if ("Action".equals(parse.getName())) {
+                    action = parse.getValue();
+                    break;
+                }
+            }
             assertThat(action, equalTo("DescribeInstances"));
 
             XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newFactory();

qa/smoke-test-client/build.gradle

@@ -20,3 +20,7 @@
 apply plugin: 'elasticsearch.rest-test'
 
 // TODO: this test works, but it isn't really a rest test...should we have another plugin for "non rest test that just needs N clusters?"
+
+dependencies {
+    testCompile project(path: ':modules:transport-netty', configuration: 'runtime') // randomly swapped in as a transport
+}

qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java

@@ -25,12 +25,16 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.transport.MockTcpTransportPlugin;
+import org.elasticsearch.transport.NettyPlugin;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -73,13 +77,20 @@ public abstract class ESSmokeClientTestCase extends LuceneTestCase {
     protected String index;
 
     private static Client startClient(Path tempDir, TransportAddress... transportAddresses) {
-        Settings clientSettings = Settings.builder()
+        TransportClient.Builder transportClientBuilder = TransportClient.builder();
-                .put("node.name", "qa_smoke_client_" + counter.getAndIncrement())
+        Settings.Builder builder = Settings.builder()
-                .put("client.transport.ignore_cluster_name", true)
+            .put("node.name", "qa_smoke_client_" + counter.getAndIncrement())
-                .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
+            .put("client.transport.ignore_cluster_name", true)
-                .put(Node.NODE_MODE_SETTING.getKey(), "network").build(); // we require network here!
+            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
-
+            .put(Node.NODE_MODE_SETTING.getKey(), "network");// we require network here!
-        TransportClient.Builder transportClientBuilder = TransportClient.builder().settings(clientSettings);
+        if (random().nextBoolean()) {
+            builder.put(NetworkModule.TRANSPORT_TYPE_KEY, NettyPlugin.NETTY_TRANSPORT_NAME);
+            transportClientBuilder.addPlugin(NettyPlugin.class);
+        } else {
+            builder.put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME);
+            transportClientBuilder.addPlugin(MockTcpTransportPlugin.class);
+        }
+        transportClientBuilder.settings(builder.build());
         TransportClient client = transportClientBuilder.build().addTransportAddresses(transportAddresses);
 
         logger.info("--> Elasticsearch Java TransportClient started");

qa/smoke-test-http/build.gradle

@@ -18,3 +18,7 @@
  */
 
 apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+    testCompile project(path: ':modules:transport-netty', configuration: 'runtime') // for http
+}

qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java

@@ -47,7 +47,6 @@ import org.elasticsearch.indices.TermsLookup;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestController;
-import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
@@ -74,7 +73,7 @@ import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 
 @ClusterScope(scope = SUITE)
-public class ContextAndHeaderTransportIT extends ESIntegTestCase {
+public class ContextAndHeaderTransportIT extends HttpSmokeTestCase {
     private static final List<RequestAndHeaders> requests =  new CopyOnWriteArrayList<>();
     private String randomHeaderKey = randomAsciiOfLength(10);
     private String randomHeaderValue = randomAsciiOfLength(20);
@@ -96,7 +95,9 @@ public class ContextAndHeaderTransportIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(ActionLoggingPlugin.class);
+        ArrayList<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(ActionLoggingPlugin.class);
+        return plugins;
     }
 
     @Before

qa/smoke-test-http/src/test/java/org/elasticsearch/http/CorsNotSetIT.java

@@ -26,7 +26,7 @@ import org.elasticsearch.test.ESIntegTestCase;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
-public class CorsNotSetIT extends ESIntegTestCase {
+public class CorsNotSetIT extends HttpSmokeTestCase {
 
 
     public void testCorsSettingDefaultBehaviourDoesNotReturnAnything() throws Exception {

qa/smoke-test-http/src/test/java/org/elasticsearch/http/CorsRegexIT.java

@@ -21,14 +21,11 @@ package org.elasticsearch.http;
 import org.apache.http.message.BasicHeader;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.jboss.netty.handler.codec.http.HttpHeaders;
 
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
@@ -41,7 +38,7 @@ import static org.hamcrest.Matchers.nullValue;
  * Test CORS where the allow origin value is a regular expression.
  */
 @ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
-public class CorsRegexIT extends ESIntegTestCase {
+public class CorsRegexIT extends HttpSmokeTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
@@ -105,7 +102,7 @@ public class CorsRegexIT extends ESIntegTestCase {
         String corsValue = "http://localhost:9200";
         try (Response response = getRestClient().performRequest("OPTIONS", "/",
                 new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue),
-                new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));) {
+                new BasicHeader("Access-Control-Request-Method", "GET"));) {
             assertResponseWithOriginheader(response, corsValue);
             assertNotNull(response.getHeader("Access-Control-Allow-Methods"));
         }
@@ -115,7 +112,7 @@ public class CorsRegexIT extends ESIntegTestCase {
         try {
             getRestClient().performRequest("OPTIONS", "/", new BasicHeader("User-Agent", "Mozilla Bar"),
                     new BasicHeader("Origin", "http://evil-host:9200"),
-                    new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));
+                    new BasicHeader("Access-Control-Request-Method", "GET"));
             fail("request should have failed");
         } catch(ResponseException e) {
             Response response = e.getResponse();

qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.rest;
+package org.elasticsearch.http;
 
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
@@ -29,10 +29,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.rest.plugins.TestDeprecatedQueryBuilder;
-import org.elasticsearch.rest.plugins.TestDeprecationHeaderRestAction;
-import org.elasticsearch.rest.plugins.TestDeprecationPlugin;
-import org.elasticsearch.test.ESIntegTestCase;
 
 import org.hamcrest.Matcher;
 
@@ -45,9 +41,9 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.rest.RestStatus.OK;
-import static org.elasticsearch.rest.plugins.TestDeprecationHeaderRestAction.TEST_DEPRECATED_SETTING_TRUE1;
+import static org.elasticsearch.http.TestDeprecationHeaderRestAction.TEST_DEPRECATED_SETTING_TRUE1;
-import static org.elasticsearch.rest.plugins.TestDeprecationHeaderRestAction.TEST_DEPRECATED_SETTING_TRUE2;
+import static org.elasticsearch.http.TestDeprecationHeaderRestAction.TEST_DEPRECATED_SETTING_TRUE2;
-import static org.elasticsearch.rest.plugins.TestDeprecationHeaderRestAction.TEST_NOT_DEPRECATED_SETTING;
+import static org.elasticsearch.http.TestDeprecationHeaderRestAction.TEST_NOT_DEPRECATED_SETTING;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
@@ -56,7 +52,7 @@ import static org.hamcrest.Matchers.hasSize;
 /**
  * Tests {@code DeprecationLogger} uses the {@code ThreadContext} to add response headers.
  */
-public class DeprecationHttpIT extends ESIntegTestCase {
+public class DeprecationHttpIT extends HttpSmokeTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
@@ -73,7 +69,9 @@ public class DeprecationHttpIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(TestDeprecationPlugin.class);
+        ArrayList<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(TestDeprecationPlugin.class);
+        return plugins;
     }
 
     /**

qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java

@@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.is;
  * Tests that when disabling detailed errors, a request with the error_trace parameter returns a HTTP 400
  */
 @ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
-public class DetailedErrorsDisabledIT extends ESIntegTestCase {
+public class DetailedErrorsDisabledIT extends HttpSmokeTestCase {
     // Build our cluster settings
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {

qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java

@@ -35,7 +35,7 @@ import static org.hamcrest.Matchers.not;
 /**
  * Tests that by default the error_trace parameter can be used to show stacktraces
  */
-public class DetailedErrorsEnabledIT extends ESIntegTestCase {
+public class DetailedErrorsEnabledIT extends HttpSmokeTestCase {
 
     public void testThatErrorTraceWorksByDefault() throws Exception {
         try {

qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpCompressionIT.java

@@ -34,8 +34,7 @@ import org.elasticsearch.test.ESIntegTestCase;
 import java.io.IOException;
 import java.util.Collections;
 
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 1, numClientNodes = 1)
+public class HttpCompressionIT extends HttpSmokeTestCase {
-public class HttpCompressionIT extends ESIntegTestCase {
     private static final String GZIP_ENCODING = "gzip";
 
     private static final StringEntity SAMPLE_DOCUMENT = new StringEntity("{\n" +
@@ -45,6 +44,10 @@ public class HttpCompressionIT extends ESIntegTestCase {
         "   }\n" +
         "}", RestClient.JSON_CONTENT_TYPE);
 
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return false;
+    }
 
     public void testCompressesResponseIfRequested() throws Exception {
         ensureGreen();

qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.http;
+
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.transport.MockTcpTransport;
+import org.elasticsearch.transport.MockTcpTransportPlugin;
+import org.elasticsearch.transport.NettyPlugin;
+
+import java.util.Collection;
+
+public abstract class HttpSmokeTestCase extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(NetworkModule.TRANSPORT_TYPE_KEY, randomFrom(NettyPlugin.NETTY_TRANSPORT_NAME,
+                MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME))
+            .put(NetworkModule.HTTP_ENABLED.getKey(), true).build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return pluginList(MockTcpTransportPlugin.class, NettyPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return pluginList(MockTcpTransportPlugin.class, NettyPlugin.class);
+    }
+
+    @Override
+    protected Settings transportClientSettings() {
+        return Settings.builder()
+            .put(super.transportClientSettings())
+            .put(NetworkModule.TRANSPORT_TYPE_KEY, randomFrom(NettyPlugin.NETTY_TRANSPORT_NAME,
+                MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME)).build();
+    }
+
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+}

qa/smoke-test-http/src/test/java/org/elasticsearch/http/ResponseHeaderPluginIT.java

@@ -16,17 +16,17 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.plugins;
+package org.elasticsearch.http;
 
 import org.apache.http.message.BasicHeader;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugins.responseheader.TestResponseHeaderPlugin;
+import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
+import java.util.ArrayList;
 import java.util.Collection;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -35,7 +35,7 @@ import static org.hamcrest.Matchers.equalTo;
  * Test a rest action that sets special response headers
  */
 @ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
-public class ResponseHeaderPluginIT extends ESIntegTestCase {
+public class ResponseHeaderPluginIT extends HttpSmokeTestCase {
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
@@ -44,9 +44,16 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase {
                 .build();
     }
 
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(TestResponseHeaderPlugin.class);
+        ArrayList<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(TestResponseHeaderPlugin.class);
+        return plugins;
     }
 
     public void testThatSettingHeadersWorks() throws Exception {

qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecatedQueryBuilder.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.rest.plugins;
+package org.elasticsearch.http;
 
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParseField;

qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.rest.plugins;
+package org.elasticsearch.http;
 
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;

qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationPlugin.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.rest.plugins;
+package org.elasticsearch.http;
 
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.plugins.ActionPlugin;

qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderPlugin.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.plugins.responseheader;
+package org.elasticsearch.http;
 
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;

qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderRestAction.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.plugins.responseheader;
+package org.elasticsearch.http;
 
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -1773,7 +1773,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
             @Override
             public Collection<Class<? extends Plugin>> transportClientPlugins() {
                 Collection<Class<? extends Plugin>> plugins = ESIntegTestCase.this.transportClientPlugins();
-                if (isNetwork) {
+                if (isNetwork && plugins.contains(MockTcpTransportPlugin.class) == false) {
                     plugins = new ArrayList<>(plugins);
                     plugins.add(MockTcpTransportPlugin.class);
                 }

test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java

@@ -29,6 +29,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -36,6 +37,8 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.transport.MockTcpTransport;
+import org.elasticsearch.transport.MockTcpTransportPlugin;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -70,14 +73,22 @@ public final class ExternalTestCluster extends TestCluster {
 
     public ExternalTestCluster(Path tempDir, Settings additionalSettings, Collection<Class<? extends Plugin>> pluginClasses, TransportAddress... transportAddresses) {
         super(0);
-        Settings clientSettings = Settings.builder()
+        Settings.Builder clientSettingsBuilder = Settings.builder()
-                .put(additionalSettings)
+            .put(additionalSettings)
-                .put("node.name", InternalTestCluster.TRANSPORT_CLIENT_PREFIX + EXTERNAL_CLUSTER_PREFIX + counter.getAndIncrement())
+            .put("node.name", InternalTestCluster.TRANSPORT_CLIENT_PREFIX + EXTERNAL_CLUSTER_PREFIX + counter.getAndIncrement())
-                .put("client.transport.ignore_cluster_name", true)
+            .put("client.transport.ignore_cluster_name", true)
-                .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
+            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
-                .put(Node.NODE_MODE_SETTING.getKey(), "network").build(); // we require network here!
+            .put(Node.NODE_MODE_SETTING.getKey(), "network");// we require network here!
-
+        TransportClient.Builder transportClientBuilder = TransportClient.builder();
-        TransportClient.Builder transportClientBuilder = TransportClient.builder().settings(clientSettings);
+        boolean addMockTcpTransport = additionalSettings.get(NetworkModule.TRANSPORT_TYPE_KEY) == null;
+        if (addMockTcpTransport) {
+            clientSettingsBuilder.put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME);
+            if (pluginClasses.contains(MockTcpTransportPlugin.class) == false) {
+                transportClientBuilder.addPlugin(MockTcpTransportPlugin.class);
+            }
+        }
+        Settings clientSettings = clientSettingsBuilder.build();
+        transportClientBuilder.settings(clientSettings);
         for (Class<? extends Plugin> pluginClass : pluginClasses) {
             transportClientBuilder.addPlugin(pluginClass);
         }

test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java

@@ -29,12 +29,12 @@ public abstract class NodeConfigurationSource {
     public static final NodeConfigurationSource EMPTY = new NodeConfigurationSource() {
         @Override
         public Settings nodeSettings(int nodeOrdinal) {
-            return null;
+            return Settings.EMPTY;
         }
 
         @Override
         public Settings transportClientSettings() {
-            return null;
+            return Settings.EMPTY;
         }
     };
 
@@ -48,7 +48,9 @@ public abstract class NodeConfigurationSource {
         return Collections.emptyList();
     }
 
-    public abstract Settings transportClientSettings();
+    public Settings transportClientSettings() {
+        return Settings.EMPTY;
+    }
 
     /** Returns plugins that should be loaded in the transport client */
     public Collection<Class<? extends Plugin>> transportClientPlugins() {

test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java

@@ -24,6 +24,7 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.env.NodeEnvironment;
@@ -125,15 +126,16 @@ public class InternalTestClusterTests extends ESTestCase {
         boolean masterNodes = randomBoolean();
         int minNumDataNodes = randomIntBetween(0, 3);
         int maxNumDataNodes = randomIntBetween(minNumDataNodes, 4);
-        final String clusterName1 = "shared1";//clusterName("shared1", clusterSeed);
+        final String clusterName1 = "shared1";
-        final String clusterName2 = "shared2";//clusterName("shared", Integer.toString(CHILD_JVM_ID), clusterSeed);
+        final String clusterName2 = "shared2";
-        /*while (clusterName.equals(clusterName1)) {
+        NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() {
-            clusterName1 = clusterName("shared", Integer.toString(CHILD_JVM_ID), clusterSeed);   // spin until the time changes
+            @Override
-        }*/
+            public Settings nodeSettings(int nodeOrdinal) {
-        NodeConfigurationSource nodeConfigurationSource = NodeConfigurationSource.EMPTY;
+                return Settings.builder().put(NetworkModule.HTTP_ENABLED.getKey(), false).build();
+            }
+        };
         int numClientNodes = randomIntBetween(0, 2);
         boolean enableHttpPipelining = randomBoolean();
-        int jvmOrdinal = randomIntBetween(0, 10);
         String nodePrefix = "foobar";
 
         Path baseDir = createTempDir();
@@ -177,8 +179,12 @@ public class InternalTestClusterTests extends ESTestCase {
         int minNumDataNodes = 2;
         int maxNumDataNodes = 2;
         final String clusterName1 = "shared1";
-        NodeConfigurationSource nodeConfigurationSource = NodeConfigurationSource.EMPTY;
+        NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() {
-        int numClientNodes = randomIntBetween(0, 2);
+            @Override
+            public Settings nodeSettings(int nodeOrdinal) {
+                return Settings.builder().put(NetworkModule.HTTP_ENABLED.getKey(), false).build();
+            }
+        };        int numClientNodes = randomIntBetween(0, 2);
         boolean enableHttpPipelining = randomBoolean();
         String nodePrefix = "test";
         Path baseDir = createTempDir();
@@ -248,7 +254,9 @@ public class InternalTestClusterTests extends ESTestCase {
             new NodeConfigurationSource() {
                 @Override
                 public Settings nodeSettings(int nodeOrdinal) {
-                    return Settings.builder().put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0).build();
+                    return Settings.builder()
+                        .put(NetworkModule.HTTP_ENABLED.getKey(), false)
+                        .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0).build();
                 }
 
                 @Override