[7.x] Introduce javaRestTest source set/task and convert modules (#59939) (#60026)

Introduce a javaRestTest source set and task to compliment the yamlRestTest.
javaRestTest differs such that the code is sourced from Java and may have
different dependencies and setup requirements for the test clusters. This also
allows the tests to run in parallel in different cluster instances to prevent any
cross test contamination between the two types of tests.

Included in this PR is all :modules no longer use the integTest task. The tests
are now driven by test, yamlRestTest, javaRestTest, and internalClusterTest.
Since only :modules (and :rest-api-spec) have been converted to yamlRestTest
we can now disable the integTest task if either yamlRestTest or javaRestTest have
been applied. Once all projects are converted, we can delete the integTest task.

related: #56841
related: #59444

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -120,6 +120,14 @@ class PluginBuildPlugin implements Plugin<Project> {
             }
         }
 
+        //disable integTest task if project has been converted to use yaml or java rest test plugin
+        project.pluginManager.withPlugin("elasticsearch.yaml-rest-test") {
+            project.tasks.integTest.enabled = false
+        }
+        project.pluginManager.withPlugin("elasticsearch.java-rest-test") {
+            project.tasks.integTest.enabled = false
+        }
+
         project.tasks.named('testingConventions').configure {
             naming.clear()
             naming {

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.test.rest;
+
+import org.elasticsearch.gradle.ElasticsearchJavaPlugin;
+import org.elasticsearch.gradle.test.RestIntegTestTask;
+import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.SourceSetContainer;
+
+import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupDependencies;
+import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupRunnerTask;
+import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupTask;
+
+/**
+ * Apply this plugin to run the Java based REST tests.
+ */
+public class JavaRestTestPlugin implements Plugin<Project> {
+
+    public static final String SOURCE_SET_NAME = "javaRestTest";
+
+    @Override
+    public void apply(Project project) {
+
+        project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
+        project.getPluginManager().apply(TestClustersPlugin.class);
+
+        // create source set
+        SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+        SourceSet javaTestSourceSet = sourceSets.create(SOURCE_SET_NAME);
+
+        // setup the javaRestTest task
+        RestIntegTestTask javaRestTestTask = setupTask(project, SOURCE_SET_NAME);
+
+        // setup the runner task
+        setupRunnerTask(project, javaRestTestTask, javaTestSourceSet);
+
+        // setup dependencies
+        setupDependencies(project, javaTestSourceSet);
+
+        // setup IDE
+        GradleUtils.setupIdeForTestSourceSet(project, javaTestSourceSet);
+
+        // wire this task into check
+        project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure(check -> check.dependsOn(javaRestTestTask));
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.test.rest;
+
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.info.BuildParams;
+import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
+import org.elasticsearch.gradle.test.RestIntegTestTask;
+import org.elasticsearch.gradle.testclusters.RestTestRunnerTask;
+import org.gradle.api.Project;
+import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.bundling.Zip;
+
+/**
+ * Utility class to configure the necessary tasks and dependencies.
+ */
+public class RestTestUtil {
+
+    private RestTestUtil() {}
+
+    /**
+     * Creates a task with the source set name of type {@link RestIntegTestTask}
+     */
+    static RestIntegTestTask setupTask(Project project, String sourceSetName) {
+        // create task - note can not use .register due to the work in RestIntegTestTask's constructor :(
+        // see: https://github.com/elastic/elasticsearch/issues/47804
+        RestIntegTestTask testTask = project.getTasks().create(sourceSetName, RestIntegTestTask.class);
+        testTask.setGroup(JavaBasePlugin.VERIFICATION_GROUP);
+        testTask.setDescription("Runs the REST tests against an external cluster");
+        // make the new test run after unit tests
+        testTask.mustRunAfter(project.getTasks().named("test"));
+        return testTask;
+    }
+
+    /**
+     * Creates the runner task and configures the test clusters
+     */
+    static RestTestRunnerTask setupRunnerTask(Project project, RestIntegTestTask testTask, SourceSet sourceSet) {
+        RestTestRunnerTask runner = testTask.getRunner();
+        runner.setTestClassesDirs(sourceSet.getOutput().getClassesDirs());
+        runner.setClasspath(sourceSet.getRuntimeClasspath());
+
+        // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster
+        project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
+            Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
+            testTask.dependsOn(bundle);
+            if (project.getPath().startsWith(":modules:")) {
+                runner.getClusters().forEach(c -> c.module(bundle.getArchiveFile()));
+            } else {
+                runner.getClusters().forEach(c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile())));
+            }
+        });
+
+        // es-plugins may declare dependencies on additional modules, add those to the test cluster too.
+        project.afterEvaluate(p -> {
+            PluginPropertiesExtension pluginPropertiesExtension = project.getExtensions().findByType(PluginPropertiesExtension.class);
+            if (pluginPropertiesExtension != null) { // not all projects are defined as plugins
+                pluginPropertiesExtension.getExtendedPlugins().forEach(pluginName -> {
+                    Project extensionProject = project.getProject().findProject(":modules:" + pluginName);
+                    if (extensionProject != null) { // extension plugin may be defined, but not required to be a module
+                        Zip extensionBundle = (Zip) extensionProject.getTasks().getByName("bundlePlugin");
+                        testTask.dependsOn(extensionBundle);
+                        runner.getClusters().forEach(c -> c.module(extensionBundle.getArchiveFile()));
+                    }
+                });
+            }
+        });
+        return runner;
+    }
+
+    /**
+     * Setup the dependencies needed for the REST tests.
+     */
+    static void setupDependencies(Project project, SourceSet sourceSet) {
+        if (BuildParams.isInternal()) {
+            project.getDependencies().add(sourceSet.getImplementationConfigurationName(), project.project(":test:framework"));
+        } else {
+            project.getDependencies()
+                .add(
+                    sourceSet.getImplementationConfigurationName(),
+                    "org.elasticsearch.test:framework:" + VersionProperties.getElasticsearch()
+                );
+        }
+
+    }
+
+}

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java

@@ -20,11 +20,7 @@
 package org.elasticsearch.gradle.test.rest;
 
 import org.elasticsearch.gradle.ElasticsearchJavaPlugin;
-import org.elasticsearch.gradle.VersionProperties;
-import org.elasticsearch.gradle.info.BuildParams;
-import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
 import org.elasticsearch.gradle.test.RestIntegTestTask;
-import org.elasticsearch.gradle.testclusters.RestTestRunnerTask;
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
@@ -32,7 +28,10 @@ import org.gradle.api.Project;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
-import org.gradle.api.tasks.bundling.Zip;
+
+import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupDependencies;
+import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupRunnerTask;
+import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupTask;
 
 /**
  * Apply this plugin to run the YAML based REST tests.
@@ -44,37 +43,22 @@ public class YamlRestTestPlugin implements Plugin<Project> {
     @Override
     public void apply(Project project) {
 
-        // yaml Rest tests require a Java test runner
         project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
-        // to spin up the external cluster
         project.getPluginManager().apply(TestClustersPlugin.class);
-        // to copy around the yaml tests and json spec
         project.getPluginManager().apply(RestResourcesPlugin.class);
 
-        // note - source sets are not created via org.elasticsearch.gradle.util.GradleUtils.addTestSourceSet since unlike normal tests
-        // we only want the yamlRestTestSourceSet on the classpath by default. The yaml tests should be pure black box testing over HTTP and
-        // such it should not need the main class on the class path. Also, there are some special setup steps unique to YAML REST tests.
-
         // create source set
         SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
         SourceSet yamlTestSourceSet = sourceSets.create(SOURCE_SET_NAME);
 
-        // create task - note can not use .register due to the work in RestIntegTestTask's constructor :(
-        // see: https://github.com/elastic/elasticsearch/issues/47804
-        RestIntegTestTask yamlRestTestTask = project.getTasks().create(SOURCE_SET_NAME, RestIntegTestTask.class);
-        yamlRestTestTask.setGroup(JavaBasePlugin.VERIFICATION_GROUP);
-        yamlRestTestTask.setDescription("Runs the YAML based REST tests against an external cluster");
+        // setup the yamlRestTest task
+        RestIntegTestTask yamlRestTestTask = setupTask(project, SOURCE_SET_NAME);
 
-        // setup task dependency
-        if (BuildParams.isInternal()) {
-            project.getDependencies().add(yamlTestSourceSet.getImplementationConfigurationName(), project.project(":test:framework"));
-        } else {
-            project.getDependencies()
-                .add(
-                    yamlTestSourceSet.getImplementationConfigurationName(),
-                    "org.elasticsearch.test:framework:" + VersionProperties.getElasticsearch()
-                );
-        }
+        // setup the runner task
+        setupRunnerTask(project, yamlRestTestTask, yamlTestSourceSet);
+
+        // setup the dependencies
+        setupDependencies(project, yamlTestSourceSet);
 
         // setup the copy for the rest resources
         project.getTasks().withType(CopyRestApiTask.class, copyRestApiTask -> {
@@ -83,40 +67,6 @@ public class YamlRestTestPlugin implements Plugin<Project> {
         });
         project.getTasks().withType(CopyRestTestsTask.class, copyRestTestTask -> { copyRestTestTask.sourceSetName = SOURCE_SET_NAME; });
 
-        // make the new test run after unit tests
-        yamlRestTestTask.mustRunAfter(project.getTasks().named("test"));
-
-        // setup the runner
-        RestTestRunnerTask runner = yamlRestTestTask.getRunner();
-        runner.setTestClassesDirs(yamlTestSourceSet.getOutput().getClassesDirs());
-        runner.setClasspath(yamlTestSourceSet.getRuntimeClasspath());
-
-        // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster
-        project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
-            Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
-            yamlRestTestTask.dependsOn(bundle);
-            if (project.getPath().startsWith(":modules:")) {
-                runner.getClusters().forEach(c -> c.module(bundle.getArchiveFile()));
-            } else {
-                runner.getClusters().forEach(c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile())));
-            }
-        });
-
-        // es-plugins may declare dependencies on additional modules, add those to the test cluster too.
-        project.afterEvaluate(p -> {
-            PluginPropertiesExtension pluginPropertiesExtension = project.getExtensions().findByType(PluginPropertiesExtension.class);
-            if (pluginPropertiesExtension != null) { // not all projects are defined as plugins
-                pluginPropertiesExtension.getExtendedPlugins().forEach(pluginName -> {
-                    Project extensionProject = project.getProject().findProject(":modules:" + pluginName);
-                    if (extensionProject != null) { // extension plugin may be defined, but not required to be a module
-                        Zip extensionBundle = (Zip) extensionProject.getTasks().getByName("bundlePlugin");
-                        yamlRestTestTask.dependsOn(extensionBundle);
-                        runner.getClusters().forEach(c -> c.module(extensionBundle.getArchiveFile()));
-                    }
-                });
-            }
-        });
-
         // setup IDE
         GradleUtils.setupIdeForTestSourceSet(project, yamlTestSourceSet);
 

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java-rest-test.properties

@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.test.rest.JavaRestTestPlugin

modules/aggs-matrix-stats/build.gradle

@@ -29,5 +29,3 @@ restResources {
     includeCore '_common', 'indices', 'cluster', 'index', 'search', 'nodes'
   }
 }
-
-integTest.enabled = false

modules/analysis-common/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description 'Adds "built in" analyzers to Elasticsearch.'

modules/geo/build.gradle

@@ -31,6 +31,4 @@ restResources {
 artifacts {
   restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test'))
 }
-
-integTest.enabled = false
 test.enabled = false

modules/ingest-common/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources'

modules/ingest-geoip/build.gradle

@@ -43,8 +43,6 @@ restResources {
   }
 }
 
-integTest.enabled = false
-
 task copyDefaultGeoIp2DatabaseFiles(type: Copy) {
   from { zipTree(configurations.testCompileClasspath.files.find { it.name.contains('geolite2-databases') }) }
   into "${project.buildDir}/ingest-geoip"

modules/ingest-user-agent/build.gradle

@@ -33,4 +33,3 @@ testClusters.all {
   extraConfigFile 'ingest-user-agent/test-regexes.yml', file('src/test/test-regexes.yml')
 }
 
-integTest.enabled = false

modules/kibana/build.gradle

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+apply plugin: 'elasticsearch.java-rest-test'
 
 esplugin {
   description 'Plugin exposing APIs for Kibana system indices'
@@ -26,6 +27,7 @@ dependencies {
   api project(path: ':modules:reindex')
 }
 
-testClusters.integTest {
+testClusters.all {
   module file(project(':modules:reindex').tasks.bundlePlugin.archiveFile)
 }
+

modules/lang-mustache/build.gradle

@@ -17,6 +17,8 @@
  * under the License.
  */
 apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.java-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description 'Mustache scripting integration for Elasticsearch'

modules/mapper-extras/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.java-rest-test'
 
 esplugin {
   description 'Adds advanced field mappers'

modules/mapper-extras/src/javaRestTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java

@@ -7,7 +7,7 @@
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an

modules/parent-join/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description 'This module adds the support parent-child queries and aggregations'

modules/percolator/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description 'Percolator module adds capability to index queries and query these queries by specifying documents'
@@ -29,22 +30,8 @@ dependencies {
   testImplementation project(':modules:geo')
 }
 
-tasks.named('integTestRunner').configure {
-  exclude '**/PercolatorQuerySearchIT.class'
-}
-
-tasks.register('internalClusterTest', Test) {
-  include '**/PercolatorQuerySearchIT.class'
-}
-
-tasks.named('check').configure {
-  dependsOn 'internalClusterTest'
-}
-
 restResources {
   restApi {
     includeCore '_common', 'indices', 'index', 'search', 'msearch'
   }
 }
-
-integTest.enabled = false

modules/rank-eval/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description 'The Rank Eval module adds APIs to evaluate ranking quality.'

modules/reindex/build.gradle

@@ -25,6 +25,8 @@ import org.elasticsearch.gradle.info.BuildParams
 apply plugin: 'elasticsearch.test-with-dependencies'
 apply plugin: 'elasticsearch.jdk-download'
 apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.java-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description 'The Reindex module adds APIs to reindex from one index to another or update documents in place.'
@@ -108,12 +110,12 @@ jdks {
 
 if (Os.isFamily(Os.FAMILY_WINDOWS)) {
   logger.warn("Disabling reindex-from-old tests because we can't get the pid file on windows")
-  integTest.runner {
+  javaRestTest.runner {
     systemProperty "tests.fromOld", "false"
   }
 } else if (rootProject.rootDir.toString().contains(" ")) {
   logger.warn("Disabling reindex-from-old tests because Elasticsearch 1.7 won't start with spaces in the path")
-  integTest.runner {
+  javaRestTest.runner {
     systemProperty "tests.fromOld", "false"
   }
 } else {
@@ -154,7 +156,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
       }
     }
 
-    integTest {
+    javaRestTest {
       dependsOn fixture
       runner {
         systemProperty "tests.fromOld", "true"
@@ -166,3 +168,6 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
     }
   }
 }
+
+//todo: remove once test/**/client/documentation can be deleted after https://github.com/elastic/docs/pull/1920
+testingConventions.enabled = false

modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentation2IT.java

@@ -0,0 +1,359 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.documentation;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.reindex.AbstractBulkByScrollRequestBuilder;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.index.reindex.BulkByScrollTask;
+import org.elasticsearch.index.reindex.CancelTests;
+import org.elasticsearch.index.reindex.DeleteByQueryAction;
+import org.elasticsearch.index.reindex.DeleteByQueryRequestBuilder;
+import org.elasticsearch.index.reindex.ReindexAction;
+import org.elasticsearch.index.reindex.ReindexPlugin;
+import org.elasticsearch.index.reindex.ReindexRequestBuilder;
+import org.elasticsearch.index.reindex.RethrottleAction;
+import org.elasticsearch.index.reindex.RethrottleRequestBuilder;
+import org.elasticsearch.index.reindex.UpdateByQueryAction;
+import org.elasticsearch.index.reindex.UpdateByQueryRequestBuilder;
+import org.elasticsearch.index.shard.IndexingOperationListener;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matcher;
+import org.junit.Before;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.equalTo;
+
+//TODO: remove the "2" after https://github.com/elastic/docs/pull/1920 is merged and the "test" variant is removed.
+public class ReindexDocumentation2IT extends ESIntegTestCase {
+
+    // Semaphore used to allow & block indexing operations during the test
+    private static final Semaphore ALLOWED_OPERATIONS = new Semaphore(0);
+    private static final String INDEX_NAME = "source_index";
+
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(ReindexPlugin.class, ReindexCancellationPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return Collections.singletonList(ReindexPlugin.class);
+    }
+
+    @Before
+    public void setup() {
+        client().admin().indices().prepareCreate(INDEX_NAME).get();
+    }
+
+    @SuppressWarnings("unused")
+    public void testReindex() {
+        Client client = client();
+        // tag::reindex1
+        BulkByScrollResponse response =
+          new ReindexRequestBuilder(client, ReindexAction.INSTANCE)
+            .source("source_index")
+            .destination("target_index")
+            .filter(QueryBuilders.matchQuery("category", "xzy")) // <1>
+            .get();
+        // end::reindex1
+    }
+
+    @SuppressWarnings("unused")
+    public void testUpdateByQuery() {
+        Client client = client();
+        client.admin().indices().prepareCreate("foo").get();
+        client.admin().indices().prepareCreate("bar").get();
+        client.admin().indices().preparePutMapping(INDEX_NAME).setType("_doc").setSource("cat", "type=keyword").get();
+        {
+            // tag::update-by-query
+            UpdateByQueryRequestBuilder updateByQuery =
+              new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE);
+            updateByQuery.source("source_index").abortOnVersionConflict(false);
+            BulkByScrollResponse response = updateByQuery.get();
+            // end::update-by-query
+        }
+        {
+            // tag::update-by-query-filter
+            UpdateByQueryRequestBuilder updateByQuery =
+              new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE);
+            updateByQuery.source("source_index")
+                .filter(QueryBuilders.termQuery("level", "awesome"))
+                .maxDocs(1000)
+                .script(new Script(ScriptType.INLINE,
+                    "painless",
+                    "ctx._source.awesome = 'absolutely'",
+                    Collections.emptyMap()));
+            BulkByScrollResponse response = updateByQuery.get();
+            // end::update-by-query-filter
+
+            // validate order of string params to Script constructor
+            assertEquals(updateByQuery.request().getScript().getLang(), "painless");
+        }
+        {
+            // tag::update-by-query-size
+            UpdateByQueryRequestBuilder updateByQuery =
+              new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE);
+            updateByQuery.source("source_index")
+                .source()
+                .setSize(500);
+            BulkByScrollResponse response = updateByQuery.get();
+            // end::update-by-query-size
+        }
+        {
+            // tag::update-by-query-sort
+            UpdateByQueryRequestBuilder updateByQuery =
+               new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE);
+            updateByQuery.source("source_index")
+                .maxDocs(100)
+                .source()
+                .addSort("cat", SortOrder.DESC);
+            BulkByScrollResponse response = updateByQuery.get();
+            // end::update-by-query-sort
+        }
+        {
+            // tag::update-by-query-script
+            UpdateByQueryRequestBuilder updateByQuery =
+              new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE);
+            updateByQuery.source("source_index")
+                .script(new Script(
+                    ScriptType.INLINE,
+                    "painless",
+                    "if (ctx._source.awesome == 'absolutely') {"
+                        + "  ctx.op='noop'"
+                        + "} else if (ctx._source.awesome == 'lame') {"
+                        + "  ctx.op='delete'"
+                        + "} else {"
+                        + "ctx._source.awesome = 'absolutely'}",
+                    Collections.emptyMap()));
+            BulkByScrollResponse response = updateByQuery.get();
+            // end::update-by-query-script
+
+            // validate order of string params to Script constructor
+            assertEquals(updateByQuery.request().getScript().getLang(), "painless");
+        }
+        {
+            // tag::update-by-query-multi-index
+            UpdateByQueryRequestBuilder updateByQuery =
+              new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE);
+            updateByQuery.source("foo", "bar");
+            BulkByScrollResponse response = updateByQuery.get();
+            // end::update-by-query-multi-index
+        }
+        {
+            // tag::update-by-query-routing
+            UpdateByQueryRequestBuilder updateByQuery =
+              new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE);
+            updateByQuery.source().setRouting("cat");
+            BulkByScrollResponse response = updateByQuery.get();
+            // end::update-by-query-routing
+        }
+        {
+            // tag::update-by-query-pipeline
+            UpdateByQueryRequestBuilder updateByQuery =
+              new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE);
+            updateByQuery.setPipeline("hurray");
+            BulkByScrollResponse response = updateByQuery.get();
+            // end::update-by-query-pipeline
+        }
+    }
+
+    public void testTasks() throws Exception {
+        final Client client = client();
+        final ReindexRequestBuilder builder = reindexAndPartiallyBlock();
+
+        {
+            // tag::update-by-query-list-tasks
+            ListTasksResponse tasksList = client.admin().cluster().prepareListTasks()
+                .setActions(UpdateByQueryAction.NAME).setDetailed(true).get();
+            for (TaskInfo info: tasksList.getTasks()) {
+                TaskId taskId = info.getTaskId();
+                BulkByScrollTask.Status status =
+                    (BulkByScrollTask.Status) info.getStatus();
+                // do stuff
+            }
+            // end::update-by-query-list-tasks
+        }
+
+        TaskInfo mainTask = CancelTests.findTaskToCancel(ReindexAction.NAME, builder.request().getSlices());
+        BulkByScrollTask.Status status = (BulkByScrollTask.Status) mainTask.getStatus();
+        assertNull(status.getReasonCancelled());
+        TaskId taskId = mainTask.getTaskId();
+        {
+            // tag::update-by-query-get-task
+            GetTaskResponse get = client.admin().cluster().prepareGetTask(taskId).get();
+            // end::update-by-query-get-task
+        }
+        {
+            // tag::update-by-query-cancel-task
+            // Cancel all update-by-query requests
+            client.admin().cluster().prepareCancelTasks()
+                .setActions(UpdateByQueryAction.NAME).get().getTasks();
+            // Cancel a specific update-by-query request
+            client.admin().cluster().prepareCancelTasks()
+                .setTaskId(taskId).get().getTasks();
+            // end::update-by-query-cancel-task
+        }
+        {
+            // tag::update-by-query-rethrottle
+            new RethrottleRequestBuilder(client, RethrottleAction.INSTANCE)
+                .setTaskId(taskId)
+                .setRequestsPerSecond(2.0f)
+                .get();
+            // end::update-by-query-rethrottle
+        }
+
+        // unblocking the blocked update
+        ALLOWED_OPERATIONS.release(builder.request().getSlices());
+    }
+
+    @SuppressWarnings("unused")
+    public void testDeleteByQuery() {
+        Client client = client();
+        client.admin().indices().prepareCreate("persons").get();
+
+        // tag::delete-by-query-sync
+        BulkByScrollResponse response =
+          new DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE)
+            .filter(QueryBuilders.matchQuery("gender", "male")) // <1>
+            .source("persons")                                  // <2>
+            .get();                                             // <3>
+        long deleted = response.getDeleted();                   // <4>
+        // end::delete-by-query-sync
+
+        // tag::delete-by-query-async
+        new DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE)
+            .filter(QueryBuilders.matchQuery("gender", "male"))     // <1>
+            .source("persons")                                      // <2>
+            .execute(new ActionListener<BulkByScrollResponse>() {   // <3>
+                @Override
+                public void onResponse(BulkByScrollResponse response) {
+                    long deleted = response.getDeleted();           // <4>
+                }
+                @Override
+                public void onFailure(Exception e) {
+                    // Handle the exception
+                }
+            });
+        // end::delete-by-query-async
+    }
+
+    /**
+     * Similar to what CancelTests does: blocks some operations to be able to catch some tasks in running state
+     * @see CancelTests#testCancel(String, AbstractBulkByScrollRequestBuilder, CancelTests.CancelAssertion, Matcher)
+     */
+    private ReindexRequestBuilder reindexAndPartiallyBlock() throws Exception {
+        final Client client = client();
+        final int numDocs = randomIntBetween(10, 100);
+        ALLOWED_OPERATIONS.release(numDocs);
+
+        indexRandom(true, false, true, IntStream.range(0, numDocs)
+            .mapToObj(i -> client().prepareIndex(INDEX_NAME, "_doc", Integer.toString(i)).setSource("n", Integer.toString(i)))
+            .collect(Collectors.toList()));
+
+        // Checks that the all documents have been indexed and correctly counted
+        assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), numDocs);
+        assertThat(ALLOWED_OPERATIONS.drainPermits(), equalTo(0));
+
+        ReindexRequestBuilder builder = new ReindexRequestBuilder(client, ReindexAction.INSTANCE).source(INDEX_NAME)
+            .destination("target_index", "_doc");
+        // Scroll by 1 so that cancellation is easier to control
+        builder.source().setSize(1);
+
+        int numModifiedDocs = randomIntBetween(builder.request().getSlices() * 2, numDocs);
+        // chose to modify some of docs - rest is still blocked
+        ALLOWED_OPERATIONS.release(numModifiedDocs - builder.request().getSlices());
+
+        // Now execute the reindex action...
+        builder.execute();
+
+        // 10 seconds is usually fine but on heavily loaded machines this can take a while
+        assertBusy(
+            () -> {
+                assertTrue("Expected some queued threads", ALLOWED_OPERATIONS.hasQueuedThreads());
+                assertEquals("Expected that no permits are available", 0, ALLOWED_OPERATIONS.availablePermits());
+            },
+            1, TimeUnit.MINUTES);
+        return builder;
+    }
+
+    public static class ReindexCancellationPlugin extends Plugin {
+
+        @Override
+        public void onIndexModule(IndexModule indexModule) {
+            indexModule.addIndexOperationListener(new BlockingOperationListener());
+        }
+    }
+
+    public static class BlockingOperationListener implements IndexingOperationListener {
+
+        @Override
+        public Engine.Index preIndex(ShardId shardId, Engine.Index index) {
+            return preCheck(index, index.type());
+        }
+
+        @Override
+        public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
+            return preCheck(delete, delete.type());
+        }
+
+        private <T extends Engine.Operation> T preCheck(T operation, String type) {
+            if (("_doc".equals(type) == false) || (operation.origin() != Engine.Operation.Origin.PRIMARY)) {
+                return operation;
+            }
+
+            try {
+                if (ALLOWED_OPERATIONS.tryAcquire(30, TimeUnit.SECONDS)) {
+                    return operation;
+                }
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+            throw new IllegalStateException("Something went wrong");
+        }
+    }
+
+}

modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java

@@ -7,7 +7,7 @@
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an

modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java

@@ -7,7 +7,7 @@
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an

modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java

@@ -7,7 +7,7 @@
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an

modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java

@@ -62,6 +62,7 @@ import java.util.stream.IntStream;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.equalTo;
 
+//TODO: delete this after https://github.com/elastic/docs/pull/1920 is merged
 public class ReindexDocumentationIT extends ESIntegTestCase {
 
     // Semaphore used to allow & block indexing operations during the test

modules/repository-url/build.gradle

@@ -36,8 +36,6 @@ restResources {
   }
 }
 
-integTest.enabled = false
-
  // This directory is shared between two URL repositories and one FS repository in YAML integration tests
 File repositoryDir = new File(project.buildDir, "shared-repository")
 

modules/transport-netty4/build.gradle

@@ -20,8 +20,12 @@
 
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.test.RestIntegTestTask
+import org.elasticsearch.gradle.test.rest.JavaRestTestPlugin
+import org.elasticsearch.gradle.test.InternalClusterTestPlugin
 
 apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.java-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
 
 /*
  TODOs:
@@ -64,11 +68,11 @@ test {
   systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
 
-integTestRunner {
-  /*
-   * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
-   * other if we allow them to set the number of available processors as it's set-once in Netty.
-   */
+internalClusterTest {
+  systemProperty 'es.set.netty.runtime.available.processors', 'false'
+}
+
+javaRestTestRunner {
   systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
 
@@ -78,15 +82,29 @@ TaskProvider<Test> pooledTest = tasks.register("pooledTest", Test) {
   systemProperty 'es.use_unpooled_allocator', 'false'
 }
 
-RestIntegTestTask pooledIntegTest = tasks.create("pooledIntegTest", RestIntegTestTask) {
+TaskProvider<Test> pooledInternalClusterTest = tasks.register("pooledInternalClusterTest", Test) {
+  include '**/*IT.class'
+  systemProperty 'es.set.netty.runtime.available.processors', 'false'
+  systemProperty 'es.use_unpooled_allocator', 'false'
+  SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+  SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
+  setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
+  setClasspath(internalTestSourceSet.getRuntimeClasspath())
+}
+
+RestIntegTestTask pooledJavaRestTest = tasks.create("pooledJavaRestTest", RestIntegTestTask) {
   runner {
     systemProperty 'es.set.netty.runtime.available.processors', 'false'
+    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+    SourceSet javaRestTestSourceSet = sourceSets.getByName(JavaRestTestPlugin.SOURCE_SET_NAME)
+    setTestClassesDirs(javaRestTestSourceSet.getOutput().getClassesDirs())
+    setClasspath(javaRestTestSourceSet.getRuntimeClasspath())
   }
 }
-testClusters.pooledIntegTest {
+testClusters.pooledJavaRestTest {
   systemProperty 'es.use_unpooled_allocator', 'false'
 }
-check.dependsOn(pooledTest, pooledIntegTest)
+check.dependsOn(pooledTest, pooledJavaRestTest, pooledInternalClusterTest)
 
 thirdPartyAudit {
   ignoreMissingClasses(

modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java

@@ -7,7 +7,7 @@
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -20,6 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.util.ReferenceCounted;
 import org.elasticsearch.ESNetty4IntegTestCase;
 import org.elasticsearch.common.collect.Tuple;
@@ -31,7 +32,6 @@ import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import io.netty.handler.codec.http.HttpResponseStatus;
 
 import java.util.ArrayList;
 import java.util.Collection;