From 92ce41cfaf8e5d9feaa8117dfe26239533198726 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Tue, 28 Jul 2020 08:39:11 -0500 Subject: [PATCH] [7.x] Introduce javaRestTest source set/task and convert modules (#59939) (#60026) Introduce a javaRestTest source set and task to compliment the yamlRestTest. javaRestTest differs such that the code is sourced from Java and may have different dependencies and setup requirements for the test clusters. This also allows the tests to run in parallel in different cluster instances to prevent any cross test contamination between the two types of tests. Included in this PR is all :modules no longer use the integTest task. The tests are now driven by test, yamlRestTest, javaRestTest, and internalClusterTest. Since only :modules (and :rest-api-spec) have been converted to yamlRestTest we can now disable the integTest task if either yamlRestTest or javaRestTest have been applied. Once all projects are converted, we can delete the integTest task. related: #56841 related: #59444 --- .../gradle/plugin/PluginBuildPlugin.groovy | 8 + .../gradle/test/rest/JavaRestTestPlugin.java | 68 ++++ .../gradle/test/rest/RestTestUtil.java | 105 +++++ .../gradle/test/rest/YamlRestTestPlugin.java | 72 +--- .../elasticsearch.java-rest-test.properties | 20 + modules/aggs-matrix-stats/build.gradle | 2 - modules/analysis-common/build.gradle | 1 + .../common/QueryStringWithAnalyzersIT.java | 0 modules/geo/build.gradle | 2 - modules/ingest-common/build.gradle | 1 + .../ingest/common/IngestRestartIT.java | 0 modules/ingest-geoip/build.gradle | 2 - modules/ingest-user-agent/build.gradle | 1 - modules/kibana/build.gradle | 4 +- .../kibana/KibanaSystemIndexIT.java | 0 modules/lang-mustache/build.gradle | 2 + .../mustache/MultiSearchTemplateIT.java | 0 .../script/mustache/SearchTemplateIT.java | 0 .../SearchTemplateWithoutContentIT.java | 0 modules/mapper-extras/build.gradle | 1 + .../TokenCountFieldMapperIntegrationIT.java | 2 +- modules/parent-join/build.gradle | 1 + .../AbstractParentChildTestCase.java | 0 .../join/aggregations/ChildrenIT.java | 0 .../join/aggregations/ParentIT.java | 0 .../join/query/ChildQuerySearchIT.java | 0 .../elasticsearch/join/query/InnerHitsIT.java | 0 .../join/query/ParentChildTestCase.java | 0 modules/percolator/build.gradle | 15 +- .../percolator/PercolatorQuerySearchIT.java | 0 modules/rank-eval/build.gradle | 1 + .../index/rankeval/RankEvalRequestIT.java | 0 modules/reindex/build.gradle | 11 +- .../ReindexDocumentation2IT.java | 359 ++++++++++++++++++ .../index/reindex/ManyDocumentsIT.java | 2 +- .../reindex/ReindexWithoutContentIT.java | 2 +- .../remote/ReindexFromOldRemoteIT.java | 2 +- .../documentation/ReindexDocumentationIT.java | 1 + modules/repository-url/build.gradle | 2 - modules/transport-netty4/build.gradle | 34 +- .../elasticsearch/ESNetty4IntegTestCase.java | 0 .../netty4/Netty4HttpRequestSizeLimitIT.java | 4 +- .../http/netty4/Netty4PipeliningIT.java | 0 .../rest/discovery/Zen2RestApiIT.java | 0 .../transport/netty4/ESLoggingHandlerIT.java | 0 ...Netty4TransportMultiPortIntegrationIT.java | 0 .../Netty4TransportPublishAddressIT.java | 0 .../rest/Netty4BadRequestIT.java | 0 .../rest/Netty4HeadBodyIsEmptyIT.java | 0 49 files changed, 623 insertions(+), 102 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java create mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java-rest-test.properties rename modules/analysis-common/src/{test => internalClusterTest}/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java (100%) rename modules/ingest-common/src/{test => internalClusterTest}/java/org/elasticsearch/ingest/common/IngestRestartIT.java (100%) rename modules/kibana/src/{test => javaRestTest}/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java (100%) rename modules/lang-mustache/src/{test => internalClusterTest}/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java (100%) rename modules/lang-mustache/src/{test => internalClusterTest}/java/org/elasticsearch/script/mustache/SearchTemplateIT.java (100%) rename modules/lang-mustache/src/{test => javaRestTest}/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java (100%) rename modules/mapper-extras/src/{test => javaRestTest}/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java (99%) rename modules/parent-join/src/{test => internalClusterTest}/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java (100%) rename modules/parent-join/src/{test => internalClusterTest}/java/org/elasticsearch/join/aggregations/ChildrenIT.java (100%) rename modules/parent-join/src/{test => internalClusterTest}/java/org/elasticsearch/join/aggregations/ParentIT.java (100%) rename modules/parent-join/src/{test => internalClusterTest}/java/org/elasticsearch/join/query/ChildQuerySearchIT.java (100%) rename modules/parent-join/src/{test => internalClusterTest}/java/org/elasticsearch/join/query/InnerHitsIT.java (100%) rename modules/parent-join/src/{test => internalClusterTest}/java/org/elasticsearch/join/query/ParentChildTestCase.java (100%) rename modules/percolator/src/{test => internalClusterTest}/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java (100%) rename modules/rank-eval/src/{test => internalClusterTest}/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java (100%) create mode 100644 modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentation2IT.java rename modules/reindex/src/{test => javaRestTest}/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java (99%) rename modules/reindex/src/{test => javaRestTest}/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java (96%) rename modules/reindex/src/{test => javaRestTest}/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java (99%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/ESNetty4IntegTestCase.java (100%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java (99%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java (100%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java (100%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java (100%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java (100%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java (100%) rename modules/transport-netty4/src/{test => javaRestTest}/java/org/elasticsearch/rest/Netty4BadRequestIT.java (100%) rename modules/transport-netty4/src/{test => javaRestTest}/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java (100%) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index d39f3d3857f..818974eb701 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -120,6 +120,14 @@ class PluginBuildPlugin implements Plugin { } } + //disable integTest task if project has been converted to use yaml or java rest test plugin + project.pluginManager.withPlugin("elasticsearch.yaml-rest-test") { + project.tasks.integTest.enabled = false + } + project.pluginManager.withPlugin("elasticsearch.java-rest-test") { + project.tasks.integTest.enabled = false + } + project.tasks.named('testingConventions').configure { naming.clear() naming { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java new file mode 100644 index 00000000000..4738d3525cf --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test.rest; + +import org.elasticsearch.gradle.ElasticsearchJavaPlugin; +import org.elasticsearch.gradle.test.RestIntegTestTask; +import org.elasticsearch.gradle.testclusters.TestClustersPlugin; +import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.plugins.JavaBasePlugin; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.SourceSetContainer; + +import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupDependencies; +import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupRunnerTask; +import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupTask; + +/** + * Apply this plugin to run the Java based REST tests. + */ +public class JavaRestTestPlugin implements Plugin { + + public static final String SOURCE_SET_NAME = "javaRestTest"; + + @Override + public void apply(Project project) { + + project.getPluginManager().apply(ElasticsearchJavaPlugin.class); + project.getPluginManager().apply(TestClustersPlugin.class); + + // create source set + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet javaTestSourceSet = sourceSets.create(SOURCE_SET_NAME); + + // setup the javaRestTest task + RestIntegTestTask javaRestTestTask = setupTask(project, SOURCE_SET_NAME); + + // setup the runner task + setupRunnerTask(project, javaRestTestTask, javaTestSourceSet); + + // setup dependencies + setupDependencies(project, javaTestSourceSet); + + // setup IDE + GradleUtils.setupIdeForTestSourceSet(project, javaTestSourceSet); + + // wire this task into check + project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure(check -> check.dependsOn(javaRestTestTask)); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java new file mode 100644 index 00000000000..ed3711183b4 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test.rest; + +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; +import org.elasticsearch.gradle.test.RestIntegTestTask; +import org.elasticsearch.gradle.testclusters.RestTestRunnerTask; +import org.gradle.api.Project; +import org.gradle.api.plugins.JavaBasePlugin; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.bundling.Zip; + +/** + * Utility class to configure the necessary tasks and dependencies. + */ +public class RestTestUtil { + + private RestTestUtil() {} + + /** + * Creates a task with the source set name of type {@link RestIntegTestTask} + */ + static RestIntegTestTask setupTask(Project project, String sourceSetName) { + // create task - note can not use .register due to the work in RestIntegTestTask's constructor :( + // see: https://github.com/elastic/elasticsearch/issues/47804 + RestIntegTestTask testTask = project.getTasks().create(sourceSetName, RestIntegTestTask.class); + testTask.setGroup(JavaBasePlugin.VERIFICATION_GROUP); + testTask.setDescription("Runs the REST tests against an external cluster"); + // make the new test run after unit tests + testTask.mustRunAfter(project.getTasks().named("test")); + return testTask; + } + + /** + * Creates the runner task and configures the test clusters + */ + static RestTestRunnerTask setupRunnerTask(Project project, RestIntegTestTask testTask, SourceSet sourceSet) { + RestTestRunnerTask runner = testTask.getRunner(); + runner.setTestClassesDirs(sourceSet.getOutput().getClassesDirs()); + runner.setClasspath(sourceSet.getRuntimeClasspath()); + + // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster + project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> { + Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin"); + testTask.dependsOn(bundle); + if (project.getPath().startsWith(":modules:")) { + runner.getClusters().forEach(c -> c.module(bundle.getArchiveFile())); + } else { + runner.getClusters().forEach(c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))); + } + }); + + // es-plugins may declare dependencies on additional modules, add those to the test cluster too. + project.afterEvaluate(p -> { + PluginPropertiesExtension pluginPropertiesExtension = project.getExtensions().findByType(PluginPropertiesExtension.class); + if (pluginPropertiesExtension != null) { // not all projects are defined as plugins + pluginPropertiesExtension.getExtendedPlugins().forEach(pluginName -> { + Project extensionProject = project.getProject().findProject(":modules:" + pluginName); + if (extensionProject != null) { // extension plugin may be defined, but not required to be a module + Zip extensionBundle = (Zip) extensionProject.getTasks().getByName("bundlePlugin"); + testTask.dependsOn(extensionBundle); + runner.getClusters().forEach(c -> c.module(extensionBundle.getArchiveFile())); + } + }); + } + }); + return runner; + } + + /** + * Setup the dependencies needed for the REST tests. + */ + static void setupDependencies(Project project, SourceSet sourceSet) { + if (BuildParams.isInternal()) { + project.getDependencies().add(sourceSet.getImplementationConfigurationName(), project.project(":test:framework")); + } else { + project.getDependencies() + .add( + sourceSet.getImplementationConfigurationName(), + "org.elasticsearch.test:framework:" + VersionProperties.getElasticsearch() + ); + } + + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java index 676d2ffbf62..45884b03670 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java @@ -20,11 +20,7 @@ package org.elasticsearch.gradle.test.rest; import org.elasticsearch.gradle.ElasticsearchJavaPlugin; -import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; import org.elasticsearch.gradle.test.RestIntegTestTask; -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Plugin; @@ -32,7 +28,10 @@ import org.gradle.api.Project; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.SourceSetContainer; -import org.gradle.api.tasks.bundling.Zip; + +import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupDependencies; +import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupRunnerTask; +import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupTask; /** * Apply this plugin to run the YAML based REST tests. @@ -44,37 +43,22 @@ public class YamlRestTestPlugin implements Plugin { @Override public void apply(Project project) { - // yaml Rest tests require a Java test runner project.getPluginManager().apply(ElasticsearchJavaPlugin.class); - // to spin up the external cluster project.getPluginManager().apply(TestClustersPlugin.class); - // to copy around the yaml tests and json spec project.getPluginManager().apply(RestResourcesPlugin.class); - // note - source sets are not created via org.elasticsearch.gradle.util.GradleUtils.addTestSourceSet since unlike normal tests - // we only want the yamlRestTestSourceSet on the classpath by default. The yaml tests should be pure black box testing over HTTP and - // such it should not need the main class on the class path. Also, there are some special setup steps unique to YAML REST tests. - // create source set SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); SourceSet yamlTestSourceSet = sourceSets.create(SOURCE_SET_NAME); - // create task - note can not use .register due to the work in RestIntegTestTask's constructor :( - // see: https://github.com/elastic/elasticsearch/issues/47804 - RestIntegTestTask yamlRestTestTask = project.getTasks().create(SOURCE_SET_NAME, RestIntegTestTask.class); - yamlRestTestTask.setGroup(JavaBasePlugin.VERIFICATION_GROUP); - yamlRestTestTask.setDescription("Runs the YAML based REST tests against an external cluster"); + // setup the yamlRestTest task + RestIntegTestTask yamlRestTestTask = setupTask(project, SOURCE_SET_NAME); - // setup task dependency - if (BuildParams.isInternal()) { - project.getDependencies().add(yamlTestSourceSet.getImplementationConfigurationName(), project.project(":test:framework")); - } else { - project.getDependencies() - .add( - yamlTestSourceSet.getImplementationConfigurationName(), - "org.elasticsearch.test:framework:" + VersionProperties.getElasticsearch() - ); - } + // setup the runner task + setupRunnerTask(project, yamlRestTestTask, yamlTestSourceSet); + + // setup the dependencies + setupDependencies(project, yamlTestSourceSet); // setup the copy for the rest resources project.getTasks().withType(CopyRestApiTask.class, copyRestApiTask -> { @@ -83,40 +67,6 @@ public class YamlRestTestPlugin implements Plugin { }); project.getTasks().withType(CopyRestTestsTask.class, copyRestTestTask -> { copyRestTestTask.sourceSetName = SOURCE_SET_NAME; }); - // make the new test run after unit tests - yamlRestTestTask.mustRunAfter(project.getTasks().named("test")); - - // setup the runner - RestTestRunnerTask runner = yamlRestTestTask.getRunner(); - runner.setTestClassesDirs(yamlTestSourceSet.getOutput().getClassesDirs()); - runner.setClasspath(yamlTestSourceSet.getRuntimeClasspath()); - - // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster - project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> { - Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin"); - yamlRestTestTask.dependsOn(bundle); - if (project.getPath().startsWith(":modules:")) { - runner.getClusters().forEach(c -> c.module(bundle.getArchiveFile())); - } else { - runner.getClusters().forEach(c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))); - } - }); - - // es-plugins may declare dependencies on additional modules, add those to the test cluster too. - project.afterEvaluate(p -> { - PluginPropertiesExtension pluginPropertiesExtension = project.getExtensions().findByType(PluginPropertiesExtension.class); - if (pluginPropertiesExtension != null) { // not all projects are defined as plugins - pluginPropertiesExtension.getExtendedPlugins().forEach(pluginName -> { - Project extensionProject = project.getProject().findProject(":modules:" + pluginName); - if (extensionProject != null) { // extension plugin may be defined, but not required to be a module - Zip extensionBundle = (Zip) extensionProject.getTasks().getByName("bundlePlugin"); - yamlRestTestTask.dependsOn(extensionBundle); - runner.getClusters().forEach(c -> c.module(extensionBundle.getArchiveFile())); - } - }); - } - }); - // setup IDE GradleUtils.setupIdeForTestSourceSet(project, yamlTestSourceSet); diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java-rest-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java-rest-test.properties new file mode 100644 index 00000000000..6eda5adec7b --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java-rest-test.properties @@ -0,0 +1,20 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +implementation-class=org.elasticsearch.gradle.test.rest.JavaRestTestPlugin diff --git a/modules/aggs-matrix-stats/build.gradle b/modules/aggs-matrix-stats/build.gradle index eb4aa9fc62c..044a64224b2 100644 --- a/modules/aggs-matrix-stats/build.gradle +++ b/modules/aggs-matrix-stats/build.gradle @@ -29,5 +29,3 @@ restResources { includeCore '_common', 'indices', 'cluster', 'index', 'search', 'nodes' } } - -integTest.enabled = false diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle index f631aa725d0..bce10572145 100644 --- a/modules/analysis-common/build.gradle +++ b/modules/analysis-common/build.gradle @@ -17,6 +17,7 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'Adds "built in" analyzers to Elasticsearch.' diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java similarity index 100% rename from modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java rename to modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java diff --git a/modules/geo/build.gradle b/modules/geo/build.gradle index c089b395df0..5381d0e4a04 100644 --- a/modules/geo/build.gradle +++ b/modules/geo/build.gradle @@ -31,6 +31,4 @@ restResources { artifacts { restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test')) } - -integTest.enabled = false test.enabled = false diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 2edf77c0c97..4b4fa8e81ef 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -17,6 +17,7 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources' diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java similarity index 100% rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java rename to modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index f0d5bf95dd0..1ff036353c0 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -43,8 +43,6 @@ restResources { } } -integTest.enabled = false - task copyDefaultGeoIp2DatabaseFiles(type: Copy) { from { zipTree(configurations.testCompileClasspath.files.find { it.name.contains('geolite2-databases') }) } into "${project.buildDir}/ingest-geoip" diff --git a/modules/ingest-user-agent/build.gradle b/modules/ingest-user-agent/build.gradle index 3d38ea99995..6870c26032e 100644 --- a/modules/ingest-user-agent/build.gradle +++ b/modules/ingest-user-agent/build.gradle @@ -33,4 +33,3 @@ testClusters.all { extraConfigFile 'ingest-user-agent/test-regexes.yml', file('src/test/test-regexes.yml') } -integTest.enabled = false diff --git a/modules/kibana/build.gradle b/modules/kibana/build.gradle index 8c102080edc..e432c328574 100644 --- a/modules/kibana/build.gradle +++ b/modules/kibana/build.gradle @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +apply plugin: 'elasticsearch.java-rest-test' esplugin { description 'Plugin exposing APIs for Kibana system indices' @@ -26,6 +27,7 @@ dependencies { api project(path: ':modules:reindex') } -testClusters.integTest { +testClusters.all { module file(project(':modules:reindex').tasks.bundlePlugin.archiveFile) } + diff --git a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java b/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java similarity index 100% rename from modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java rename to modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle index e71ce215fdf..9bfb72b1dc3 100644 --- a/modules/lang-mustache/build.gradle +++ b/modules/lang-mustache/build.gradle @@ -17,6 +17,8 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.java-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'Mustache scripting integration for Elasticsearch' diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java similarity index 100% rename from modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java rename to modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java similarity index 100% rename from modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java rename to modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java b/modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java similarity index 100% rename from modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java rename to modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java diff --git a/modules/mapper-extras/build.gradle b/modules/mapper-extras/build.gradle index b15d0c7f9a3..70e0048e4db 100644 --- a/modules/mapper-extras/build.gradle +++ b/modules/mapper-extras/build.gradle @@ -17,6 +17,7 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.java-rest-test' esplugin { description 'Adds advanced field mappers' diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java b/modules/mapper-extras/src/javaRestTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java similarity index 99% rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java rename to modules/mapper-extras/src/javaRestTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java index b5348ac9146..d805df51395 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java +++ b/modules/mapper-extras/src/javaRestTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an diff --git a/modules/parent-join/build.gradle b/modules/parent-join/build.gradle index ab07d943a3e..4fd3c4cb57c 100644 --- a/modules/parent-join/build.gradle +++ b/modules/parent-join/build.gradle @@ -17,6 +17,7 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'This module adds the support parent-child queries and aggregations' diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java rename to modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java rename to modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ParentIT.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentIT.java rename to modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ParentIT.java diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ChildQuerySearchIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/query/ChildQuerySearchIT.java rename to modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java rename to modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ParentChildTestCase.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java rename to modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ParentChildTestCase.java diff --git a/modules/percolator/build.gradle b/modules/percolator/build.gradle index bae92b94984..888bf603f51 100644 --- a/modules/percolator/build.gradle +++ b/modules/percolator/build.gradle @@ -17,6 +17,7 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'Percolator module adds capability to index queries and query these queries by specifying documents' @@ -29,22 +30,8 @@ dependencies { testImplementation project(':modules:geo') } -tasks.named('integTestRunner').configure { - exclude '**/PercolatorQuerySearchIT.class' -} - -tasks.register('internalClusterTest', Test) { - include '**/PercolatorQuerySearchIT.class' -} - -tasks.named('check').configure { - dependsOn 'internalClusterTest' -} - restResources { restApi { includeCore '_common', 'indices', 'index', 'search', 'msearch' } } - -integTest.enabled = false diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java similarity index 100% rename from modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java rename to modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle index b6a67994404..ba818caae18 100644 --- a/modules/rank-eval/build.gradle +++ b/modules/rank-eval/build.gradle @@ -17,6 +17,7 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'The Rank Eval module adds APIs to evaluate ranking quality.' diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java similarity index 100% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java rename to modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index b5eca353a89..7ccda51509b 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -25,6 +25,8 @@ import org.elasticsearch.gradle.info.BuildParams apply plugin: 'elasticsearch.test-with-dependencies' apply plugin: 'elasticsearch.jdk-download' apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.java-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'The Reindex module adds APIs to reindex from one index to another or update documents in place.' @@ -108,12 +110,12 @@ jdks { if (Os.isFamily(Os.FAMILY_WINDOWS)) { logger.warn("Disabling reindex-from-old tests because we can't get the pid file on windows") - integTest.runner { + javaRestTest.runner { systemProperty "tests.fromOld", "false" } } else if (rootProject.rootDir.toString().contains(" ")) { logger.warn("Disabling reindex-from-old tests because Elasticsearch 1.7 won't start with spaces in the path") - integTest.runner { + javaRestTest.runner { systemProperty "tests.fromOld", "false" } } else { @@ -154,7 +156,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { } } - integTest { + javaRestTest { dependsOn fixture runner { systemProperty "tests.fromOld", "true" @@ -166,3 +168,6 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { } } } + +//todo: remove once test/**/client/documentation can be deleted after https://github.com/elastic/docs/pull/1920 +testingConventions.enabled = false diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentation2IT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentation2IT.java new file mode 100644 index 00000000000..599bf79deeb --- /dev/null +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentation2IT.java @@ -0,0 +1,359 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.documentation; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.AbstractBulkByScrollRequestBuilder; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.BulkByScrollTask; +import org.elasticsearch.index.reindex.CancelTests; +import org.elasticsearch.index.reindex.DeleteByQueryAction; +import org.elasticsearch.index.reindex.DeleteByQueryRequestBuilder; +import org.elasticsearch.index.reindex.ReindexAction; +import org.elasticsearch.index.reindex.ReindexPlugin; +import org.elasticsearch.index.reindex.ReindexRequestBuilder; +import org.elasticsearch.index.reindex.RethrottleAction; +import org.elasticsearch.index.reindex.RethrottleRequestBuilder; +import org.elasticsearch.index.reindex.UpdateByQueryAction; +import org.elasticsearch.index.reindex.UpdateByQueryRequestBuilder; +import org.elasticsearch.index.shard.IndexingOperationListener; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskInfo; +import org.elasticsearch.test.ESIntegTestCase; +import org.hamcrest.Matcher; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.equalTo; + +//TODO: remove the "2" after https://github.com/elastic/docs/pull/1920 is merged and the "test" variant is removed. +public class ReindexDocumentation2IT extends ESIntegTestCase { + + // Semaphore used to allow & block indexing operations during the test + private static final Semaphore ALLOWED_OPERATIONS = new Semaphore(0); + private static final String INDEX_NAME = "source_index"; + + @Override + protected boolean ignoreExternalCluster() { + return true; + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(ReindexPlugin.class, ReindexCancellationPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return Collections.singletonList(ReindexPlugin.class); + } + + @Before + public void setup() { + client().admin().indices().prepareCreate(INDEX_NAME).get(); + } + + @SuppressWarnings("unused") + public void testReindex() { + Client client = client(); + // tag::reindex1 + BulkByScrollResponse response = + new ReindexRequestBuilder(client, ReindexAction.INSTANCE) + .source("source_index") + .destination("target_index") + .filter(QueryBuilders.matchQuery("category", "xzy")) // <1> + .get(); + // end::reindex1 + } + + @SuppressWarnings("unused") + public void testUpdateByQuery() { + Client client = client(); + client.admin().indices().prepareCreate("foo").get(); + client.admin().indices().prepareCreate("bar").get(); + client.admin().indices().preparePutMapping(INDEX_NAME).setType("_doc").setSource("cat", "type=keyword").get(); + { + // tag::update-by-query + UpdateByQueryRequestBuilder updateByQuery = + new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE); + updateByQuery.source("source_index").abortOnVersionConflict(false); + BulkByScrollResponse response = updateByQuery.get(); + // end::update-by-query + } + { + // tag::update-by-query-filter + UpdateByQueryRequestBuilder updateByQuery = + new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE); + updateByQuery.source("source_index") + .filter(QueryBuilders.termQuery("level", "awesome")) + .maxDocs(1000) + .script(new Script(ScriptType.INLINE, + "painless", + "ctx._source.awesome = 'absolutely'", + Collections.emptyMap())); + BulkByScrollResponse response = updateByQuery.get(); + // end::update-by-query-filter + + // validate order of string params to Script constructor + assertEquals(updateByQuery.request().getScript().getLang(), "painless"); + } + { + // tag::update-by-query-size + UpdateByQueryRequestBuilder updateByQuery = + new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE); + updateByQuery.source("source_index") + .source() + .setSize(500); + BulkByScrollResponse response = updateByQuery.get(); + // end::update-by-query-size + } + { + // tag::update-by-query-sort + UpdateByQueryRequestBuilder updateByQuery = + new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE); + updateByQuery.source("source_index") + .maxDocs(100) + .source() + .addSort("cat", SortOrder.DESC); + BulkByScrollResponse response = updateByQuery.get(); + // end::update-by-query-sort + } + { + // tag::update-by-query-script + UpdateByQueryRequestBuilder updateByQuery = + new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE); + updateByQuery.source("source_index") + .script(new Script( + ScriptType.INLINE, + "painless", + "if (ctx._source.awesome == 'absolutely') {" + + " ctx.op='noop'" + + "} else if (ctx._source.awesome == 'lame') {" + + " ctx.op='delete'" + + "} else {" + + "ctx._source.awesome = 'absolutely'}", + Collections.emptyMap())); + BulkByScrollResponse response = updateByQuery.get(); + // end::update-by-query-script + + // validate order of string params to Script constructor + assertEquals(updateByQuery.request().getScript().getLang(), "painless"); + } + { + // tag::update-by-query-multi-index + UpdateByQueryRequestBuilder updateByQuery = + new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE); + updateByQuery.source("foo", "bar"); + BulkByScrollResponse response = updateByQuery.get(); + // end::update-by-query-multi-index + } + { + // tag::update-by-query-routing + UpdateByQueryRequestBuilder updateByQuery = + new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE); + updateByQuery.source().setRouting("cat"); + BulkByScrollResponse response = updateByQuery.get(); + // end::update-by-query-routing + } + { + // tag::update-by-query-pipeline + UpdateByQueryRequestBuilder updateByQuery = + new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE); + updateByQuery.setPipeline("hurray"); + BulkByScrollResponse response = updateByQuery.get(); + // end::update-by-query-pipeline + } + } + + public void testTasks() throws Exception { + final Client client = client(); + final ReindexRequestBuilder builder = reindexAndPartiallyBlock(); + + { + // tag::update-by-query-list-tasks + ListTasksResponse tasksList = client.admin().cluster().prepareListTasks() + .setActions(UpdateByQueryAction.NAME).setDetailed(true).get(); + for (TaskInfo info: tasksList.getTasks()) { + TaskId taskId = info.getTaskId(); + BulkByScrollTask.Status status = + (BulkByScrollTask.Status) info.getStatus(); + // do stuff + } + // end::update-by-query-list-tasks + } + + TaskInfo mainTask = CancelTests.findTaskToCancel(ReindexAction.NAME, builder.request().getSlices()); + BulkByScrollTask.Status status = (BulkByScrollTask.Status) mainTask.getStatus(); + assertNull(status.getReasonCancelled()); + TaskId taskId = mainTask.getTaskId(); + { + // tag::update-by-query-get-task + GetTaskResponse get = client.admin().cluster().prepareGetTask(taskId).get(); + // end::update-by-query-get-task + } + { + // tag::update-by-query-cancel-task + // Cancel all update-by-query requests + client.admin().cluster().prepareCancelTasks() + .setActions(UpdateByQueryAction.NAME).get().getTasks(); + // Cancel a specific update-by-query request + client.admin().cluster().prepareCancelTasks() + .setTaskId(taskId).get().getTasks(); + // end::update-by-query-cancel-task + } + { + // tag::update-by-query-rethrottle + new RethrottleRequestBuilder(client, RethrottleAction.INSTANCE) + .setTaskId(taskId) + .setRequestsPerSecond(2.0f) + .get(); + // end::update-by-query-rethrottle + } + + // unblocking the blocked update + ALLOWED_OPERATIONS.release(builder.request().getSlices()); + } + + @SuppressWarnings("unused") + public void testDeleteByQuery() { + Client client = client(); + client.admin().indices().prepareCreate("persons").get(); + + // tag::delete-by-query-sync + BulkByScrollResponse response = + new DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .filter(QueryBuilders.matchQuery("gender", "male")) // <1> + .source("persons") // <2> + .get(); // <3> + long deleted = response.getDeleted(); // <4> + // end::delete-by-query-sync + + // tag::delete-by-query-async + new DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .filter(QueryBuilders.matchQuery("gender", "male")) // <1> + .source("persons") // <2> + .execute(new ActionListener() { // <3> + @Override + public void onResponse(BulkByScrollResponse response) { + long deleted = response.getDeleted(); // <4> + } + @Override + public void onFailure(Exception e) { + // Handle the exception + } + }); + // end::delete-by-query-async + } + + /** + * Similar to what CancelTests does: blocks some operations to be able to catch some tasks in running state + * @see CancelTests#testCancel(String, AbstractBulkByScrollRequestBuilder, CancelTests.CancelAssertion, Matcher) + */ + private ReindexRequestBuilder reindexAndPartiallyBlock() throws Exception { + final Client client = client(); + final int numDocs = randomIntBetween(10, 100); + ALLOWED_OPERATIONS.release(numDocs); + + indexRandom(true, false, true, IntStream.range(0, numDocs) + .mapToObj(i -> client().prepareIndex(INDEX_NAME, "_doc", Integer.toString(i)).setSource("n", Integer.toString(i))) + .collect(Collectors.toList())); + + // Checks that the all documents have been indexed and correctly counted + assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), numDocs); + assertThat(ALLOWED_OPERATIONS.drainPermits(), equalTo(0)); + + ReindexRequestBuilder builder = new ReindexRequestBuilder(client, ReindexAction.INSTANCE).source(INDEX_NAME) + .destination("target_index", "_doc"); + // Scroll by 1 so that cancellation is easier to control + builder.source().setSize(1); + + int numModifiedDocs = randomIntBetween(builder.request().getSlices() * 2, numDocs); + // chose to modify some of docs - rest is still blocked + ALLOWED_OPERATIONS.release(numModifiedDocs - builder.request().getSlices()); + + // Now execute the reindex action... + builder.execute(); + + // 10 seconds is usually fine but on heavily loaded machines this can take a while + assertBusy( + () -> { + assertTrue("Expected some queued threads", ALLOWED_OPERATIONS.hasQueuedThreads()); + assertEquals("Expected that no permits are available", 0, ALLOWED_OPERATIONS.availablePermits()); + }, + 1, TimeUnit.MINUTES); + return builder; + } + + public static class ReindexCancellationPlugin extends Plugin { + + @Override + public void onIndexModule(IndexModule indexModule) { + indexModule.addIndexOperationListener(new BlockingOperationListener()); + } + } + + public static class BlockingOperationListener implements IndexingOperationListener { + + @Override + public Engine.Index preIndex(ShardId shardId, Engine.Index index) { + return preCheck(index, index.type()); + } + + @Override + public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) { + return preCheck(delete, delete.type()); + } + + private T preCheck(T operation, String type) { + if (("_doc".equals(type) == false) || (operation.origin() != Engine.Operation.Origin.PRIMARY)) { + return operation; + } + + try { + if (ALLOWED_OPERATIONS.tryAcquire(30, TimeUnit.SECONDS)) { + return operation; + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new IllegalStateException("Something went wrong"); + } + } + +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java similarity index 99% rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java rename to modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java index fd8cf49cc39..6eed3520080 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java +++ b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java similarity index 96% rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java rename to modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java index 73745ca690d..683f02b6f96 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java +++ b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java similarity index 99% rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java rename to modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java index cc49d28471a..407b081abbe 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java +++ b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an diff --git a/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java b/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java index 20b25c805ac..09fe585a837 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java +++ b/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java @@ -62,6 +62,7 @@ import java.util.stream.IntStream; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.equalTo; +//TODO: delete this after https://github.com/elastic/docs/pull/1920 is merged public class ReindexDocumentationIT extends ESIntegTestCase { // Semaphore used to allow & block indexing operations during the test diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle index 6f2dff252d9..1d79fce831d 100644 --- a/modules/repository-url/build.gradle +++ b/modules/repository-url/build.gradle @@ -36,8 +36,6 @@ restResources { } } -integTest.enabled = false - // This directory is shared between two URL repositories and one FS repository in YAML integration tests File repositoryDir = new File(project.buildDir, "shared-repository") diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index 7ee95e38fdb..1ae3711f078 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -20,8 +20,12 @@ import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.test.RestIntegTestTask +import org.elasticsearch.gradle.test.rest.JavaRestTestPlugin +import org.elasticsearch.gradle.test.InternalClusterTestPlugin apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.java-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' /* TODOs: @@ -64,11 +68,11 @@ test { systemProperty 'es.set.netty.runtime.available.processors', 'false' } -integTestRunner { - /* - * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each - * other if we allow them to set the number of available processors as it's set-once in Netty. - */ +internalClusterTest { + systemProperty 'es.set.netty.runtime.available.processors', 'false' +} + +javaRestTestRunner { systemProperty 'es.set.netty.runtime.available.processors', 'false' } @@ -78,15 +82,29 @@ TaskProvider pooledTest = tasks.register("pooledTest", Test) { systemProperty 'es.use_unpooled_allocator', 'false' } -RestIntegTestTask pooledIntegTest = tasks.create("pooledIntegTest", RestIntegTestTask) { +TaskProvider pooledInternalClusterTest = tasks.register("pooledInternalClusterTest", Test) { + include '**/*IT.class' + systemProperty 'es.set.netty.runtime.available.processors', 'false' + systemProperty 'es.use_unpooled_allocator', 'false' + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs()) + setClasspath(internalTestSourceSet.getRuntimeClasspath()) +} + +RestIntegTestTask pooledJavaRestTest = tasks.create("pooledJavaRestTest", RestIntegTestTask) { runner { systemProperty 'es.set.netty.runtime.available.processors', 'false' + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet javaRestTestSourceSet = sourceSets.getByName(JavaRestTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(javaRestTestSourceSet.getOutput().getClassesDirs()) + setClasspath(javaRestTestSourceSet.getRuntimeClasspath()) } } -testClusters.pooledIntegTest { +testClusters.pooledJavaRestTest { systemProperty 'es.use_unpooled_allocator', 'false' } -check.dependsOn(pooledTest, pooledIntegTest) +check.dependsOn(pooledTest, pooledJavaRestTest, pooledInternalClusterTest) thirdPartyAudit { ignoreMissingClasses( diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/ESNetty4IntegTestCase.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/ESNetty4IntegTestCase.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java similarity index 99% rename from modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java index 7c89dc094ae..abe469a487b 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -20,6 +20,7 @@ package org.elasticsearch.http.netty4; import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.util.ReferenceCounted; import org.elasticsearch.ESNetty4IntegTestCase; import org.elasticsearch.common.collect.Tuple; @@ -31,7 +32,6 @@ import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import io.netty.handler.codec.http.HttpResponseStatus; import java.util.ArrayList; import java.util.Collection; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java rename to modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java rename to modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java