From 912fbb2211e4d04d305491de6e875c3e44d9475a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 27 Apr 2018 14:04:37 -0400 Subject: [PATCH] Reindex: Fold "from old" tests into reindex module (#30142) This folds the `:qa:reindex-from-old` project into the `:modules:reindex` project. This should speed up the build marginally by removing a single clsuter start up at the cost of having to wait for old versions of Elasticsearch to start up when checking reindex's integration tests. Those don't take that long so this feels worth it. --- modules/reindex/build.gradle | 62 ++++++++++++ .../remote}/ReindexFromOldRemoteIT.java | 6 +- qa/reindex-from-old/build.gradle | 94 ------------------- 3 files changed, 67 insertions(+), 95 deletions(-) rename {qa/reindex-from-old/src/test/java/org/elasticsearch/smoketest => modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote}/ReindexFromOldRemoteIT.java (95%) delete mode 100644 qa/reindex-from-old/build.gradle diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 479fe78cc80..f34f4cf52e0 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -17,6 +17,10 @@ * under the License. */ +import org.apache.tools.ant.taskdefs.condition.Os + +import static org.elasticsearch.gradle.BuildPlugin.getJavaHome + apply plugin: 'elasticsearch.test-with-dependencies' esplugin { @@ -60,3 +64,61 @@ thirdPartyAudit.excludes = [ 'org.apache.log.Hierarchy', 'org.apache.log.Logger', ] + +// Support for testing reindex-from-remote against old Elaticsearch versions +configurations { + oldesFixture + es2 + es1 + es090 +} + +dependencies { + oldesFixture project(':test:fixtures:old-elasticsearch') + /* Right now we just test against the latest version of each major we expect + * reindex-from-remote to work against. We could randomize the versions but + * that doesn't seem worth it at this point. */ + es2 'org.elasticsearch.distribution.zip:elasticsearch:2.4.5@zip' + es1 'org.elasticsearch:elasticsearch:1.7.6@zip' + es090 'org.elasticsearch:elasticsearch:0.90.13@zip' +} + +if (Os.isFamily(Os.FAMILY_WINDOWS)) { + // we can't get the pid files in windows so we skip reindex-from-old + integTestRunner.systemProperty "tests.fromOld", "false" +} else { + integTestRunner.systemProperty "tests.fromOld", "true" + /* Set up tasks to unzip and run the old versions of ES before running the + * integration tests. */ + for (String version : ['2', '1', '090']) { + Task unzip = task("unzipEs${version}", type: Sync) { + Configuration oldEsDependency = configurations['es' + version] + dependsOn oldEsDependency + /* Use a closure here to delay resolution of the dependency until we need + * it */ + from { + oldEsDependency.collect { zipTree(it) } + } + into temporaryDir + } + Task fixture = task("oldEs${version}Fixture", + type: org.elasticsearch.gradle.test.AntFixture) { + dependsOn project.configurations.oldesFixture + dependsOn unzip + executable = new File(project.runtimeJavaHome, 'bin/java') + env 'CLASSPATH', "${ -> project.configurations.oldesFixture.asPath }" + env 'JAVA_HOME', getJavaHome(it, 7) + args 'oldes.OldElasticsearch', + baseDir, + unzip.temporaryDir, + version == '090' + } + integTest.dependsOn fixture + integTestRunner { + /* Use a closure on the string to delay evaluation until right before we + * run the integration tests so that we can be sure that the file is + * ready. */ + systemProperty "es${version}.port", "${ -> fixture.addressAndPort }" + } + } +} diff --git a/qa/reindex-from-old/src/test/java/org/elasticsearch/smoketest/ReindexFromOldRemoteIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java similarity index 95% rename from qa/reindex-from-old/src/test/java/org/elasticsearch/smoketest/ReindexFromOldRemoteIT.java rename to modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java index 459aff34397..5d359053a66 100644 --- a/qa/reindex-from-old/src/test/java/org/elasticsearch/smoketest/ReindexFromOldRemoteIT.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.smoketest; +package org.elasticsearch.index.reindex.remote; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -27,6 +27,7 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Booleans; import org.elasticsearch.test.rest.ESRestTestCase; import java.io.IOException; @@ -38,6 +39,9 @@ import static org.hamcrest.Matchers.containsString; public class ReindexFromOldRemoteIT extends ESRestTestCase { private void oldEsTestCase(String portPropertyName, String requestsPerSecond) throws IOException { + boolean enabled = Booleans.parseBoolean(System.getProperty("tests.fromOld")); + assumeTrue("test is disabled, probably because this is windows", enabled); + int oldEsPort = Integer.parseInt(System.getProperty(portPropertyName)); try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) { try { diff --git a/qa/reindex-from-old/build.gradle b/qa/reindex-from-old/build.gradle deleted file mode 100644 index 8da714dd627..00000000000 --- a/qa/reindex-from-old/build.gradle +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -description = """\ -Tests reindex-from-remote against some specific versions of -Elasticsearch prior to 5.0. Versions of Elasticsearch >= 5.0 -should be able to use the standard launching mechanism which -is more flexible and reliable. -""" - - -import org.apache.tools.ant.taskdefs.condition.Os - -import static org.elasticsearch.gradle.BuildPlugin.getJavaHome - -apply plugin: 'elasticsearch.standalone-rest-test' -apply plugin: 'elasticsearch.rest-test' - -integTestCluster { - // Whitelist reindexing from the local node so we can test it. - setting 'reindex.remote.whitelist', '127.0.0.1:*' -} - -configurations { - oldesFixture - es2 - es1 - es090 -} - -dependencies { - oldesFixture project(':test:fixtures:old-elasticsearch') - /* Right now we just test against the latest version of each major we expect - * reindex-from-remote to work against. We could randomize the versions but - * that doesn't seem worth it at this point. */ - es2 'org.elasticsearch.distribution.zip:elasticsearch:2.4.5@zip' - es1 'org.elasticsearch:elasticsearch:1.7.6@zip' - es090 'org.elasticsearch:elasticsearch:0.90.13@zip' -} - -if (Os.isFamily(Os.FAMILY_WINDOWS)) { - // we can't get the pid files in windows so we skip that - integTest.enabled = false -} else { - /* Set up tasks to unzip and run the old versions of ES before running the - * integration tests. */ - for (String version : ['2', '1', '090']) { - Task unzip = task("unzipEs${version}", type: Sync) { - Configuration oldEsDependency = configurations['es' + version] - dependsOn oldEsDependency - /* Use a closure here to delay resolution of the dependency until we need - * it */ - from { - oldEsDependency.collect { zipTree(it) } - } - into temporaryDir - } - Task fixture = task("oldEs${version}Fixture", - type: org.elasticsearch.gradle.test.AntFixture) { - dependsOn project.configurations.oldesFixture - dependsOn unzip - executable = new File(project.runtimeJavaHome, 'bin/java') - env 'CLASSPATH', "${ -> project.configurations.oldesFixture.asPath }" - env 'JAVA_HOME', getJavaHome(it, 7) - args 'oldes.OldElasticsearch', - baseDir, - unzip.temporaryDir, - version == '090' - } - integTest.dependsOn fixture - integTestRunner { - /* Use a closure on the string to delay evaluation until right before we - * run the integration tests so that we can be sure that the file is - * ready. */ - systemProperty "es${version}.port", "${ -> fixture.addressAndPort }" - } - } -}