From f5556224f519a39038c073cc55fe64811ec43fb5 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 2 Nov 2015 17:48:15 -0800 Subject: [PATCH 1/3] Build: Add back integ tests to distributions rpm and deb are still skipped, but this configures rest tests to run for zip and tgz distributions closes #14361 --- .../gradle/test/ClusterConfiguration.groovy | 3 + .../gradle/test/ClusterFormationTasks.groovy | 89 +++++++++++++------ distribution/build.gradle | 38 ++++---- distribution/deb/build.gradle | 27 +++++- distribution/rpm/build.gradle | 27 +++++- distribution/tar/build.gradle | 20 ++++- distribution/zip/build.gradle | 20 ++++- 7 files changed, 169 insertions(+), 55 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index 388259c7a9c..ab68c07a1ef 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -24,6 +24,9 @@ import org.gradle.api.tasks.Input /** Configuration for an elasticsearch cluster, used for integration tests. */ class ClusterConfiguration { + @Input + String distribution = 'zip' + @Input int numNodes = 1 diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 3a06da929dd..d90143de931 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -22,6 +22,7 @@ import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.ElasticsearchProperties import org.gradle.api.DefaultTask import org.gradle.api.GradleException +import org.gradle.api.InvalidUserDataException import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.tasks.Copy @@ -42,7 +43,7 @@ class ClusterFormationTasks { // no need to cluster formation if the task won't run! return } - addZipConfiguration(project) + configureDistributionDependency(project, config.distribution) File clusterDir = new File(project.buildDir, 'cluster' + File.separator + task.name) if (config.numNodes == 1) { addNodeStartupTasks(project, task, config, clusterDir) @@ -57,22 +58,33 @@ class ClusterFormationTasks { } static void addNodeStartupTasks(Project project, Task task, ClusterConfiguration config, File baseDir) { + File pidFile = pidFile(baseDir) String clusterName = "${task.path.replace(':', '_').substring(1)}" - File home = new File(baseDir, "elasticsearch-${ElasticsearchProperties.version}") - List setupDependsOn = [project.configurations.elasticsearchZip] - setupDependsOn.addAll(task.dependsOn) - Task setup = project.tasks.create(name: task.name + '#setup', type: Copy, dependsOn: setupDependsOn) { - from { project.zipTree(project.configurations.elasticsearchZip.singleFile) } - into baseDir + File home = homeDir(baseDir, config.distribution) + List setupDeps = [] // need to copy the deps, since start will later be added, which would create a circular task dep! + setupDeps.addAll(task.dependsOn) + Task setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setupDeps) { + delete baseDir } + setup = configureExtractTask(project, "${task.name}#extract", config.distribution, baseDir, setup) // chain setup tasks to maintain their order - setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setup) { - delete new File(home, 'plugins'), new File(home, 'data'), new File(home, 'logs') - } setup = project.tasks.create(name: "${task.name}#configure", type: DefaultTask, dependsOn: setup) << { - File configFile = new File(home, 'config' + File.separator + 'elasticsearch.yml') + File configFile = new File(home, 'config/elasticsearch.yml') logger.info("Configuring ${configFile}") - configFile.setText("cluster.name: ${clusterName}", 'UTF-8') + Map esConfig = [ + 'cluster.name': clusterName, + 'http.port': config.httpPort, + 'transport.tcp.port': config.transportPort, + 'pidfile': pidFile, + // TODO: make this work for multi node! + 'discovery.zen.ping.unicast.hosts': "localhost:${config.transportPort}", + 'path.repo': "${home}/repo", + 'path.shared_data': "${home}/../", + // Define a node attribute so we can test that it exists + 'node.testattr': 'test', + 'repositories.url.allowed_urls': 'http://snapshot.test*' + ] + configFile.setText(esConfig.collect { key, value -> "${key}: ${value}" }.join('\n'), 'UTF-8') } for (Map.Entry command : config.setupCommands.entrySet()) { Task nextSetup = project.tasks.create(name: "${task.name}#${command.getKey()}", type: Exec, dependsOn: setup) { @@ -100,15 +112,7 @@ class ClusterFormationTasks { setup = nextSetup } - File pidFile = pidFile(baseDir) - List esArgs = [ - "-Des.http.port=${config.httpPort}", - "-Des.transport.tcp.port=${config.transportPort}", - "-Des.pidfile=${pidFile}", - "-Des.path.repo=${home}/repo", - "-Des.path.shared_data=${home}/../", - ] - esArgs.addAll(config.systemProperties.collect {key, value -> "-D${key}=${value}"}) + List esArgs = config.systemProperties.collect {key, value -> "-D${key}=${value}"} Closure esPostStartActions = { ant, logger -> ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${task.name}#start") { and { @@ -157,9 +161,44 @@ class ClusterFormationTasks { task.dependsOn(start) } + static Task configureExtractTask(Project project, String name, String distro, File baseDir, Task setup) { + List extractDependsOn = [project.configurations.elasticsearchDistro, setup] + Task extract + switch (distro) { + case 'zip': + extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) { + from { project.zipTree(project.configurations.elasticsearchDistro.singleFile) } + into baseDir + } + break; + case 'tar': + extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) { + from { project.tarTree(project.resources.gzip(project.configurations.elasticsearchDistro.singleFile)) } + into baseDir + } + break; + default: + throw new InvalidUserDataException("Unknown distribution: ${distro}") + } + return extract + } + + static File homeDir(File baseDir, String distro) { + String path + switch (distro) { + case 'zip': + case 'tar': + path = "elasticsearch-${ElasticsearchProperties.version}" + break; + default: + throw new InvalidUserDataException("Unknown distribution: ${distro}") + } + return new File(baseDir, path) + } + static void addNodeStopTask(Project project, Task task, File baseDir) { LazyPidReader pidFile = new LazyPidReader(pidFile: pidFile(baseDir)) - Task stop = project.tasks.create(name: task.name + '#stop', type: Exec) { + Task stop = project.tasks.create(name: "${task.name}#stop", type: Exec) { if (Os.isFamily(Os.FAMILY_WINDOWS)) { executable 'Taskkill' args '/PID', pidFile, '/F' @@ -187,13 +226,13 @@ class ClusterFormationTasks { return new File(dir, 'es.pid') } - static void addZipConfiguration(Project project) { + static void configureDistributionDependency(Project project, String distro) { String elasticsearchVersion = ElasticsearchProperties.version project.configurations { - elasticsearchZip + elasticsearchDistro } project.dependencies { - elasticsearchZip "org.elasticsearch.distribution.zip:elasticsearch:${elasticsearchVersion}@zip" + elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${distro}" } } } diff --git a/distribution/build.gradle b/distribution/build.gradle index 0ea94659d36..beb07687266 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -36,11 +36,17 @@ buildscript { allprojects { project.ext { // this is common configuration for distributions, but we also add it here for the license check to use - deps = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api') + dependencyFiles = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api') } } subprojects { + apply plugin: 'elasticsearch.rest-test' + + integTest { + includePackaged true + } + /***************************************************************************** * Maven config * *****************************************************************************/ @@ -50,8 +56,8 @@ subprojects { // we must create our own install task, because it is only added when the java plugin is added task install(type: Upload, description: "Installs the 'archives' artifacts into the local Maven repository.", group: 'Upload') { configuration = configurations.archives - MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention); - repositoriesHandler.mavenInstaller(); + MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention) + repositoriesHandler.mavenInstaller() } // TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run... @@ -81,7 +87,7 @@ subprojects { libFiles = copySpec { into 'lib' from project("${projectsPrefix}:core").jar - from deps + from dependencyFiles } configFiles = copySpec { @@ -151,7 +157,7 @@ buildDeb.dependsOn createEmptyDir /***************************************************************************** * Deb and rpm configuration * *****************************************************************************/ -configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) { +configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) { apply plugin: 'nebula.ospackage-base' ospackage { packageName = 'elasticsearch' @@ -173,29 +179,17 @@ configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) { } directory('/etc/elasticsearch/scripts') } - if (project.name == 'deb') { - task buildDeb(type: Deb) { - dependsOn deps - } - artifacts { - archives buildDeb - } - } else if (project.name == 'rpm') { - task buildRpm(type: Rpm) { - dependsOn deps - } - artifacts { - archives buildRpm - } - } + + // TODO: re-enable tests when we have real rpm and deb distros! + integTest.enabled = false } // TODO: dependency checks should really be when building the jar itself, which would remove the need // for this hackery and instead we can do this inside the BuildPlugin task check(group: 'Verification', description: 'Runs all checks.') {} // dummy task! DependencyLicensesTask.configure(project) { - dependsOn = [deps] - dependencies = deps + dependsOn = [dependencyFiles] + dependencies = dependencyFiles mapping from: /lucene-.*/, to: 'lucene' mapping from: /jackson-.*/, to: 'jackson' } diff --git a/distribution/deb/build.gradle b/distribution/deb/build.gradle index 66a6c3b5426..7f1cf12d970 100644 --- a/distribution/deb/build.gradle +++ b/distribution/deb/build.gradle @@ -1,4 +1,25 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ -/*task buildDeb(type: Deb) { - dependsOn deps -}*/ +task buildDeb(type: Deb) { + dependsOn dependencyFiles +} +artifacts { + archives buildDeb +} diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle index fdf800ba2a4..25af1dee341 100644 --- a/distribution/rpm/build.gradle +++ b/distribution/rpm/build.gradle @@ -1,4 +1,25 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ -/*task buildRpm(type: Rpm) { - dependsOn deps -}*/ +task buildRpm(type: Rpm) { + dependsOn dependencyFiles +} +artifacts { + archives buildRpm +} diff --git a/distribution/tar/build.gradle b/distribution/tar/build.gradle index f5991f349f7..8df7ff39534 100644 --- a/distribution/tar/build.gradle +++ b/distribution/tar/build.gradle @@ -1,5 +1,23 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ -task buildTar(type: Tar, dependsOn: deps) { +task buildTar(type: Tar, dependsOn: dependencyFiles) { baseName = 'elasticsearch' with archivesFiles compression = Compression.GZIP diff --git a/distribution/zip/build.gradle b/distribution/zip/build.gradle index 53ad5bc0762..98c51c24106 100644 --- a/distribution/zip/build.gradle +++ b/distribution/zip/build.gradle @@ -1,5 +1,23 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ -task buildZip(type: Zip, dependsOn: deps) { +task buildZip(type: Zip, dependsOn: dependencyFiles) { baseName = 'elasticsearch' with archivesFiles } From 3e4f87d4d62b86eabacf392c66d1f9097089b9d1 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 2 Nov 2015 22:19:29 -0800 Subject: [PATCH 2/3] Fix tar issues and build distro dependency for integ tests --- build.gradle | 1 + .../elasticsearch/gradle/test/ClusterFormationTasks.groovy | 3 ++- distribution/build.gradle | 4 +++- distribution/deb/build.gradle | 3 +++ distribution/rpm/build.gradle | 3 +++ distribution/tar/build.gradle | 6 ++++++ distribution/zip/build.gradle | 2 ++ 7 files changed, 20 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index b3e5e09e07a..369bd30cc2f 100644 --- a/build.gradle +++ b/build.gradle @@ -142,6 +142,7 @@ subprojects { substitute module("org.elasticsearch:test-framework:${version}") with project("${projectsPrefix}:test-framework") } substitute module("org.elasticsearch.distribution.zip:elasticsearch:${version}") with project("${projectsPrefix}:distribution:zip") + substitute module("org.elasticsearch.distribution.tar:elasticsearch:${version}") with project("${projectsPrefix}:distribution:tar") } } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index d90143de931..1f88eb63ee1 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -228,11 +228,12 @@ class ClusterFormationTasks { static void configureDistributionDependency(Project project, String distro) { String elasticsearchVersion = ElasticsearchProperties.version + String packaging = distro == 'tar' ? 'tgz' : distro project.configurations { elasticsearchDistro } project.dependencies { - elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${distro}" + elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${packaging}" } } } diff --git a/distribution/build.gradle b/distribution/build.gradle index beb07687266..7770c0d98ab 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -41,8 +41,10 @@ allprojects { } subprojects { + /***************************************************************************** + * Rest test config * + *****************************************************************************/ apply plugin: 'elasticsearch.rest-test' - integTest { includePackaged true } diff --git a/distribution/deb/build.gradle b/distribution/deb/build.gradle index 7f1cf12d970..24eabe78598 100644 --- a/distribution/deb/build.gradle +++ b/distribution/deb/build.gradle @@ -20,6 +20,9 @@ task buildDeb(type: Deb) { dependsOn dependencyFiles } + artifacts { archives buildDeb } + +integTest.dependsOn buildDeb diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle index 25af1dee341..04f0c0bda9b 100644 --- a/distribution/rpm/build.gradle +++ b/distribution/rpm/build.gradle @@ -20,6 +20,9 @@ task buildRpm(type: Rpm) { dependsOn dependencyFiles } + artifacts { archives buildRpm } + +integTest.dependsOn buildRpm diff --git a/distribution/tar/build.gradle b/distribution/tar/build.gradle index 8df7ff39534..b49058b3e42 100644 --- a/distribution/tar/build.gradle +++ b/distribution/tar/build.gradle @@ -24,5 +24,11 @@ task buildTar(type: Tar, dependsOn: dependencyFiles) { } artifacts { + 'default' buildTar archives buildTar } + +integTest { + dependsOn buildTar + clusterConfig.distribution = 'tar' +} diff --git a/distribution/zip/build.gradle b/distribution/zip/build.gradle index 98c51c24106..d636e66f152 100644 --- a/distribution/zip/build.gradle +++ b/distribution/zip/build.gradle @@ -27,3 +27,5 @@ artifacts { archives buildZip } +integTest.dependsOn buildZip + From bd158e5e88d0a20a1c387a1daa91da051ce7f621 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 3 Nov 2015 10:13:02 -0800 Subject: [PATCH 3/3] Add environment vars for JAVA_HOME and ES_GC_OPTS back to integ tests --- .../gradle/test/ClusterConfiguration.groovy | 3 ++ .../gradle/test/ClusterFormationTasks.groovy | 35 +++++++++++-------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index ab68c07a1ef..2e2062e175f 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -36,6 +36,9 @@ class ClusterConfiguration { @Input int transportPort = 9500 + @Input + String jvmArgs = System.getProperty('tests.jvm.argline', '') + Map systemProperties = new HashMap<>() @Input diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 1f88eb63ee1..3a6c8865d2a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -61,6 +61,24 @@ class ClusterFormationTasks { File pidFile = pidFile(baseDir) String clusterName = "${task.path.replace(':', '_').substring(1)}" File home = homeDir(baseDir, config.distribution) + Map esConfig = [ + 'cluster.name': clusterName, + 'http.port': config.httpPort, + 'transport.tcp.port': config.transportPort, + 'pidfile': pidFile, + // TODO: make this work for multi node! + 'discovery.zen.ping.unicast.hosts': "localhost:${config.transportPort}", + 'path.repo': "${home}/repo", + 'path.shared_data': "${home}/../", + // Define a node attribute so we can test that it exists + 'node.testattr': 'test', + 'repositories.url.allowed_urls': 'http://snapshot.test*' + ] + Map esEnv = [ + 'JAVA_HOME': System.getProperty('java.home'), + 'ES_GC_OPTS': config.jvmArgs + ] + List setupDeps = [] // need to copy the deps, since start will later be added, which would create a circular task dep! setupDeps.addAll(task.dependsOn) Task setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setupDeps) { @@ -71,19 +89,6 @@ class ClusterFormationTasks { setup = project.tasks.create(name: "${task.name}#configure", type: DefaultTask, dependsOn: setup) << { File configFile = new File(home, 'config/elasticsearch.yml') logger.info("Configuring ${configFile}") - Map esConfig = [ - 'cluster.name': clusterName, - 'http.port': config.httpPort, - 'transport.tcp.port': config.transportPort, - 'pidfile': pidFile, - // TODO: make this work for multi node! - 'discovery.zen.ping.unicast.hosts': "localhost:${config.transportPort}", - 'path.repo': "${home}/repo", - 'path.shared_data': "${home}/../", - // Define a node attribute so we can test that it exists - 'node.testattr': 'test', - 'repositories.url.allowed_urls': 'http://snapshot.test*' - ] configFile.setText(esConfig.collect { key, value -> "${key}: ${value}" }.join('\n'), 'UTF-8') } for (Map.Entry command : config.setupCommands.entrySet()) { @@ -129,12 +134,13 @@ class ClusterFormationTasks { throw new GradleException('Failed to start elasticsearch') } } - Task start; + Task start if (Os.isFamily(Os.FAMILY_WINDOWS)) { // elasticsearch.bat is spawned as it has no daemon mode start = project.tasks.create(name: "${task.name}#start", type: DefaultTask, dependsOn: setup) << { // Fall back to Ant exec task as Gradle Exec task does not support spawning yet ant.exec(executable: 'cmd', spawn: true, dir: home) { + esEnv.each { env(key: key, value: value) } (['/C', 'call', 'bin/elasticsearch'] + esArgs).each { arg(value: it) } } esPostStartActions(ant, logger) @@ -145,6 +151,7 @@ class ClusterFormationTasks { executable 'sh' args 'bin/elasticsearch', '-d' // daemonize! args esArgs + environment esEnv errorOutput = new ByteArrayOutputStream() doLast { if (errorOutput.toString().isEmpty() == false) {