Merge pull request #14460 from rjernst/distro_tests

Add back integ tests to distributions

build.gradle

@@ -142,6 +142,7 @@ subprojects {
             substitute module("org.elasticsearch:test-framework:${version}") with project("${projectsPrefix}:test-framework")
           }
           substitute module("org.elasticsearch.distribution.zip:elasticsearch:${version}") with project("${projectsPrefix}:distribution:zip")
+          substitute module("org.elasticsearch.distribution.tar:elasticsearch:${version}") with project("${projectsPrefix}:distribution:tar")
         }
       }
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -24,6 +24,9 @@ import org.gradle.api.tasks.Input
 /** Configuration for an elasticsearch cluster, used for integration tests. */
 class ClusterConfiguration {
 
+    @Input
+    String distribution = 'zip'
+
     @Input
     int numNodes = 1
 
@@ -33,6 +36,9 @@ class ClusterConfiguration {
     @Input
     int transportPort = 9500
 
+    @Input
+    String jvmArgs = System.getProperty('tests.jvm.argline', '')
+
     Map<String, String> systemProperties = new HashMap<>()
 
     @Input

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -22,6 +22,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.ElasticsearchProperties
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
+import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.tasks.Copy
@@ -42,7 +43,7 @@ class ClusterFormationTasks {
             // no need to cluster formation if the task won't run!
             return
         }
-        addZipConfiguration(project)
+        configureDistributionDependency(project, config.distribution)
         File clusterDir = new File(project.buildDir, 'cluster' + File.separator + task.name)
         if (config.numNodes == 1) {
             addNodeStartupTasks(project, task, config, clusterDir)
@@ -57,22 +58,38 @@ class ClusterFormationTasks {
     }
 
     static void addNodeStartupTasks(Project project, Task task, ClusterConfiguration config, File baseDir) {
+        File pidFile = pidFile(baseDir)
         String clusterName = "${task.path.replace(':', '_').substring(1)}"
-        File home = new File(baseDir, "elasticsearch-${ElasticsearchProperties.version}")
+        File home = homeDir(baseDir, config.distribution)
-        List setupDependsOn = [project.configurations.elasticsearchZip]
+        Map esConfig = [
-        setupDependsOn.addAll(task.dependsOn)
+            'cluster.name': clusterName,
-        Task setup = project.tasks.create(name: task.name + '#setup', type: Copy, dependsOn: setupDependsOn) {
+            'http.port': config.httpPort,
-            from { project.zipTree(project.configurations.elasticsearchZip.singleFile) }
+            'transport.tcp.port': config.transportPort,
-            into baseDir
+            'pidfile': pidFile,
+            // TODO: make this work for multi node!
+            'discovery.zen.ping.unicast.hosts': "localhost:${config.transportPort}",
+            'path.repo': "${home}/repo",
+            'path.shared_data': "${home}/../",
+            // Define a node attribute so we can test that it exists
+            'node.testattr': 'test',
+            'repositories.url.allowed_urls': 'http://snapshot.test*'
+        ]
+        Map esEnv = [
+            'JAVA_HOME': System.getProperty('java.home'),
+            'ES_GC_OPTS': config.jvmArgs
+        ]
+
+        List setupDeps = [] // need to copy the deps, since start will later be added, which would create a circular task dep!
+        setupDeps.addAll(task.dependsOn)
+        Task setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setupDeps) {
+            delete baseDir
         }
+        setup = configureExtractTask(project, "${task.name}#extract", config.distribution, baseDir, setup)
         // chain setup tasks to maintain their order
-        setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setup) {
-            delete new File(home, 'plugins'), new File(home, 'data'), new File(home, 'logs')
-        }
         setup = project.tasks.create(name: "${task.name}#configure", type: DefaultTask, dependsOn: setup) << {
-            File configFile = new File(home, 'config' + File.separator + 'elasticsearch.yml')
+            File configFile = new File(home, 'config/elasticsearch.yml')
             logger.info("Configuring ${configFile}")
-            configFile.setText("cluster.name: ${clusterName}", 'UTF-8')
+            configFile.setText(esConfig.collect { key, value -> "${key}: ${value}" }.join('\n'), 'UTF-8')
         }
         for (Map.Entry<String, String> command : config.setupCommands.entrySet()) {
             Task nextSetup = project.tasks.create(name: "${task.name}#${command.getKey()}", type: Exec, dependsOn: setup) {
@@ -100,15 +117,7 @@ class ClusterFormationTasks {
             setup = nextSetup
         }
 
-        File pidFile = pidFile(baseDir)
+        List esArgs = config.systemProperties.collect {key, value -> "-D${key}=${value}"}
-        List esArgs = [
-            "-Des.http.port=${config.httpPort}",
-            "-Des.transport.tcp.port=${config.transportPort}",
-            "-Des.pidfile=${pidFile}",
-            "-Des.path.repo=${home}/repo",
-            "-Des.path.shared_data=${home}/../",
-        ]
-        esArgs.addAll(config.systemProperties.collect {key, value -> "-D${key}=${value}"})
         Closure esPostStartActions = { ant, logger ->
             ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${task.name}#start") {
                 and {
@@ -125,12 +134,13 @@ class ClusterFormationTasks {
                 throw new GradleException('Failed to start elasticsearch')
             }
         }
-        Task start;
+        Task start
         if (Os.isFamily(Os.FAMILY_WINDOWS)) {
             // elasticsearch.bat is spawned as it has no daemon mode
             start = project.tasks.create(name: "${task.name}#start", type: DefaultTask, dependsOn: setup) << {
                 // Fall back to Ant exec task as Gradle Exec task does not support spawning yet
                 ant.exec(executable: 'cmd', spawn: true, dir: home) {
+                    esEnv.each { env(key: key, value: value) }
                     (['/C', 'call', 'bin/elasticsearch'] + esArgs).each { arg(value: it) }
                 }
                 esPostStartActions(ant, logger)
@@ -141,6 +151,7 @@ class ClusterFormationTasks {
                 executable 'sh'
                 args 'bin/elasticsearch', '-d' // daemonize!
                 args esArgs
+                environment esEnv
                 errorOutput = new ByteArrayOutputStream()
                 doLast {
                     if (errorOutput.toString().isEmpty() == false) {
@@ -157,9 +168,44 @@ class ClusterFormationTasks {
         task.dependsOn(start)
     }
 
+    static Task configureExtractTask(Project project, String name, String distro, File baseDir, Task setup) {
+        List extractDependsOn = [project.configurations.elasticsearchDistro, setup]
+        Task extract
+        switch (distro) {
+            case 'zip':
+                extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
+                    from { project.zipTree(project.configurations.elasticsearchDistro.singleFile) }
+                    into baseDir
+                }
+                break;
+            case 'tar':
+                extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
+                    from { project.tarTree(project.resources.gzip(project.configurations.elasticsearchDistro.singleFile)) }
+                    into baseDir
+                }
+                break;
+            default:
+                throw new InvalidUserDataException("Unknown distribution: ${distro}")
+        }
+        return extract
+    }
+
+    static File homeDir(File baseDir, String distro) {
+        String path
+        switch (distro) {
+            case 'zip':
+            case 'tar':
+                path = "elasticsearch-${ElasticsearchProperties.version}"
+                break;
+            default:
+                throw new InvalidUserDataException("Unknown distribution: ${distro}")
+        }
+        return new File(baseDir, path)
+    }
+
     static void addNodeStopTask(Project project, Task task, File baseDir) {
         LazyPidReader pidFile = new LazyPidReader(pidFile: pidFile(baseDir))
-        Task stop = project.tasks.create(name: task.name + '#stop', type: Exec) {
+        Task stop = project.tasks.create(name: "${task.name}#stop", type: Exec) {
             if (Os.isFamily(Os.FAMILY_WINDOWS)) {
                 executable 'Taskkill'
                 args '/PID', pidFile, '/F'
@@ -187,13 +233,14 @@ class ClusterFormationTasks {
         return new File(dir, 'es.pid')
     }
 
-    static void addZipConfiguration(Project project) {
+    static void configureDistributionDependency(Project project, String distro) {
         String elasticsearchVersion = ElasticsearchProperties.version
+        String packaging = distro == 'tar' ? 'tgz' : distro
         project.configurations {
-            elasticsearchZip
+            elasticsearchDistro
         }
         project.dependencies {
-            elasticsearchZip "org.elasticsearch.distribution.zip:elasticsearch:${elasticsearchVersion}@zip"
+            elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${packaging}"
         }
     }
 }

distribution/build.gradle

@@ -36,11 +36,19 @@ buildscript {
 allprojects {
   project.ext {
     // this is common configuration for distributions, but we also add it here for the license check to use
-    deps = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api')
+    dependencyFiles = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api')
   }
 }
 
 subprojects {
+  /*****************************************************************************
+   *                            Rest test config                               *
+   *****************************************************************************/
+  apply plugin: 'elasticsearch.rest-test'
+  integTest {
+    includePackaged true
+  }
+
   /*****************************************************************************
    *                              Maven config                                 *
    *****************************************************************************/
@@ -50,8 +58,8 @@ subprojects {
   // we must create our own install task, because it is only added when the java plugin is added
   task install(type: Upload, description: "Installs the 'archives' artifacts into the local Maven repository.", group: 'Upload') {
     configuration = configurations.archives
-    MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention);
+    MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention)
-    repositoriesHandler.mavenInstaller();
+    repositoriesHandler.mavenInstaller()
   }
 
   // TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run...
@@ -81,7 +89,7 @@ subprojects {
     libFiles = copySpec {
       into 'lib'
       from project("${projectsPrefix}:core").jar
-      from deps
+      from dependencyFiles
     }
 
     configFiles = copySpec {
@@ -151,7 +159,7 @@ buildDeb.dependsOn createEmptyDir
 /*****************************************************************************
  *                         Deb and rpm configuration                         *
  *****************************************************************************/
-configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) {
+configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) {
   apply plugin: 'nebula.ospackage-base'
   ospackage {
     packageName = 'elasticsearch'
@@ -173,29 +181,17 @@ configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) {
     }
     directory('/etc/elasticsearch/scripts')
   }
-  if (project.name == 'deb') {
+  
-    task buildDeb(type: Deb) {
+  // TODO: re-enable tests when we have real rpm and deb distros!
-      dependsOn deps
+  integTest.enabled = false
-    }
-    artifacts {
-      archives buildDeb
-    }
-  } else if (project.name == 'rpm') {
-    task buildRpm(type: Rpm) {
-      dependsOn deps
-    }
-    artifacts {
-      archives buildRpm
-    }
-  }
 }
 
 // TODO: dependency checks should really be when building the jar itself, which would remove the need
 // for this hackery and instead we can do this inside the BuildPlugin
 task check(group: 'Verification', description: 'Runs all checks.') {} // dummy task!
 DependencyLicensesTask.configure(project) {
-  dependsOn = [deps]
+  dependsOn = [dependencyFiles]
-  dependencies = deps
+  dependencies = dependencyFiles
   mapping from: /lucene-.*/, to: 'lucene'
   mapping from: /jackson-.*/, to: 'jackson'
 }

distribution/deb/build.gradle

@@ -1,4 +1,28 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 
-/*task buildDeb(type: Deb) {
+task buildDeb(type: Deb) {
-  dependsOn deps
+  dependsOn dependencyFiles
-}*/
+}
+
+artifacts {
+  archives buildDeb
+}
+
+integTest.dependsOn buildDeb

distribution/rpm/build.gradle

@@ -1,4 +1,28 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 
-/*task buildRpm(type: Rpm) {
+task buildRpm(type: Rpm) {
-  dependsOn deps
+  dependsOn dependencyFiles
-}*/
+}
+
+artifacts {
+  archives buildRpm
+}
+
+integTest.dependsOn buildRpm

distribution/tar/build.gradle

@@ -1,10 +1,34 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 
-task buildTar(type: Tar, dependsOn: deps) {
+task buildTar(type: Tar, dependsOn: dependencyFiles) {
   baseName = 'elasticsearch'
   with archivesFiles
   compression = Compression.GZIP
 }
 
 artifacts {
+  'default' buildTar
   archives buildTar
 }
+
+integTest {
+  dependsOn buildTar
+  clusterConfig.distribution = 'tar'
+}

distribution/zip/build.gradle

@@ -1,5 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 
-task buildZip(type: Zip, dependsOn: deps) {
+task buildZip(type: Zip, dependsOn: dependencyFiles) {
   baseName = 'elasticsearch'
   with archivesFiles
 }
@@ -9,3 +27,5 @@ artifacts {
   archives buildZip
 }
 
+integTest.dependsOn buildZip
+