Build: Add back integ tests to distributions

rpm and deb are still skipped, but this configures rest tests to run for
zip and tgz distributions

closes #14361

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -24,6 +24,9 @@ import org.gradle.api.tasks.Input
 /** Configuration for an elasticsearch cluster, used for integration tests. */
 class ClusterConfiguration {
 
+    @Input
+    String distribution = 'zip'
+
     @Input
     int numNodes = 1
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -22,6 +22,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.ElasticsearchProperties
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
+import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.tasks.Copy
@@ -42,7 +43,7 @@ class ClusterFormationTasks {
             // no need to cluster formation if the task won't run!
             return
         }
-        addZipConfiguration(project)
+        configureDistributionDependency(project, config.distribution)
         File clusterDir = new File(project.buildDir, 'cluster' + File.separator + task.name)
         if (config.numNodes == 1) {
             addNodeStartupTasks(project, task, config, clusterDir)
@@ -57,22 +58,33 @@ class ClusterFormationTasks {
     }
 
     static void addNodeStartupTasks(Project project, Task task, ClusterConfiguration config, File baseDir) {
+        File pidFile = pidFile(baseDir)
         String clusterName = "${task.path.replace(':', '_').substring(1)}"
-        File home = new File(baseDir, "elasticsearch-${ElasticsearchProperties.version}")
+        File home = homeDir(baseDir, config.distribution)
-        List setupDependsOn = [project.configurations.elasticsearchZip]
+        List setupDeps = [] // need to copy the deps, since start will later be added, which would create a circular task dep!
-        setupDependsOn.addAll(task.dependsOn)
+        setupDeps.addAll(task.dependsOn)
-        Task setup = project.tasks.create(name: task.name + '#setup', type: Copy, dependsOn: setupDependsOn) {
+        Task setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setupDeps) {
-            from { project.zipTree(project.configurations.elasticsearchZip.singleFile) }
+            delete baseDir
-            into baseDir
         }
+        setup = configureExtractTask(project, "${task.name}#extract", config.distribution, baseDir, setup)
         // chain setup tasks to maintain their order
-        setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setup) {
-            delete new File(home, 'plugins'), new File(home, 'data'), new File(home, 'logs')
-        }
         setup = project.tasks.create(name: "${task.name}#configure", type: DefaultTask, dependsOn: setup) << {
-            File configFile = new File(home, 'config' + File.separator + 'elasticsearch.yml')
+            File configFile = new File(home, 'config/elasticsearch.yml')
             logger.info("Configuring ${configFile}")
-            configFile.setText("cluster.name: ${clusterName}", 'UTF-8')
+            Map esConfig = [
+                'cluster.name': clusterName,
+                'http.port': config.httpPort,
+                'transport.tcp.port': config.transportPort,
+                'pidfile': pidFile,
+                // TODO: make this work for multi node!
+                'discovery.zen.ping.unicast.hosts': "localhost:${config.transportPort}",
+                'path.repo': "${home}/repo",
+                'path.shared_data': "${home}/../",
+                // Define a node attribute so we can test that it exists
+                'node.testattr': 'test',
+                'repositories.url.allowed_urls': 'http://snapshot.test*'
+            ]
+            configFile.setText(esConfig.collect { key, value -> "${key}: ${value}" }.join('\n'), 'UTF-8')
         }
         for (Map.Entry<String, String> command : config.setupCommands.entrySet()) {
             Task nextSetup = project.tasks.create(name: "${task.name}#${command.getKey()}", type: Exec, dependsOn: setup) {
@@ -100,15 +112,7 @@ class ClusterFormationTasks {
             setup = nextSetup
         }
 
-        File pidFile = pidFile(baseDir)
+        List esArgs = config.systemProperties.collect {key, value -> "-D${key}=${value}"}
-        List esArgs = [
-            "-Des.http.port=${config.httpPort}",
-            "-Des.transport.tcp.port=${config.transportPort}",
-            "-Des.pidfile=${pidFile}",
-            "-Des.path.repo=${home}/repo",
-            "-Des.path.shared_data=${home}/../",
-        ]
-        esArgs.addAll(config.systemProperties.collect {key, value -> "-D${key}=${value}"})
         Closure esPostStartActions = { ant, logger ->
             ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${task.name}#start") {
                 and {
@@ -157,9 +161,44 @@ class ClusterFormationTasks {
         task.dependsOn(start)
     }
 
+    static Task configureExtractTask(Project project, String name, String distro, File baseDir, Task setup) {
+        List extractDependsOn = [project.configurations.elasticsearchDistro, setup]
+        Task extract
+        switch (distro) {
+            case 'zip':
+                extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
+                    from { project.zipTree(project.configurations.elasticsearchDistro.singleFile) }
+                    into baseDir
+                }
+                break;
+            case 'tar':
+                extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
+                    from { project.tarTree(project.resources.gzip(project.configurations.elasticsearchDistro.singleFile)) }
+                    into baseDir
+                }
+                break;
+            default:
+                throw new InvalidUserDataException("Unknown distribution: ${distro}")
+        }
+        return extract
+    }
+
+    static File homeDir(File baseDir, String distro) {
+        String path
+        switch (distro) {
+            case 'zip':
+            case 'tar':
+                path = "elasticsearch-${ElasticsearchProperties.version}"
+                break;
+            default:
+                throw new InvalidUserDataException("Unknown distribution: ${distro}")
+        }
+        return new File(baseDir, path)
+    }
+
     static void addNodeStopTask(Project project, Task task, File baseDir) {
         LazyPidReader pidFile = new LazyPidReader(pidFile: pidFile(baseDir))
-        Task stop = project.tasks.create(name: task.name + '#stop', type: Exec) {
+        Task stop = project.tasks.create(name: "${task.name}#stop", type: Exec) {
             if (Os.isFamily(Os.FAMILY_WINDOWS)) {
                 executable 'Taskkill'
                 args '/PID', pidFile, '/F'
@@ -187,13 +226,13 @@ class ClusterFormationTasks {
         return new File(dir, 'es.pid')
     }
 
-    static void addZipConfiguration(Project project) {
+    static void configureDistributionDependency(Project project, String distro) {
         String elasticsearchVersion = ElasticsearchProperties.version
         project.configurations {
-            elasticsearchZip
+            elasticsearchDistro
         }
         project.dependencies {
-            elasticsearchZip "org.elasticsearch.distribution.zip:elasticsearch:${elasticsearchVersion}@zip"
+            elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${distro}"
         }
     }
 }

distribution/build.gradle

@@ -36,11 +36,17 @@ buildscript {
 allprojects {
   project.ext {
     // this is common configuration for distributions, but we also add it here for the license check to use
-    deps = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api')
+    dependencyFiles = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api')
   }
 }
 
 subprojects {
+  apply plugin: 'elasticsearch.rest-test'
+
+  integTest {
+    includePackaged true
+  }
+
   /*****************************************************************************
    *                              Maven config                                 *
    *****************************************************************************/
@@ -50,8 +56,8 @@ subprojects {
   // we must create our own install task, because it is only added when the java plugin is added
   task install(type: Upload, description: "Installs the 'archives' artifacts into the local Maven repository.", group: 'Upload') {
     configuration = configurations.archives
-    MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention);
+    MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention)
-    repositoriesHandler.mavenInstaller();
+    repositoriesHandler.mavenInstaller()
   }
 
   // TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run...
@@ -81,7 +87,7 @@ subprojects {
     libFiles = copySpec {
       into 'lib'
       from project("${projectsPrefix}:core").jar
-      from deps
+      from dependencyFiles
     }
 
     configFiles = copySpec {
@@ -151,7 +157,7 @@ buildDeb.dependsOn createEmptyDir
 /*****************************************************************************
  *                         Deb and rpm configuration                         *
  *****************************************************************************/
-configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) {
+configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) {
   apply plugin: 'nebula.ospackage-base'
   ospackage {
     packageName = 'elasticsearch'
@@ -173,29 +179,17 @@ configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) {
     }
     directory('/etc/elasticsearch/scripts')
   }
-  if (project.name == 'deb') {
+  
-    task buildDeb(type: Deb) {
+  // TODO: re-enable tests when we have real rpm and deb distros!
-      dependsOn deps
+  integTest.enabled = false
-    }
-    artifacts {
-      archives buildDeb
-    }
-  } else if (project.name == 'rpm') {
-    task buildRpm(type: Rpm) {
-      dependsOn deps
-    }
-    artifacts {
-      archives buildRpm
-    }
-  }
 }
 
 // TODO: dependency checks should really be when building the jar itself, which would remove the need
 // for this hackery and instead we can do this inside the BuildPlugin
 task check(group: 'Verification', description: 'Runs all checks.') {} // dummy task!
 DependencyLicensesTask.configure(project) {
-  dependsOn = [deps]
+  dependsOn = [dependencyFiles]
-  dependencies = deps
+  dependencies = dependencyFiles
   mapping from: /lucene-.*/, to: 'lucene'
   mapping from: /jackson-.*/, to: 'jackson'
 }

distribution/deb/build.gradle

@@ -1,4 +1,25 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 
-/*task buildDeb(type: Deb) {
+task buildDeb(type: Deb) {
-  dependsOn deps
+  dependsOn dependencyFiles
-}*/
+}
+artifacts {
+  archives buildDeb
+}

distribution/rpm/build.gradle

@@ -1,4 +1,25 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 
-/*task buildRpm(type: Rpm) {
+task buildRpm(type: Rpm) {
-  dependsOn deps
+  dependsOn dependencyFiles
-}*/
+}
+artifacts {
+  archives buildRpm
+}

distribution/tar/build.gradle

@@ -1,5 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 
-task buildTar(type: Tar, dependsOn: deps) {
+task buildTar(type: Tar, dependsOn: dependencyFiles) {
   baseName = 'elasticsearch'
   with archivesFiles
   compression = Compression.GZIP

distribution/zip/build.gradle

@@ -1,5 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 
-task buildZip(type: Zip, dependsOn: deps) {
+task buildZip(type: Zip, dependsOn: dependencyFiles) {
   baseName = 'elasticsearch'
   with archivesFiles
 }