Build: Add back integ tests to distributions

rpm and deb are still skipped, but this configures rest tests to run for zip and tgz distributions closes #14361
2015-11-02 17:48:15 -08:00 · 2015-11-02 17:48:15 -08:00 · f5556224f5
parent c44fe5c907
commit f5556224f5
7 changed files with 169 additions and 55 deletions
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
@ -24,6 +24,9 @@ import org.gradle.api.tasks.Input
 /** Configuration for an elasticsearch cluster, used for integration tests. */
 class ClusterConfiguration {

+    @Input
+    String distribution = 'zip'
+
    @Input
    int numNodes = 1

--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@ -22,6 +22,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.ElasticsearchProperties
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
+import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.tasks.Copy
@ -42,7 +43,7 @@ class ClusterFormationTasks {
            // no need to cluster formation if the task won't run!
            return
        }
-        addZipConfiguration(project)
+        configureDistributionDependency(project, config.distribution)
        File clusterDir = new File(project.buildDir, 'cluster' + File.separator + task.name)
        if (config.numNodes == 1) {
            addNodeStartupTasks(project, task, config, clusterDir)
@ -57,22 +58,33 @@ class ClusterFormationTasks {
    }

    static void addNodeStartupTasks(Project project, Task task, ClusterConfiguration config, File baseDir) {
+        File pidFile = pidFile(baseDir)
        String clusterName = "${task.path.replace(':', '_').substring(1)}"
-        File home = new File(baseDir, "elasticsearch-${ElasticsearchProperties.version}")
-        List setupDependsOn = [project.configurations.elasticsearchZip]
-        setupDependsOn.addAll(task.dependsOn)
-        Task setup = project.tasks.create(name: task.name + '#setup', type: Copy, dependsOn: setupDependsOn) {
-            from { project.zipTree(project.configurations.elasticsearchZip.singleFile) }
-            into baseDir
+        File home = homeDir(baseDir, config.distribution)
+        List setupDeps = [] // need to copy the deps, since start will later be added, which would create a circular task dep!
+        setupDeps.addAll(task.dependsOn)
+        Task setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setupDeps) {
+            delete baseDir
        }
+        setup = configureExtractTask(project, "${task.name}#extract", config.distribution, baseDir, setup)
        // chain setup tasks to maintain their order
-        setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: setup) {
-            delete new File(home, 'plugins'), new File(home, 'data'), new File(home, 'logs')
-        }
        setup = project.tasks.create(name: "${task.name}#configure", type: DefaultTask, dependsOn: setup) << {
-            File configFile = new File(home, 'config' + File.separator + 'elasticsearch.yml')
+            File configFile = new File(home, 'config/elasticsearch.yml')
            logger.info("Configuring ${configFile}")
-            configFile.setText("cluster.name: ${clusterName}", 'UTF-8')
+            Map esConfig = [
+                'cluster.name': clusterName,
+                'http.port': config.httpPort,
+                'transport.tcp.port': config.transportPort,
+                'pidfile': pidFile,
+                // TODO: make this work for multi node!
+                'discovery.zen.ping.unicast.hosts': "localhost:${config.transportPort}",
+                'path.repo': "${home}/repo",
+                'path.shared_data': "${home}/../",
+                // Define a node attribute so we can test that it exists
+                'node.testattr': 'test',
+                'repositories.url.allowed_urls': 'http://snapshot.test*'
+            ]
+            configFile.setText(esConfig.collect { key, value -> "${key}: ${value}" }.join('\n'), 'UTF-8')
        }
        for (Map.Entry<String, String> command : config.setupCommands.entrySet()) {
            Task nextSetup = project.tasks.create(name: "${task.name}#${command.getKey()}", type: Exec, dependsOn: setup) {
@ -100,15 +112,7 @@ class ClusterFormationTasks {
            setup = nextSetup
        }

-        File pidFile = pidFile(baseDir)
-        List esArgs = [
-            "-Des.http.port=${config.httpPort}",
-            "-Des.transport.tcp.port=${config.transportPort}",
-            "-Des.pidfile=${pidFile}",
-            "-Des.path.repo=${home}/repo",
-            "-Des.path.shared_data=${home}/../",
-        ]
-        esArgs.addAll(config.systemProperties.collect {key, value -> "-D${key}=${value}"})
+        List esArgs = config.systemProperties.collect {key, value -> "-D${key}=${value}"}
        Closure esPostStartActions = { ant, logger ->
            ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${task.name}#start") {
                and {
@ -157,9 +161,44 @@ class ClusterFormationTasks {
        task.dependsOn(start)
    }

+    static Task configureExtractTask(Project project, String name, String distro, File baseDir, Task setup) {
+        List extractDependsOn = [project.configurations.elasticsearchDistro, setup]
+        Task extract
+        switch (distro) {
+            case 'zip':
+                extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
+                    from { project.zipTree(project.configurations.elasticsearchDistro.singleFile) }
+                    into baseDir
+                }
+                break;
+            case 'tar':
+                extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
+                    from { project.tarTree(project.resources.gzip(project.configurations.elasticsearchDistro.singleFile)) }
+                    into baseDir
+                }
+                break;
+            default:
+                throw new InvalidUserDataException("Unknown distribution: ${distro}")
+        }
+        return extract
+    }
+
+    static File homeDir(File baseDir, String distro) {
+        String path
+        switch (distro) {
+            case 'zip':
+            case 'tar':
+                path = "elasticsearch-${ElasticsearchProperties.version}"
+                break;
+            default:
+                throw new InvalidUserDataException("Unknown distribution: ${distro}")
+        }
+        return new File(baseDir, path)
+    }
+
    static void addNodeStopTask(Project project, Task task, File baseDir) {
        LazyPidReader pidFile = new LazyPidReader(pidFile: pidFile(baseDir))
-        Task stop = project.tasks.create(name: task.name + '#stop', type: Exec) {
+        Task stop = project.tasks.create(name: "${task.name}#stop", type: Exec) {
            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
                executable 'Taskkill'
                args '/PID', pidFile, '/F'
@ -187,13 +226,13 @@ class ClusterFormationTasks {
        return new File(dir, 'es.pid')
    }

-    static void addZipConfiguration(Project project) {
+    static void configureDistributionDependency(Project project, String distro) {
        String elasticsearchVersion = ElasticsearchProperties.version
        project.configurations {
-            elasticsearchZip
+            elasticsearchDistro
        }
        project.dependencies {
-            elasticsearchZip "org.elasticsearch.distribution.zip:elasticsearch:${elasticsearchVersion}@zip"
+            elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${distro}"
        }
    }
 }
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@ -36,11 +36,17 @@ buildscript {
 allprojects {
  project.ext {
    // this is common configuration for distributions, but we also add it here for the license check to use
-    deps = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api')
+    dependencyFiles = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api')
  }
 }

 subprojects {
+  apply plugin: 'elasticsearch.rest-test'
+
+  integTest {
+    includePackaged true
+  }
+
  /*****************************************************************************
   *                              Maven config                                 *
   *****************************************************************************/
@ -50,8 +56,8 @@ subprojects {
  // we must create our own install task, because it is only added when the java plugin is added
  task install(type: Upload, description: "Installs the 'archives' artifacts into the local Maven repository.", group: 'Upload') {
    configuration = configurations.archives
-    MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention);
-    repositoriesHandler.mavenInstaller();
+    MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention)
+    repositoriesHandler.mavenInstaller()
  }

  // TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run...
@ -81,7 +87,7 @@ subprojects {
    libFiles = copySpec {
      into 'lib'
      from project("${projectsPrefix}:core").jar
-      from deps
+      from dependencyFiles
    }

    configFiles = copySpec {
@ -151,7 +157,7 @@ buildDeb.dependsOn createEmptyDir
 /*****************************************************************************
 *                         Deb and rpm configuration                         *
 *****************************************************************************/
-configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) {
+configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) {
  apply plugin: 'nebula.ospackage-base'
  ospackage {
    packageName = 'elasticsearch'
@ -173,29 +179,17 @@ configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) {
    }
    directory('/etc/elasticsearch/scripts')
  }
-  if (project.name == 'deb') {
-    task buildDeb(type: Deb) {
-      dependsOn deps
-    }
-    artifacts {
-      archives buildDeb
-    }
-  } else if (project.name == 'rpm') {
-    task buildRpm(type: Rpm) {
-      dependsOn deps
-    }
-    artifacts {
-      archives buildRpm
-    }
-  }
+  
+  // TODO: re-enable tests when we have real rpm and deb distros!
+  integTest.enabled = false
 }

 // TODO: dependency checks should really be when building the jar itself, which would remove the need
 // for this hackery and instead we can do this inside the BuildPlugin
 task check(group: 'Verification', description: 'Runs all checks.') {} // dummy task!
 DependencyLicensesTask.configure(project) {
-  dependsOn = [deps]
-  dependencies = deps
+  dependsOn = [dependencyFiles]
+  dependencies = dependencyFiles
  mapping from: /lucene-.*/, to: 'lucene'
  mapping from: /jackson-.*/, to: 'jackson'
 }
--- a/distribution/deb/build.gradle
+++ b/distribution/deb/build.gradle
@ -1,4 +1,25 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */

-/*task buildDeb(type: Deb) {
-  dependsOn deps
-}*/
+task buildDeb(type: Deb) {
+  dependsOn dependencyFiles
+}
+artifacts {
+  archives buildDeb
+}
--- a/distribution/rpm/build.gradle
+++ b/distribution/rpm/build.gradle
@ -1,4 +1,25 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */

-/*task buildRpm(type: Rpm) {
-  dependsOn deps
-}*/
+task buildRpm(type: Rpm) {
+  dependsOn dependencyFiles
+}
+artifacts {
+  archives buildRpm
+}
--- a/distribution/tar/build.gradle
+++ b/distribution/tar/build.gradle
@ -1,5 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */

-task buildTar(type: Tar, dependsOn: deps) {
+task buildTar(type: Tar, dependsOn: dependencyFiles) {
  baseName = 'elasticsearch'
  with archivesFiles
  compression = Compression.GZIP
--- a/distribution/zip/build.gradle
+++ b/distribution/zip/build.gradle
@ -1,5 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */

-task buildZip(type: Zip, dependsOn: deps) {
+task buildZip(type: Zip, dependsOn: dependencyFiles) {
  baseName = 'elasticsearch'
  with archivesFiles
 }