Merge pull request #15031 from nik9000/integ_for_packages

Fix deb and rpm tests

build.gradle

@@ -45,7 +45,7 @@ subprojects {
           }
         }
       }
-    }  
+    }
     extraArchive {
       javadoc = true
       tests = false
@@ -86,8 +86,8 @@ subprojects {
     tasks.withType(Jar) {
       into('META-INF') {
         from project.rootProject.rootDir
-        include 'LICENSE.txt'   
+        include 'LICENSE.txt'
-        include 'NOTICE.txt'   
+        include 'NOTICE.txt'
       }
     }
     // ignore missing javadocs
@@ -101,12 +101,18 @@ subprojects {
     }
   }
 
+  /* Sets up the dependencies that we build as part of this project but
+    register as thought they were external to resolve internally. We register
+    them as external dependencies so the build plugin that we use can be used
+    to build elasticsearch plugins outside of the elasticsearch source tree. */
   ext.projectSubstitutions = [
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':core',
     "org.elasticsearch:test-framework:${version}": ':test-framework',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
-    "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar'
+    "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar',
+    "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm',
+    "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:deb',
   ]
   configurations.all {
     resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
@@ -226,7 +232,7 @@ class Run extends DefaultTask {
   )
   public void setDebug(boolean enabled) {
     project.project(':distribution').run.clusterConfig.debug = enabled
-  } 
+  }
 }
 task run(type: Run) {
   dependsOn ':distribution:run'
@@ -234,4 +240,3 @@ task run(type: Run) {
   group = 'Verification'
   impliesSubProjects = true
 }
-

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -27,9 +27,7 @@ import org.gradle.api.*
 import org.gradle.api.artifacts.Configuration
 import org.gradle.api.file.FileCollection
 import org.gradle.api.logging.Logger
-import org.gradle.api.tasks.Copy
+import org.gradle.api.tasks.*
-import org.gradle.api.tasks.Delete
-import org.gradle.api.tasks.Exec
 
 import java.nio.file.Paths
 
@@ -132,6 +130,12 @@ class ClusterFormationTasks {
     /** Adds a task to extract the elasticsearch distribution */
     static Task configureExtractTask(String name, Project project, Task setup, NodeInfo node) {
         List extractDependsOn = [project.configurations.elasticsearchDistro, setup]
+        /* project.configurations.elasticsearchDistro.singleFile will be an
+          external artifact if this is being run by a plugin not living in the
+          elasticsearch source tree. If this is a plugin built in the
+          elasticsearch source tree or this is a distro in the elasticsearch
+          source tree then this should be the version of elasticsearch built
+          by the source tree. If it isn't then Bad Things(TM) will happen. */
         Task extract
         switch (node.config.distribution) {
             case 'zip':
@@ -148,6 +152,33 @@ class ClusterFormationTasks {
                     into node.baseDir
                 }
                 break;
+            case 'rpm':
+                File rpmDatabase = new File(node.baseDir, 'rpm-database')
+                File rpmExtracted = new File(node.baseDir, 'rpm-extracted')
+                /* Delay reading the location of the rpm file until task execution */
+                Object rpm = "${ -> project.configurations.elasticsearchDistro.singleFile}"
+                extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) {
+                    commandLine 'rpm', '--badreloc', '--nodeps', '--noscripts', '--notriggers',
+                        '--dbpath', rpmDatabase,
+                        '--relocate', "/=${rpmExtracted}",
+                        '-i', rpm
+                    doFirst {
+                        rpmDatabase.deleteDir()
+                        rpmExtracted.deleteDir()
+                    }
+                }
+                break;
+            case 'deb':
+                /* Delay reading the location of the deb file until task execution */
+                File debExtracted = new File(node.baseDir, 'deb-extracted')
+                Object deb = "${ -> project.configurations.elasticsearchDistro.singleFile}"
+                extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) {
+                    commandLine 'dpkg-deb', '-x', deb, debExtracted
+                    doFirst {
+                        debExtracted.deleteDir()
+                    }
+                }
+                break;
             default:
                 throw new InvalidUserDataException("Unknown distribution: ${node.config.distribution}")
         }
@@ -172,7 +203,7 @@ class ClusterFormationTasks {
 
         Task writeConfig = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup)
         writeConfig.doFirst {
-            File configFile = new File(node.homeDir, 'config/elasticsearch.yml')
+            File configFile = new File(node.confDir, 'elasticsearch.yml')
             logger.info("Configuring ${configFile}")
             configFile.setText(esConfig.collect { key, value -> "${key}: ${value}" }.join('\n'), 'UTF-8')
         }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -45,6 +45,9 @@ class NodeInfo {
     /** elasticsearch home dir */
     File homeDir
 
+    /** config directory */
+    File confDir
+
     /** working directory for the node process */
     File cwd
 
@@ -77,6 +80,7 @@ class NodeInfo {
         baseDir = new File(project.buildDir, "cluster/${task.name} node${nodeNum}")
         pidFile = new File(baseDir, 'es.pid')
         homeDir = homeDir(baseDir, config.distribution)
+        confDir = confDir(baseDir, config.distribution)
         cwd = new File(baseDir, "cwd")
         failedMarker = new File(cwd, 'run.failed')
         startLog = new File(cwd, 'run.log')
@@ -92,6 +96,7 @@ class NodeInfo {
                 args.add("-D${property.getKey()}=${property.getValue()}")
             }
         }
+        args.add("-Des.default.path.conf=${confDir}")
         // running with cmd on windows will look for this with the .bat extension
         esScript = new File(homeDir, 'bin/elasticsearch').toString()
     }
@@ -122,10 +127,28 @@ class NodeInfo {
             case 'zip':
             case 'tar':
                 path = "elasticsearch-${VersionProperties.elasticsearch}"
-                break;
+                break
+            case 'rpm':
+            case 'deb':
+                path = "${distro}-extracted/usr/share/elasticsearch"
+                break
             default:
                 throw new InvalidUserDataException("Unknown distribution: ${distro}")
         }
         return new File(baseDir, path)
     }
+
+    static File confDir(File baseDir, String distro) {
+        String Path
+        switch (distro) {
+            case 'zip':
+            case 'tar':
+                return new File(homeDir(baseDir, distro), 'config')
+            case 'rpm':
+            case 'deb':
+                return new File(baseDir, "${distro}-extracted/etc/elasticsearch")
+            default:
+                throw new InvalidUserDataException("Unkown distribution: ${distro}")
+        }
+    }
 }

distribution/build.gradle

@@ -143,9 +143,7 @@ configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) {
  *    MavenFilteringHack or any other copy-style action.
  */
 configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) {
-  // Currently disabled these because they are broken.
+  integTest.enabled = Os.isFamily(Os.FAMILY_WINDOWS) == false
-  // integTest.enabled = Os.isFamily(Os.FAMILY_WINDOWS) == false
-  integTest.enabled = false
   File packagingFiles = new File(buildDir, 'packaging')
   project.ext.packagingFiles = packagingFiles
   task processPackagingFiles(type: Copy) {

distribution/deb/build.gradle

@@ -35,7 +35,15 @@ task buildDeb(type: Deb) {
 }
 
 artifacts {
+  'default' buildDeb
   archives buildDeb
 }
 
-integTest.dependsOn buildDeb
+integTest {
+  /* We use real deb tools to extract the deb file for testing so we have to
+    skip the test if they aren't around. */
+  enabled = new File('/usr/bin/dpkg-deb').exists() || // Standard location
+      new File('/usr/local/bin/dpkg-deb').exists()    // Homebrew location
+  dependsOn buildDeb
+  clusterConfig.distribution = 'deb'
+}

distribution/rpm/build.gradle

@@ -21,7 +21,7 @@ task buildRpm(type: Rpm) {
   dependsOn dependencyFiles, preparePackagingFiles
   baseName 'elasticsearch' // this is what pom generation uses for artifactId
   // Follow elasticsearch's rpm file naming convention
-  archiveName = "${packageName}-${project.version}.rpm"
+  archiveName "${packageName}-${project.version}.rpm"
   packageGroup 'Application/Internet'
   prefix '/usr'
   packager 'Elasticsearch'
@@ -32,7 +32,16 @@ task buildRpm(type: Rpm) {
 }
 
 artifacts {
+  'default' buildRpm
   archives buildRpm
 }
 
-integTest.dependsOn buildRpm
+integTest {
+  /* We use real rpm tools to extract the rpm file for testing so we have to
+    skip the test if they aren't around. */
+  enabled = new File('/bin/rpm').exists() ||   // Standard location
+      new File('/usr/bin/rpm').exists() ||     // Debian location
+      new File('/usr/local/bin/rpm').exists()  // Homebrew location
+  dependsOn buildRpm
+  clusterConfig.distribution = 'rpm'
+}