From 45e71329ee33fe86a28c68c8be9fcb3b9b512aa9 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 25 Nov 2015 19:21:38 -0500 Subject: [PATCH 1/2] [build] Fix deb and rpm tests Gradle is causing so much trouble here! Its too cute for its own good. --- build.gradle | 17 ++++--- .../gradle/test/ClusterFormationTasks.groovy | 47 ++++++++++++++++--- .../elasticsearch/gradle/test/NodeInfo.groovy | 25 +++++++++- distribution/build.gradle | 4 +- distribution/deb/build.gradle | 10 +++- distribution/rpm/build.gradle | 13 ++++- 6 files changed, 97 insertions(+), 19 deletions(-) diff --git a/build.gradle b/build.gradle index 785db7ec0c4..1326c8225e1 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ subprojects { } } } - } + } extraArchive { javadoc = true tests = false @@ -86,8 +86,8 @@ subprojects { tasks.withType(Jar) { into('META-INF') { from project.rootProject.rootDir - include 'LICENSE.txt' - include 'NOTICE.txt' + include 'LICENSE.txt' + include 'NOTICE.txt' } } // ignore missing javadocs @@ -101,12 +101,18 @@ subprojects { } } + /* Sets up the dependencies that we build as part of this project but + register as thought they were external to resolve internally. We register + them as external dependencies so the build plugin that we use can be used + to build elasticsearch plugins outside of the elasticsearch source tree. */ ext.projectSubstitutions = [ "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec', "org.elasticsearch:elasticsearch:${version}": ':core', "org.elasticsearch:test-framework:${version}": ':test-framework', "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip', - "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar' + "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar', + "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm', + "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:deb', ] configurations.all { resolutionStrategy.dependencySubstitution { DependencySubstitutions subs -> @@ -226,7 +232,7 @@ class Run extends DefaultTask { ) public void setDebug(boolean enabled) { project.project(':distribution').run.clusterConfig.debug = enabled - } + } } task run(type: Run) { dependsOn ':distribution:run' @@ -234,4 +240,3 @@ task run(type: Run) { group = 'Verification' impliesSubProjects = true } - diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index a3c06e0444c..dfb91c7b5dd 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -27,9 +27,7 @@ import org.gradle.api.* import org.gradle.api.artifacts.Configuration import org.gradle.api.file.FileCollection import org.gradle.api.logging.Logger -import org.gradle.api.tasks.Copy -import org.gradle.api.tasks.Delete -import org.gradle.api.tasks.Exec +import org.gradle.api.tasks.* import java.nio.file.Paths @@ -62,12 +60,11 @@ class ClusterFormationTasks { /** Adds a dependency on the given distribution */ static void configureDistributionDependency(Project project, String distro) { String elasticsearchVersion = VersionProperties.elasticsearch - String packaging = distro == 'tar' ? 'tar.gz' : distro project.configurations { elasticsearchDistro } project.dependencies { - elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${packaging}" + elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}" } } @@ -132,6 +129,12 @@ class ClusterFormationTasks { /** Adds a task to extract the elasticsearch distribution */ static Task configureExtractTask(String name, Project project, Task setup, NodeInfo node) { List extractDependsOn = [project.configurations.elasticsearchDistro, setup] + /* project.configurations.elasticsearchDistro.singleFile will be an + external artifact if this is being run by a plugin not living in the + elasticsearch source tree. If this is a plugin built in the + elasticsearch source tree or this is a distro in the elasticsearch + source tree then this should be the version of elasticsearch built + by the source tree. If it isn't then Bad Things(TM) will happen. */ Task extract switch (node.config.distribution) { case 'zip': @@ -148,6 +151,38 @@ class ClusterFormationTasks { into node.baseDir } break; + case 'rpm': + File rpmDatabase = new File(node.baseDir, 'rpm-database') + File rpmExtracted = new File(node.baseDir, 'rpm-extracted') + /* Delay reading the location of the rpm file until task execution */ + Object rpm = "${ -> project.configurations.elasticsearchDistro.singleFile}" + extract = project.tasks.create(name: name, type: Exec, dependsOn: extractDependsOn) { + commandLine 'rpm', '--badreloc', '--nodeps', '--noscripts', '--notriggers', + '--dbpath', rpmDatabase, + '--relocate', "/=${rpmExtracted}", + '-i', rpm + standardOutput = new ByteArrayOutputStream() + errorOutput = standardOutput + /* rpm complains about the database being corrupted and exits. But it gets far + enough for us to use it as an extractor. This is kind of funky but it works + and its how Elasticsearch's maven build used to do it. */ + ignoreExitValue true + doLast { + String out = standardOutput.toString() + if (out.indexOf('DB_RUNRECOVERY') < 0) { + throw new GradleException("Didn't detect the usual error message when exracting the rpm. Something went wrong? Output:\n${out}") + } + } + } + break; + case 'deb': + /* Delay reading the location of the deb file until task execution */ + File debExtracted = new File(node.baseDir, 'deb-extracted') + Object deb = "${ -> project.configurations.elasticsearchDistro.singleFile}" + extract = project.tasks.create(name: name, type: Exec, dependsOn: extractDependsOn) { + commandLine 'dpkg-deb', '-x', deb, debExtracted + } + break; default: throw new InvalidUserDataException("Unknown distribution: ${node.config.distribution}") } @@ -172,7 +207,7 @@ class ClusterFormationTasks { Task writeConfig = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup) writeConfig.doFirst { - File configFile = new File(node.homeDir, 'config/elasticsearch.yml') + File configFile = new File(node.confDir, 'elasticsearch.yml') logger.info("Configuring ${configFile}") configFile.setText(esConfig.collect { key, value -> "${key}: ${value}" }.join('\n'), 'UTF-8') } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy index 3955b9e0269..c3b6abf6aea 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy @@ -45,6 +45,9 @@ class NodeInfo { /** elasticsearch home dir */ File homeDir + /** config directory */ + File confDir + /** working directory for the node process */ File cwd @@ -77,6 +80,7 @@ class NodeInfo { baseDir = new File(project.buildDir, "cluster/${task.name} node${nodeNum}") pidFile = new File(baseDir, 'es.pid') homeDir = homeDir(baseDir, config.distribution) + confDir = confDir(baseDir, config.distribution) cwd = new File(baseDir, "cwd") failedMarker = new File(cwd, 'run.failed') startLog = new File(cwd, 'run.log') @@ -92,6 +96,7 @@ class NodeInfo { args.add("-D${property.getKey()}=${property.getValue()}") } } + args.add("-Des.default.path.conf=${confDir}") // running with cmd on windows will look for this with the .bat extension esScript = new File(homeDir, 'bin/elasticsearch').toString() } @@ -122,10 +127,28 @@ class NodeInfo { case 'zip': case 'tar': path = "elasticsearch-${VersionProperties.elasticsearch}" - break; + break + case 'rpm': + case 'deb': + path = "${distro}-extracted/usr/share/elasticsearch" + break default: throw new InvalidUserDataException("Unknown distribution: ${distro}") } return new File(baseDir, path) } + + static File confDir(File baseDir, String distro) { + String Path + switch (distro) { + case 'zip': + case 'tar': + return new File(homeDir(baseDir, distro), 'config') + case 'rpm': + case 'deb': + return new File(baseDir, "${distro}-extracted/etc/elasticsearch") + default: + throw new InvalidUserDataException("Unkown distribution: ${distro}") + } + } } diff --git a/distribution/build.gradle b/distribution/build.gradle index 9573fa4afba..6ceb940e4a6 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -143,9 +143,7 @@ configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) { * MavenFilteringHack or any other copy-style action. */ configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) { - // Currently disabled these because they are broken. - // integTest.enabled = Os.isFamily(Os.FAMILY_WINDOWS) == false - integTest.enabled = false + integTest.enabled = Os.isFamily(Os.FAMILY_WINDOWS) == false File packagingFiles = new File(buildDir, 'packaging') project.ext.packagingFiles = packagingFiles task processPackagingFiles(type: Copy) { diff --git a/distribution/deb/build.gradle b/distribution/deb/build.gradle index afe55e0cc02..72f6216d7b9 100644 --- a/distribution/deb/build.gradle +++ b/distribution/deb/build.gradle @@ -35,7 +35,15 @@ task buildDeb(type: Deb) { } artifacts { + 'default' buildDeb archives buildDeb } -integTest.dependsOn buildDeb +integTest { + /* We use real deb tools to extract the deb file for testing so we have to + skip the test if they aren't around. */ + enabled = new File('/usr/bin/dpkg-deb').exists() || // Standard location + new File('/usr/local/bin/dpkg-deb').exists() // Homebrew location + dependsOn buildDeb + clusterConfig.distribution = 'deb' +} diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle index a665316a3e6..0af164bdb20 100644 --- a/distribution/rpm/build.gradle +++ b/distribution/rpm/build.gradle @@ -21,7 +21,7 @@ task buildRpm(type: Rpm) { dependsOn dependencyFiles, preparePackagingFiles baseName 'elasticsearch' // this is what pom generation uses for artifactId // Follow elasticsearch's rpm file naming convention - archiveName = "${packageName}-${project.version}.rpm" + archiveName "${packageName}-${project.version}.rpm" packageGroup 'Application/Internet' prefix '/usr' packager 'Elasticsearch' @@ -32,7 +32,16 @@ task buildRpm(type: Rpm) { } artifacts { + 'default' buildRpm archives buildRpm } -integTest.dependsOn buildRpm +integTest { + /* We use real rpm tools to extract the rpm file for testing so we have to + skip the test if they aren't around. */ + enabled = new File('/bin/rpm').exists() || // Standard location + new File('/usr/bin/rpm').exists() || // Debian location + new File('/usr/local/bin/rpm').exists() // Homebrew location + dependsOn buildRpm + clusterConfig.distribution = 'rpm' +} From eba594fc0820189740aaebc666f380e4e915c618 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Sat, 28 Nov 2015 19:44:24 -0500 Subject: [PATCH 2/2] Remove strange checks and clean dirs --- .../gradle/test/ClusterFormationTasks.groovy | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index dfb91c7b5dd..4484e32bdd2 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -60,11 +60,12 @@ class ClusterFormationTasks { /** Adds a dependency on the given distribution */ static void configureDistributionDependency(Project project, String distro) { String elasticsearchVersion = VersionProperties.elasticsearch + String packaging = distro == 'tar' ? 'tar.gz' : distro project.configurations { elasticsearchDistro } project.dependencies { - elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}" + elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${packaging}" } } @@ -156,22 +157,14 @@ class ClusterFormationTasks { File rpmExtracted = new File(node.baseDir, 'rpm-extracted') /* Delay reading the location of the rpm file until task execution */ Object rpm = "${ -> project.configurations.elasticsearchDistro.singleFile}" - extract = project.tasks.create(name: name, type: Exec, dependsOn: extractDependsOn) { + extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) { commandLine 'rpm', '--badreloc', '--nodeps', '--noscripts', '--notriggers', '--dbpath', rpmDatabase, '--relocate', "/=${rpmExtracted}", '-i', rpm - standardOutput = new ByteArrayOutputStream() - errorOutput = standardOutput - /* rpm complains about the database being corrupted and exits. But it gets far - enough for us to use it as an extractor. This is kind of funky but it works - and its how Elasticsearch's maven build used to do it. */ - ignoreExitValue true - doLast { - String out = standardOutput.toString() - if (out.indexOf('DB_RUNRECOVERY') < 0) { - throw new GradleException("Didn't detect the usual error message when exracting the rpm. Something went wrong? Output:\n${out}") - } + doFirst { + rpmDatabase.deleteDir() + rpmExtracted.deleteDir() } } break; @@ -179,8 +172,11 @@ class ClusterFormationTasks { /* Delay reading the location of the deb file until task execution */ File debExtracted = new File(node.baseDir, 'deb-extracted') Object deb = "${ -> project.configurations.elasticsearchDistro.singleFile}" - extract = project.tasks.create(name: name, type: Exec, dependsOn: extractDependsOn) { + extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) { commandLine 'dpkg-deb', '-x', deb, debExtracted + doFirst { + debExtracted.deleteDir() + } } break; default: