From 9b0a47d8e315a7bcd41606977278ead9a485581d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 5 Nov 2015 15:47:54 -0500 Subject: [PATCH] Fix rpm and deb distributions and test them with vagrant --- TESTING.asciidoc | 2 +- Vagrantfile | 8 +- .../elasticsearch/gradle/EmptyDirTask.groovy | 48 +++ .../gradle/FileContentsTask.groovy | 50 +++ buildSrc/src/main/resources/deb/README | 4 + buildSrc/src/main/resources/deb/conffiles.ftl | 3 + buildSrc/src/main/resources/deb/postinst.ftl | 2 + buildSrc/src/main/resources/deb/preinst.ftl | 2 + distribution/build.gradle | 291 ++++++++++++++---- distribution/deb/build.gradle | 14 +- .../src/main/packaging/init.d/elasticsearch | 2 +- .../src/main/packaging/packaging.properties | 15 - .../deb/src/main/packaging/scripts/conffiles | 5 - .../deb/src/main/packaging/scripts/control | 9 - distribution/rpm/build.gradle | 11 +- .../src/main/packaging/init.d/elasticsearch | 26 +- .../src/main/packaging/packaging.properties | 18 -- .../src/main/packaging/env/elasticsearch | 34 +- .../src/main/packaging/packaging.properties | 30 -- .../src/main/packaging/scripts/postinst | 10 +- .../src/main/packaging/scripts/postrm | 18 +- .../src/main/packaging/scripts/preinst | 10 +- distribution/src/main/packaging/scripts/prerm | 4 +- .../main/packaging/systemd/elasticsearch.conf | 2 +- .../packaging/systemd/elasticsearch.service | 28 +- .../systemd/sysctl/elasticsearch.conf | 2 +- .../src/main/resources/bin/elasticsearch | 9 +- .../main/resources/bin/elasticsearch.in.bat | 8 +- .../main/resources/bin/elasticsearch.in.sh | 6 +- distribution/src/main/resources/bin/plugin | 6 +- .../src/main/resources/bin/service.bat | 4 +- .../tar/src/main/assemblies/targz-bin.xml | 13 - .../zip/src/main/assemblies/zip-bin.xml | 13 - qa/vagrant/build.gradle | 99 ++++-- .../packaging/scripts/plugin_test_cases.bash | 9 +- 35 files changed, 526 insertions(+), 289 deletions(-) create mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/EmptyDirTask.groovy create mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/FileContentsTask.groovy create mode 100644 buildSrc/src/main/resources/deb/README create mode 100644 buildSrc/src/main/resources/deb/conffiles.ftl create mode 100644 buildSrc/src/main/resources/deb/postinst.ftl create mode 100644 buildSrc/src/main/resources/deb/preinst.ftl delete mode 100644 distribution/deb/src/main/packaging/packaging.properties delete mode 100644 distribution/deb/src/main/packaging/scripts/conffiles delete mode 100644 distribution/deb/src/main/packaging/scripts/control delete mode 100644 distribution/rpm/src/main/packaging/packaging.properties delete mode 100644 distribution/src/main/packaging/packaging.properties delete mode 100644 distribution/tar/src/main/assemblies/targz-bin.xml delete mode 100644 distribution/zip/src/main/assemblies/zip-bin.xml diff --git a/TESTING.asciidoc b/TESTING.asciidoc index c1b17e7b31a..ac1f89e439a 100644 --- a/TESTING.asciidoc +++ b/TESTING.asciidoc @@ -424,7 +424,7 @@ sudo bats $BATS/*rpm*.bats If you wanted to retest all the release artifacts on a single VM you could: ------------------------------------------------- -gradle copyDepsToTestRoot +gradle prepareTestRoot vagrant up trusty --provider virtualbox && vagrant ssh trusty cd $TESTROOT sudo bats $BATS/*.bats diff --git a/Vagrantfile b/Vagrantfile index 32b48f5c610..454d114f1a2 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -251,10 +251,10 @@ def provision(config, rm -rf /tmp/bats } cat \<\ /etc/profile.d/elasticsearch_vars.sh -export ZIP=/elasticsearch/distribution/zip/build/releases -export TAR=/elasticsearch/distribution/tar/build/releases -export RPM=/elasticsearch/distribution/rpm/build/releases -export DEB=/elasticsearch/distribution/deb/build/releases +export ZIP=/elasticsearch/distribution/zip/build/distributions +export TAR=/elasticsearch/distribution/tar/build/distributions +export RPM=/elasticsearch/distribution/rpm/build/distributions +export DEB=/elasticsearch/distribution/deb/build/distributions export TESTROOT=/elasticsearch/qa/vagrant/build/testroot export BATS=/elasticsearch/qa/vagrant/src/test/resources/packaging/scripts VARS diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/EmptyDirTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/EmptyDirTask.groovy new file mode 100644 index 00000000000..d2059bc4719 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/EmptyDirTask.groovy @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.* +import org.gradle.internal.nativeintegration.filesystem.Chmod +import java.io.File +import javax.inject.Inject + +/** + * Creates an empty directory. + */ +class EmptyDirTask extends DefaultTask { + @Input + Object dir + + @Input + int dirMode = 0755 + + @TaskAction + void create() { + dir = dir as File + dir.mkdirs() + getChmod().chmod(dir, dirMode) + } + + @Inject + Chmod getChmod() { + throw new UnsupportedOperationException() + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/FileContentsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/FileContentsTask.groovy new file mode 100644 index 00000000000..4df6d1b32df --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/FileContentsTask.groovy @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.* +import java.io.File + +/** + * Creates a file and sets it contents to something. + */ +class FileContentsTask extends DefaultTask { + /** + * The file to be built. Must be of type File to make @OutputFile happy. + */ + @OutputFile + File file + + @Input + Object contents + + /** + * The file to be built. Takes any objecct and coerces to a file. + */ + void setFile(Object file) { + this.file = file as File + } + + @TaskAction + void setContents() { + file = file as File + file.text = contents.toString() + } +} diff --git a/buildSrc/src/main/resources/deb/README b/buildSrc/src/main/resources/deb/README new file mode 100644 index 00000000000..cdecf00756d --- /dev/null +++ b/buildSrc/src/main/resources/deb/README @@ -0,0 +1,4 @@ +This directory contains templates that work around gradle-ospackage-plugin +trying to be helpful and adding templates for your os packaging scripts. We +have relatively nice scripts already so we just override the templates to be +mostly noops. diff --git a/buildSrc/src/main/resources/deb/conffiles.ftl b/buildSrc/src/main/resources/deb/conffiles.ftl new file mode 100644 index 00000000000..db7788a4e44 --- /dev/null +++ b/buildSrc/src/main/resources/deb/conffiles.ftl @@ -0,0 +1,3 @@ +<% files.each {file -> %><%= file +%> +<% } %> diff --git a/buildSrc/src/main/resources/deb/postinst.ftl b/buildSrc/src/main/resources/deb/postinst.ftl new file mode 100644 index 00000000000..5f67242c265 --- /dev/null +++ b/buildSrc/src/main/resources/deb/postinst.ftl @@ -0,0 +1,2 @@ +#!/bin/sh -e +<% commands.each {command -> %><%= command %><% } %> diff --git a/buildSrc/src/main/resources/deb/preinst.ftl b/buildSrc/src/main/resources/deb/preinst.ftl new file mode 100644 index 00000000000..5f67242c265 --- /dev/null +++ b/buildSrc/src/main/resources/deb/preinst.ftl @@ -0,0 +1,2 @@ +#!/bin/sh -e +<% commands.each {command -> %><%= command %><% } %> diff --git a/distribution/build.gradle b/distribution/build.gradle index d6466b6c16c..51305dd0a0b 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -20,7 +20,10 @@ import org.apache.tools.ant.filters.FixCrLfFilter import org.elasticsearch.gradle.precommit.DependencyLicensesTask import org.elasticsearch.gradle.test.RunTask +import org.elasticsearch.gradle.EmptyDirTask import org.elasticsearch.gradle.MavenFilteringHack +import org.gradle.api.InvalidUserDataException +import org.gradle.internal.nativeintegration.filesystem.Chmod // for deb/rpm buildscript { @@ -68,21 +71,7 @@ subprojects { * Properties to expand when copying packaging files * *****************************************************************************/ project.ext { - expansions = [ - 'project.version': version, - 'project.parent.artifactId': 'distributions', - // Default values for min/max heap memory allocated to elasticsearch java process - 'packaging.elasticsearch.heap.min': '256m', - 'packaging.elasticsearch.heap.max': '1g', - 'project.build.finalName': "elasticsearch-${version}", - // Default configuration directory and file to use in bin/plugin script - 'packaging.plugin.default.config.dir': '$ES_HOME/config', - 'packaging.plugin.default.config.file': '$ES_HOME/config/elasticsearch.yml', - 'packaging.env.file': '', - // TODO: do we really need this marker? the tgz and zip are exactly the same, - // we should not need to specify twice just to change this - 'packaging.type': 'tar.gz', - ] + expansions = expansionsForDistribution(project.name) /***************************************************************************** * Common files in all distributions * @@ -104,6 +93,7 @@ subprojects { exclude 'bin/*.exe' exclude 'config/**' filesMatching('bin/*') { it.setMode(0755) } + MavenFilteringHack.filter(it, expansions) } } } @@ -121,70 +111,182 @@ configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) { with copySpec { with commonFiles from('../src/main/resources') { - include 'bin/*.bat' + include 'bin/*.bat' filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf')) } MavenFilteringHack.filter(it, expansions) } from('../src/main/resources') { - include 'bin/*.exe' - } + include 'bin/*.exe' + } } } } /***************************************************************************** * Deb and rpm configuration * - *****************************************************************************/ -// ospackage supports adding empty dirs with directory() to rpm, but not deb...yet -// https://github.com/nebula-plugins/gradle-ospackage-plugin/issues/115 -// however, even adding just for rpm doesn't seem to work... -// gradle may also get native support https://issues.gradle.org/browse/GRADLE-1671 -// in the meantime, we hack this by copying an empty dir -// TODO: HACK DOES NOT WORK -/*ext.emptyDir = new File(project.buildDir, 'empty') -Closure emptyDirSpec() { - return { - from emptyDir - addParentDirs false - createDirectoryEntry true - } -} -task createEmptyDir << { - emptyDir.mkdirs() -} -buildRpm.dependsOn createEmptyDir -buildDeb.dependsOn createEmptyDir -*/ - -/***************************************************************************** - * Deb and rpm configuration * - *****************************************************************************/ + ***************************************************************************** + * + * The general strategy here is to build a directory on disk, packagingFiles + * that contains stuff that needs to be copied into the distributions. This is + * important for two reasons: + * 1. ospackage wants to copy the directory permissions that it sees off of the + * filesystem. If you ask it to create a directory that doesn't already + * exist on disk it petulantly creates it with 0755 permissions, no matter + * how hard you try to convince it otherwise. + * 2. Convincing ospackage to pick up an empty directory as part of a set of + * directories on disk is reasonably easy. Convincing it to just create an + * empty directory requires more wits than I have. + * 3. ospackage really wants to suck up some of the debian control scripts + * directly from the filesystem. It doesn't want to process them through + * MavenFilteringHack or any other copy-style action. + */ configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) { + File packagingFiles = new File(buildDir, 'packaging') + project.ext.packagingFiles = packagingFiles + task processPackagingFiles(type: Copy) { + from '../src/main/packaging' + from 'src/main/packaging' + + MavenFilteringHack.filter(it, expansions) + into packagingFiles + /* Explicitly declare the outputs so that gradle won't skip this task if + one of the other tasks like createEtc run first and create the packaging + directory as a side effect. */ + outputs.dir("${packagingFiles}/scripts") + outputs.dir("${packagingFiles}/env") + outputs.dir("${packagingFiles}/systemd") + } + + task createEtc(type: EmptyDirTask) { + dir "${packagingFiles}/etc/elasticsearch" + dirMode 0750 + } + + task createEtcScripts(type: EmptyDirTask) { + dependsOn createEtc + dir "${packagingFiles}/etc/elasticsearch/scripts" + dirMode 0750 + } + + task fillEtc(type: Copy) { + dependsOn createEtc, createEtcScripts + with configFiles + into "${packagingFiles}/etc/elasticsearch" + /* Explicitly declare the output files so this task doesn't consider itself + up to date when the directory is created, which it would by default. And + that'll happen when createEtc runs. */ + outputs.file "${packagingFiles}/etc/elasticsearch/elasticsearch.yml" + outputs.file "${packagingFiles}/etc/elasticsearch/logging.yml" + } + + task createPidDir(type: EmptyDirTask) { + dir "${packagingFiles}/var/run/elasticsearch" + } + task createLogDir(type: EmptyDirTask) { + dir "${packagingFiles}/var/log/elasticsearch" + } + task createDataDir(type: EmptyDirTask) { + dir "${packagingFiles}/var/lib/elasticsearch" + } + task createPluginsDir(type: EmptyDirTask) { + dir "${packagingFiles}/usr/share/elasticsearch/plugins" + } + + /** + * Setup the build/packaging directory to be like the target filesystem + * because ospackage will use arbitrary permissions if you try to create a + * directory that doesn't exist on the filesystem. + */ + task preparePackagingFiles { + dependsOn processPackagingFiles, fillEtc, createPidDir, createLogDir, + createDataDir, createPluginsDir + } + apply plugin: 'nebula.ospackage-base' ospackage { - packageName = 'elasticsearch' - // TODO: '-' is an illegal character in rpm version...redline croaks - version = '3.0.0' + packageName 'elasticsearch' + maintainer 'Elasticsearch Team ' + summary ''' + Elasticsearch is a distributed RESTful search engine built for the cloud. + Reference documentation can be found at + https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html + and the 'Elasticsearch: The Definitive Guide' book can be found at + https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html + '''.stripIndent().replace('\n', ' ').trim() + url 'https://www.elastic.co/' + + /* The version of the package can't contain -SNAPSHOT so we rip it off if + we see it. We'll add it back on to the file name though. */ + version project.version.replace('-SNAPSHOT', '') + + String scripts = "${packagingFiles}/scripts" + preInstall file("${scripts}/preinst") + postInstall file("${scripts}/postinst") + preUninstall file("${scripts}/prerm") + postUninstall file("${scripts}/postrm") + into '/usr/share/elasticsearch' user 'root' permissionGroup 'root' with libFiles with copySpec { with commonFiles - // TODO: omit LICENSE.txt file on deb?? + if (project.name == 'deb') { + // Deb gets a copyright file instead. + exclude 'LICENSE.txt' + } } - into('/etc/elasticsearch') { - with configFiles - //into('scripts', emptyDirSpec()) - createDirectoryEntry = true - includeEmptyDirs = true + + configurationFile '/etc/elasticsearch/elasticsearch.yml' + configurationFile '/etc/elasticsearch/logging.yml' + into('/etc') { + from "${packagingFiles}/etc" + fileMode 0750 + permissionGroup 'elasticsearch' + includeEmptyDirs true + createDirectoryEntry true } - directory('/etc/elasticsearch/scripts') + + into('/usr/lib/tmpfiles.d') { + from "${packagingFiles}/systemd/elasticsearch.conf" + } + configurationFile '/usr/lib/systemd/system/elasticsearch.service' + into('/usr/lib/systemd/system') { + from "${packagingFiles}/systemd/elasticsearch.service" + } + into('/usr/lib/sysctl.d') { + from "${packagingFiles}/systemd/sysctl/elasticsearch.conf" + } + configurationFile '/etc/init.d/elasticsearch' + into('/etc/init.d') { + from "${packagingFiles}/init.d/elasticsearch" + fileMode 0755 + } + configurationFile project.expansions['path.env'] + into(new File(project.expansions['path.env']).getParent()) { + from "${project.packagingFiles}/env/elasticsearch" + } + + /** + * Suck up all the empty directories that we need to install into the path. + */ + Closure suckUpEmptyDirectories = { path -> + into(path) { + from "${packagingFiles}/${path}" + includeEmptyDirs true + createDirectoryEntry true + /* All of these empty directories have this ownership. We're just + lucky! */ + user 'elasticsearch' + permissionGroup 'elasticsearch' + } + } + suckUpEmptyDirectories('/var/run') + suckUpEmptyDirectories('/var/log') + suckUpEmptyDirectories('/var/lib') + suckUpEmptyDirectories('/usr/share/elasticsearch') } - - // TODO: re-enable tests when we have real rpm and deb distros! - integTest.enabled = false } // TODO: dependency checks should really be when building the jar itself, which would remove the need @@ -199,3 +301,80 @@ DependencyLicensesTask.configure(project) { RunTask.configure(project) +/** + * Build some variables that are replaced in the packages. This includes both + * scripts like bin/elasticsearch and bin/plugin that a user might run and also + * scripts like postinst which are run as part of the installation. + * + *
+ *
package.name
+ *
The name of the project. Its sprinkled throughout the scripts.
+ *
package.version
+ *
The version of the project. Its mostly used to find the exact jar name. + * + *
path.conf
+ *
The default directory from which to load configuration. This is used in + * the packaging scripts, but in that context it is always + * /etc/elasticsearch. Its also used in bin/plugin, where it is + * /etc/elasticsearch for the os packages but $ESHOME/config otherwise.
+ *
path.env
+ *
The env file sourced before bin/elasticsearch to set environment + * variables. Think /etc/defaults/elasticsearch.
+ *
heap.min and heap.max
+ *
Default min and max heap
+ *
scripts.footer
+ *
Footer appended to control scripts embedded in the distribution that is + * (almost) entirely there for cosmetic reasons.
+ *
stopping.timeout
+ *
RPM's init script needs to wait for elasticsearch to stop before + * returning from stop and it needs a maximum time to wait. This is it. One + * day. DEB retries forever.
+ *
+ */ +Map expansionsForDistribution(distributionType) { + String footer = "# Built for ${project.name}-${project.version} " + + "(${distributionType})" + Map expansions = [ + 'project.name': project.name, + 'project.version': version, + + 'path.conf': [ + 'tar': '$ES_HOME/config', + 'zip': '$ES_HOME/config', + 'def': '/etc/elasticsearch', + ], + 'path.env': [ + 'deb': '/etc/default/elasticsearch', + 'rpm': '/etc/sysconfig/elasticsearch', + /* There isn't one of these files for tar or zip but its important to + make an empty string here so the script can properly skip it. */ + 'def': '', + ], + + 'heap.min': '256m', + 'heap.max': '1g', + + 'stopping.timeout': [ + 'rpm': 86400, + ], + + 'scripts.footer': [ + /* Debian needs exit 0 on these scripts so we add it here and preserve + the pretty footer. */ + 'deb': "exit 0\n${footer}", + 'def': footer + ], + ] + Map result = [:] + expansions = expansions.each { key, value -> + if (value instanceof Map) { + // 'def' is for default but its three characters like 'rpm' and 'deb' + value = value[distributionType] ?: value['def'] + if (value == null) { + return + } + } + result[key] = value + } + return result +} diff --git a/distribution/deb/build.gradle b/distribution/deb/build.gradle index 24eabe78598..9689f5997e8 100644 --- a/distribution/deb/build.gradle +++ b/distribution/deb/build.gradle @@ -18,7 +18,19 @@ */ task buildDeb(type: Deb) { - dependsOn dependencyFiles + dependsOn dependencyFiles, preparePackagingFiles + // Follow elasticsearch's deb file naming convention + archiveName "${packageName}-${project.version}.deb" + packageGroup 'web' + requires 'libc6' + requires 'adduser' + + into('/usr/share/lintian/overrides') { + from("${project.packagingFiles}/lintian/elasticsearch") + } + into('/usr/share/doc/elasticsearch') { + from "${project.packagingFiles}/copyright" + } } artifacts { diff --git a/distribution/deb/src/main/packaging/init.d/elasticsearch b/distribution/deb/src/main/packaging/init.d/elasticsearch index 3a82bbe7f76..9f1f1479a51 100755 --- a/distribution/deb/src/main/packaging/init.d/elasticsearch +++ b/distribution/deb/src/main/packaging/init.d/elasticsearch @@ -81,7 +81,7 @@ MAX_MAP_COUNT=262144 #ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log # Elasticsearch PID file directory -PID_DIR="${packaging.elasticsearch.pid.dir}" +PID_DIR="/var/run/elasticsearch" # End of variables that can be overwritten in $DEFAULT diff --git a/distribution/deb/src/main/packaging/packaging.properties b/distribution/deb/src/main/packaging/packaging.properties deleted file mode 100644 index 3635928c2ee..00000000000 --- a/distribution/deb/src/main/packaging/packaging.properties +++ /dev/null @@ -1,15 +0,0 @@ -# Properties used to build to the DEB package -# - -# Environment file -packaging.env.file=/etc/default/elasticsearch - -# Default configuration directory and file to use in bin/plugin script -packaging.plugin.default.config.dir=${packaging.elasticsearch.conf.dir} - -# Simple marker to check that properties are correctly overridden -packaging.type=deb - -# Custom header for package scripts -packaging.scripts.header=#!/bin/sh${line.separator}set -e -packaging.scripts.footer=exit 0${line.separator}# Built for ${project.name}-${project.version} (${packaging.type}) diff --git a/distribution/deb/src/main/packaging/scripts/conffiles b/distribution/deb/src/main/packaging/scripts/conffiles deleted file mode 100644 index 9f658416784..00000000000 --- a/distribution/deb/src/main/packaging/scripts/conffiles +++ /dev/null @@ -1,5 +0,0 @@ -${packaging.env.file} -${packaging.elasticsearch.conf.dir}/elasticsearch.yml -${packaging.elasticsearch.conf.dir}/logging.yml -/etc/init.d/elasticsearch -/usr/lib/systemd/system/elasticsearch.service diff --git a/distribution/deb/src/main/packaging/scripts/control b/distribution/deb/src/main/packaging/scripts/control deleted file mode 100644 index 1913de78738..00000000000 --- a/distribution/deb/src/main/packaging/scripts/control +++ /dev/null @@ -1,9 +0,0 @@ -Package: elasticsearch -Version: [[version]] -Architecture: all -Maintainer: Elasticsearch Team -Depends: libc6, adduser -Section: web -Priority: optional -Homepage: https://www.elastic.co/ -Description: Elasticsearch is a distributed RESTful search engine built for the cloud. Reference documentation can be found at https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html and the 'Elasticsearch: The Definitive Guide' book can be found at https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle index 04f0c0bda9b..ab2e61ad463 100644 --- a/distribution/rpm/build.gradle +++ b/distribution/rpm/build.gradle @@ -18,7 +18,16 @@ */ task buildRpm(type: Rpm) { - dependsOn dependencyFiles + dependsOn dependencyFiles, preparePackagingFiles + // Follow elasticsearch's rpm file naming convention + archiveName = "${packageName}-${project.version}.rpm" + packageGroup 'Application/Internet' + prefix '/usr' + packager 'Elasticsearch' + release '1' + arch NOARCH + os LINUX + // TODO ospackage doesn't support icon but we used to have one } artifacts { diff --git a/distribution/rpm/src/main/packaging/init.d/elasticsearch b/distribution/rpm/src/main/packaging/init.d/elasticsearch index 6d1f3a4321e..12fed7dbc33 100644 --- a/distribution/rpm/src/main/packaging/init.d/elasticsearch +++ b/distribution/rpm/src/main/packaging/init.d/elasticsearch @@ -3,7 +3,7 @@ # elasticsearch # # chkconfig: 2345 80 20 -# description: Starts and stops a single elasticsearch instance on this system +# description: Starts and stops a single elasticsearch instance on this system # ### BEGIN INIT INFO @@ -32,19 +32,19 @@ if [ -f /etc/rc.d/init.d/functions ]; then fi # Sets the default values for elasticsearch variables used in this script -ES_USER="${packaging.elasticsearch.user}" -ES_GROUP="${packaging.elasticsearch.group}" -ES_HOME="${packaging.elasticsearch.home.dir}" -MAX_OPEN_FILES=${packaging.os.max.open.files} -MAX_MAP_COUNT=${packaging.os.max.map.count} -LOG_DIR="${packaging.elasticsearch.log.dir}" -DATA_DIR="${packaging.elasticsearch.data.dir}" -CONF_DIR="${packaging.elasticsearch.conf.dir}" +ES_USER="elasticsearch" +ES_GROUP="elasticsearch" +ES_HOME="/usr/share/elasticsearch" +MAX_OPEN_FILES=65535 +MAX_MAP_COUNT=262144 +LOG_DIR="/var/log/elasticsearch" +DATA_DIR="/var/lib/elasticsearch" +CONF_DIR="${path.conf}" -PID_DIR="${packaging.elasticsearch.pid.dir}" +PID_DIR="/var/run/elasticsearch" # Source the default env file -ES_ENV_FILE="${packaging.env.file}" +ES_ENV_FILE="${path.env}" if [ -f "$ES_ENV_FILE" ]; then . "$ES_ENV_FILE" fi @@ -70,7 +70,7 @@ export JAVA_HOME lockfile=/var/lock/subsys/$prog # backwards compatibility for old config sysconfig files, pre 0.90.1 -if [ -n $USER ] && [ -z $ES_USER ] ; then +if [ -n $USER ] && [ -z $ES_USER ] ; then ES_USER=$USER fi @@ -126,7 +126,7 @@ start() { stop() { echo -n $"Stopping $prog: " # stop it here, often "killproc $prog" - killproc -p $pidfile -d ${packaging.elasticsearch.stopping.timeout} $prog + killproc -p $pidfile -d ${stopping.timeout} $prog retval=$? echo [ $retval -eq 0 ] && rm -f $lockfile diff --git a/distribution/rpm/src/main/packaging/packaging.properties b/distribution/rpm/src/main/packaging/packaging.properties deleted file mode 100644 index bc4af5f5ceb..00000000000 --- a/distribution/rpm/src/main/packaging/packaging.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Properties used to build to the RPM package -# - -# Environment file -packaging.env.file=/etc/sysconfig/elasticsearch - -# Default configuration directory and file to use in bin/plugin script -packaging.plugin.default.config.dir=${packaging.elasticsearch.conf.dir} - -# Simple marker to check that properties are correctly overridden -packaging.type=rpm - -# Custom header for package scripts -packaging.scripts.header= -packaging.scripts.footer=# Built for ${project.name}-${project.version} (${packaging.type}) - -# Maximum time to wait for elasticsearch to stop (default to 1 day) -packaging.elasticsearch.stopping.timeout=86400 diff --git a/distribution/src/main/packaging/env/elasticsearch b/distribution/src/main/packaging/env/elasticsearch index 178c1e6094f..edacea3ec3b 100644 --- a/distribution/src/main/packaging/env/elasticsearch +++ b/distribution/src/main/packaging/env/elasticsearch @@ -3,21 +3,21 @@ ################################ # Elasticsearch home directory -#ES_HOME=${packaging.elasticsearch.home.dir} +#ES_HOME=/usr/share/elasticsearch # Elasticsearch configuration directory -#CONF_DIR=${packaging.elasticsearch.conf.dir} +#CONF_DIR=${path.conf} # Elasticsearch data directory -#DATA_DIR=${packaging.elasticsearch.data.dir} +#DATA_DIR=/var/lib/elasticsearch # Elasticsearch logs directory -#LOG_DIR=${packaging.elasticsearch.log.dir} +#LOG_DIR=/var/log/elasticsearch # Elasticsearch PID directory -#PID_DIR=${packaging.elasticsearch.pid.dir} +#PID_DIR=/var/run/elasticsearch -# Heap size defaults to ${packaging.elasticsearch.heap.min} min, ${packaging.elasticsearch.heap.max} max +# Heap size defaults to ${heap.min} min, ${heap.max} max # Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g #ES_HEAP_SIZE=2g @@ -34,7 +34,7 @@ #ES_RESTART_ON_UPGRADE=true # Path to the GC log file -#ES_GC_LOG_FILE=${packaging.elasticsearch.log.dir}/gc.log +#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log ################################ # Elasticsearch service @@ -43,15 +43,15 @@ # SysV init.d # # When executing the init script, this user will be used to run the elasticsearch service. -# The default value is '${packaging.elasticsearch.user}' and is declared in the init.d file. +# The default value is 'elasticsearch' and is declared in the init.d file. # Note that this setting is only used by the init script. If changed, make sure that # the configured user can read and write into the data, work, plugins and log directories. -# For systemd service, the user is usually configured in file ${packaging.elasticsearch.systemd.dir}/elasticsearch.service -#ES_USER=${packaging.elasticsearch.user} -#ES_GROUP=${packaging.elasticsearch.group} +# For systemd service, the user is usually configured in file /usr/lib/systemd/system/elasticsearch.service +#ES_USER=elasticsearch +#ES_GROUP=elasticsearch # The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process -ES_STARTUP_SLEEP_TIME=${packaging.elasticsearch.startup.sleep.time} +ES_STARTUP_SLEEP_TIME=5 ################################ # System properties @@ -59,17 +59,17 @@ ES_STARTUP_SLEEP_TIME=${packaging.elasticsearch.startup.sleep.time} # Specifies the maximum file descriptor number that can be opened by this process # When using Systemd, this setting is ignored and the LimitNOFILE defined in -# ${packaging.elasticsearch.systemd.dir}/elasticsearch.service takes precedence -#MAX_OPEN_FILES=${packaging.os.max.open.files} +# /usr/lib/systemd/system/elasticsearch.service takes precedence +#MAX_OPEN_FILES=65535 # The maximum number of bytes of memory that may be locked into RAM # Set to "unlimited" if you use the 'bootstrap.mlockall: true' option # in elasticsearch.yml (ES_HEAP_SIZE must also be set). # When using Systemd, the LimitMEMLOCK property must be set -# in ${packaging.elasticsearch.systemd.dir}/elasticsearch.service +# in /usr/lib/systemd/system/elasticsearch.service #MAX_LOCKED_MEMORY=unlimited # Maximum number of VMA (Virtual Memory Areas) a process can own # When using Systemd, this setting is ignored and the 'vm.max_map_count' -# property is set at boot time in ${packaging.elasticsearch.systemd.sysctl.dir}/elasticsearch.conf -#MAX_MAP_COUNT=${packaging.os.max.map.count} +# property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf +#MAX_MAP_COUNT=262144 diff --git a/distribution/src/main/packaging/packaging.properties b/distribution/src/main/packaging/packaging.properties deleted file mode 100644 index 7cba6ab9460..00000000000 --- a/distribution/src/main/packaging/packaging.properties +++ /dev/null @@ -1,30 +0,0 @@ -# Common properties for building ZIP,GZ,RPM and DEB packages -# -# Properties defined here can be overridden with specific settings, -# like in rpm/packaging.properties and deb/packaging.properties. - -# Environment file -packaging.env.file= - -# Default configuration directory and file to use in bin/plugin script -packaging.plugin.default.config.dir=$ES_HOME/config - -# Default values for min/max heap memory allocated to elasticsearch java process -packaging.elasticsearch.heap.min=256m -packaging.elasticsearch.heap.max=1g - -# Specifies the maximum file descriptor number -packaging.os.max.open.files=65535 - -# Maximum number of VMA (Virtual Memory Areas) a process can own -packaging.os.max.map.count=262144 - -# Default number of seconds to wait before checking if Elasticsearch started successfully as a daemon process -packaging.elasticsearch.startup.sleep.time=5 - -# Simple marker to check that properties are correctly overridden -packaging.type=tar.gz - -# Custom header for package scripts -packaging.scripts.header= -packaging.scripts.footer= diff --git a/distribution/src/main/packaging/scripts/postinst b/distribution/src/main/packaging/scripts/postinst index 3d47a0338d4..61ac5f27cd7 100644 --- a/distribution/src/main/packaging/scripts/postinst +++ b/distribution/src/main/packaging/scripts/postinst @@ -1,5 +1,3 @@ -${packaging.scripts.header} - # # This script is executed in the post-installation phase # @@ -13,11 +11,11 @@ ${packaging.scripts.header} # Sets the default values for elasticsearch variables used in this script -ES_USER="${packaging.elasticsearch.user}" -ES_GROUP="${packaging.elasticsearch.group}" +ES_USER="elasticsearch" +ES_GROUP="elasticsearch" # Source the default env file -ES_ENV_FILE="${packaging.env.file}" +ES_ENV_FILE="${path.env}" if [ -f "$ES_ENV_FILE" ]; then . "$ES_ENV_FILE" fi @@ -98,4 +96,4 @@ elif [ "$RESTART_ON_UPGRADE" = "true" ]; then echo " OK" fi -${packaging.scripts.footer} +${scripts.footer} diff --git a/distribution/src/main/packaging/scripts/postrm b/distribution/src/main/packaging/scripts/postrm index ee1c49b14ad..d4104845249 100644 --- a/distribution/src/main/packaging/scripts/postrm +++ b/distribution/src/main/packaging/scripts/postrm @@ -1,5 +1,3 @@ -${packaging.scripts.header} - # # This script is executed in the post-removal phase # @@ -51,16 +49,16 @@ case "$1" in esac # Sets the default values for elasticsearch variables used in this script -ES_USER="${packaging.elasticsearch.user}" -ES_GROUP="${packaging.elasticsearch.group}" -LOG_DIR="${packaging.elasticsearch.log.dir}" -PLUGINS_DIR="${packaging.elasticsearch.plugins.dir}" -PID_DIR="${packaging.elasticsearch.pid.dir}" -DATA_DIR="${packaging.elasticsearch.data.dir}" +ES_USER="elasticsearch" +ES_GROUP="elasticsearch" +LOG_DIR="/var/log/elasticsearch" +PLUGINS_DIR="/usr/share/elasticsearch/plugins" +PID_DIR="/var/run/elasticsearch" +DATA_DIR="/var/lib/elasticsearch" # Source the default env file if [ "$SOURCE_ENV_FILE" = "true" ]; then - ES_ENV_FILE="${packaging.env.file}" + ES_ENV_FILE="${path.env}" if [ -f "$ES_ENV_FILE" ]; then . "$ES_ENV_FILE" fi @@ -116,4 +114,4 @@ if [ "$REMOVE_USER_AND_GROUP" = "true" ]; then fi fi -${packaging.scripts.footer} +${scripts.footer} diff --git a/distribution/src/main/packaging/scripts/preinst b/distribution/src/main/packaging/scripts/preinst index d3df84ecd02..21c4f137c85 100644 --- a/distribution/src/main/packaging/scripts/preinst +++ b/distribution/src/main/packaging/scripts/preinst @@ -1,5 +1,3 @@ -${packaging.scripts.header} - # # This script is executed in the pre-installation phase # @@ -14,11 +12,11 @@ ${packaging.scripts.header} # Sets the default values for elasticsearch variables used in this script -ES_USER="${packaging.elasticsearch.user}" -ES_GROUP="${packaging.elasticsearch.group}" +ES_USER="elasticsearch" +ES_GROUP="elasticsearch" # Source the default env file -ES_ENV_FILE="${packaging.env.file}" +ES_ENV_FILE="${path.env}" if [ -f "$ES_ENV_FILE" ]; then . "$ES_ENV_FILE" fi @@ -80,4 +78,4 @@ case "$1" in ;; esac -${packaging.scripts.footer} +${scripts.footer} diff --git a/distribution/src/main/packaging/scripts/prerm b/distribution/src/main/packaging/scripts/prerm index 11d51c68637..07f39759aee 100644 --- a/distribution/src/main/packaging/scripts/prerm +++ b/distribution/src/main/packaging/scripts/prerm @@ -1,5 +1,3 @@ -${packaging.scripts.header} - # # This script is executed in the pre-remove phase # @@ -66,4 +64,4 @@ if [ "$STOP_REQUIRED" = "true" ]; then echo " OK" fi -${packaging.scripts.footer} +${scripts.footer} diff --git a/distribution/src/main/packaging/systemd/elasticsearch.conf b/distribution/src/main/packaging/systemd/elasticsearch.conf index 98dd5e61c25..d079f28b995 100644 --- a/distribution/src/main/packaging/systemd/elasticsearch.conf +++ b/distribution/src/main/packaging/systemd/elasticsearch.conf @@ -1 +1 @@ -d ${packaging.elasticsearch.pid.dir} 0755 ${packaging.elasticsearch.user} ${packaging.elasticsearch.group} - - +d /var/run/elasticsearch 0755 elasticsearch elasticsearch - - diff --git a/distribution/src/main/packaging/systemd/elasticsearch.service b/distribution/src/main/packaging/systemd/elasticsearch.service index d8f56f7d053..4a280a09d38 100644 --- a/distribution/src/main/packaging/systemd/elasticsearch.service +++ b/distribution/src/main/packaging/systemd/elasticsearch.service @@ -5,21 +5,21 @@ Wants=network-online.target After=network-online.target [Service] -Environment=ES_HOME=${packaging.elasticsearch.home.dir} -Environment=CONF_DIR=${packaging.elasticsearch.conf.dir} -Environment=DATA_DIR=${packaging.elasticsearch.data.dir} -Environment=LOG_DIR=${packaging.elasticsearch.log.dir} -Environment=PID_DIR=${packaging.elasticsearch.pid.dir} -EnvironmentFile=-${packaging.env.file} +Environment=ES_HOME=/usr/share/elasticsearch +Environment=CONF_DIR=${path.conf} +Environment=DATA_DIR=/var/lib/elasticsearch +Environment=LOG_DIR=/var/log/elasticsearch +Environment=PID_DIR=/var/run/elasticsearch +EnvironmentFile=-${path.env} -WorkingDirectory=${packaging.elasticsearch.home.dir} +WorkingDirectory=/usr/share/elasticsearch -User=${packaging.elasticsearch.user} -Group=${packaging.elasticsearch.group} +User=elasticsearch +Group=elasticsearch -ExecStartPre=${packaging.elasticsearch.bin.dir}/elasticsearch-systemd-pre-exec +ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec -ExecStart=${packaging.elasticsearch.bin.dir}/elasticsearch \ +ExecStart=/usr/share/elasticsearch/bin/elasticsearch \ -Des.pidfile=${PID_DIR}/elasticsearch.pid \ -Des.default.path.home=${ES_HOME} \ -Des.default.path.logs=${LOG_DIR} \ @@ -33,11 +33,11 @@ StandardOutput=null StandardError=journal # Specifies the maximum file descriptor number that can be opened by this process -LimitNOFILE=${packaging.os.max.open.files} +LimitNOFILE=65535 # Specifies the maximum number of bytes of memory that may be locked into RAM # Set to "infinity" if you use the 'bootstrap.mlockall: true' option -# in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in ${packaging.env.file} +# in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in ${path.env} #LimitMEMLOCK=infinity # Disable timeout logic and wait until process is stopped @@ -55,4 +55,4 @@ SuccessExitStatus=143 [Install] WantedBy=multi-user.target -# Built for ${project.name}-${project.version} (${packaging.type}) +# Built for ${project.name}-${project.version} (${project.name}) diff --git a/distribution/src/main/packaging/systemd/sysctl/elasticsearch.conf b/distribution/src/main/packaging/systemd/sysctl/elasticsearch.conf index 052cd89cf0b..62ea54d8697 100644 --- a/distribution/src/main/packaging/systemd/sysctl/elasticsearch.conf +++ b/distribution/src/main/packaging/systemd/sysctl/elasticsearch.conf @@ -1 +1 @@ -vm.max_map_count=${packaging.os.max.map.count} +vm.max_map_count=262144 diff --git a/distribution/src/main/resources/bin/elasticsearch b/distribution/src/main/resources/bin/elasticsearch index 3a69a3cdbbc..459169bc3c4 100755 --- a/distribution/src/main/resources/bin/elasticsearch +++ b/distribution/src/main/resources/bin/elasticsearch @@ -42,12 +42,9 @@ # Be aware that you will be entirely responsible for populating the needed # environment variables. -# Maven will replace the project.name with elasticsearch below. If that -# hasn't been done, we assume that this is not a packaged version and the -# user has forgotten to run Maven to create a package. - -IS_PACKAGED_VERSION='${project.parent.artifactId}' -if [ "$IS_PACKAGED_VERSION" != "distributions" ]; then +# Check to see if you are trying to run this without building it first. Gradle +# will replace the project.name with _something_. +if echo '${project.name}' | grep project.name > /dev/null ; then cat >&2 << EOF Error: You must build the project with Maven or download a pre-built package before you can run Elasticsearch. See 'Building from Source' in README.textile diff --git a/distribution/src/main/resources/bin/elasticsearch.in.bat b/distribution/src/main/resources/bin/elasticsearch.in.bat index 4e88d22f1dd..6f6550dcdf0 100644 --- a/distribution/src/main/resources/bin/elasticsearch.in.bat +++ b/distribution/src/main/resources/bin/elasticsearch.in.bat @@ -4,7 +4,7 @@ if DEFINED JAVA_HOME goto cont :err ECHO JAVA_HOME environment variable must be set! 1>&2 -EXIT /B 1 +EXIT /B 1 :cont set SCRIPT_DIR=%~dp0 @@ -14,11 +14,11 @@ for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI REM ***** JAVA options ***** if "%ES_MIN_MEM%" == "" ( -set ES_MIN_MEM=${packaging.elasticsearch.heap.min} +set ES_MIN_MEM=${heap.min} ) if "%ES_MAX_MEM%" == "" ( -set ES_MAX_MEM=${packaging.elasticsearch.heap.max} +set ES_MAX_MEM=${heap.max} ) if NOT "%ES_HEAP_SIZE%" == "" ( @@ -93,7 +93,7 @@ set JAVA_OPTS=%JAVA_OPTS% -Djna.nosys=true REM check in case a user was using this mechanism if "%ES_CLASSPATH%" == "" ( -set ES_CLASSPATH=%ES_HOME%/lib/${project.build.finalName}.jar;%ES_HOME%/lib/* +set ES_CLASSPATH=%ES_HOME%/lib/elasticsearch-${project.version}.jar;%ES_HOME%/lib/* ) else ( ECHO Error: Don't modify the classpath with ES_CLASSPATH, Best is to add 1>&2 ECHO additional elements via the plugin mechanism, or if code must really be 1>&2 diff --git a/distribution/src/main/resources/bin/elasticsearch.in.sh b/distribution/src/main/resources/bin/elasticsearch.in.sh index 5ac0025fe29..f859a06ffab 100644 --- a/distribution/src/main/resources/bin/elasticsearch.in.sh +++ b/distribution/src/main/resources/bin/elasticsearch.in.sh @@ -10,13 +10,13 @@ EOF exit 1 fi -ES_CLASSPATH="$ES_HOME/lib/${project.build.finalName}.jar:$ES_HOME/lib/*" +ES_CLASSPATH="$ES_HOME/lib/elasticsearch-${project.version}.jar:$ES_HOME/lib/*" if [ "x$ES_MIN_MEM" = "x" ]; then - ES_MIN_MEM=${packaging.elasticsearch.heap.min} + ES_MIN_MEM=${heap.min} fi if [ "x$ES_MAX_MEM" = "x" ]; then - ES_MAX_MEM=${packaging.elasticsearch.heap.max} + ES_MAX_MEM=${heap.max} fi if [ "x$ES_HEAP_SIZE" != "x" ]; then ES_MIN_MEM=$ES_HEAP_SIZE diff --git a/distribution/src/main/resources/bin/plugin b/distribution/src/main/resources/bin/plugin index 35dbe3a620a..95011870358 100755 --- a/distribution/src/main/resources/bin/plugin +++ b/distribution/src/main/resources/bin/plugin @@ -25,12 +25,12 @@ ES_HOME=`cd "$ES_HOME"; pwd` # Sets the default values for elasticsearch variables used in this script if [ -z "$CONF_DIR" ]; then - CONF_DIR="${packaging.plugin.default.config.dir}" + CONF_DIR="${path.conf}" fi # The default env file is defined at building/packaging time. -# For a ${packaging.type} package, the value is "${packaging.env.file}". -ES_ENV_FILE="${packaging.env.file}" +# For a ${project.name} package, the value is "${path.env}". +ES_ENV_FILE="${path.env}" # If an include is specified with the ES_INCLUDE environment variable, use it if [ -n "$ES_INCLUDE" ]; then diff --git a/distribution/src/main/resources/bin/service.bat b/distribution/src/main/resources/bin/service.bat index 9822e6bbdc0..5b5fbff7522 100644 --- a/distribution/src/main/resources/bin/service.bat +++ b/distribution/src/main/resources/bin/service.bat @@ -128,8 +128,8 @@ goto:eof ) :foundJVM -if "%ES_MIN_MEM%" == "" set ES_MIN_MEM=${packaging.elasticsearch.heap.min} -if "%ES_MAX_MEM%" == "" set ES_MAX_MEM=${packaging.elasticsearch.heap.max} +if "%ES_MIN_MEM%" == "" set ES_MIN_MEM=${heap.min} +if "%ES_MAX_MEM%" == "" set ES_MAX_MEM=${heap.max} if NOT "%ES_HEAP_SIZE%" == "" set ES_MIN_MEM=%ES_HEAP_SIZE% if NOT "%ES_HEAP_SIZE%" == "" set ES_MAX_MEM=%ES_HEAP_SIZE% diff --git a/distribution/tar/src/main/assemblies/targz-bin.xml b/distribution/tar/src/main/assemblies/targz-bin.xml deleted file mode 100644 index 7a42a2de986..00000000000 --- a/distribution/tar/src/main/assemblies/targz-bin.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - targz - - tar.gz - - - true - - - ../src/main/assemblies/common-bin.xml - - diff --git a/distribution/zip/src/main/assemblies/zip-bin.xml b/distribution/zip/src/main/assemblies/zip-bin.xml deleted file mode 100644 index 661d5540cf7..00000000000 --- a/distribution/zip/src/main/assemblies/zip-bin.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - zip - - zip - - - true - - - ../src/main/assemblies/common-bin.xml - - diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle index bdc68a07552..2b701842c18 100644 --- a/qa/vagrant/build.gradle +++ b/qa/vagrant/build.gradle @@ -19,30 +19,43 @@ import org.elasticsearch.gradle.vagrant.VagrantCommandTask import org.elasticsearch.gradle.vagrant.BatsOverVagrantTask +import org.elasticsearch.gradle.FileContentsTask import org.gradle.api.InvalidUserDataException -def testScripts = '2*.bats' // Once the rpm and deb are build this should be *.bats -def testCommand = "cd \$TESTROOT && sudo bats --tap \$BATS/$testScripts" -def smokeTestCommand = 'echo I work' -def representativeBoxes = ['ubuntu-1404', 'centos-7'] -def boxes = representativeBoxes + ['ubuntu-1204', 'ubuntu-1504', 'debian-8', - 'centos-6', 'oel-7', 'fedora-22', 'opensuse-13', 'sles-12'] +String testScripts = '*.bats' +String testCommand = "cd \$TESTROOT && sudo bats --tap \$BATS/$testScripts" +String smokeTestCommand = 'echo I work' +List representativeBoxes = ['ubuntu-1404', 'centos-7'] +List boxes = representativeBoxes + ['ubuntu-1204', 'ubuntu-1504', + 'debian-8', 'centos-6', 'oel-7', 'fedora-22', 'opensuse-13', 'sles-12'] + +/* The version of elasticsearch that we upgrade *from* as part of testing + * upgrades. */ +String upgradeFromVersion = '2.0.0' configurations { test } +repositories { + mavenCentral() +} + dependencies { - test project("${projectsPrefix}:distribution:tar") - // NOCOMMMIT: we need the rpm and deb! - // test project(path: ':distribution:rpm', configuration: 'archives') - // test project(path: ':distribution:deb', configuration: 'archives') + test project(path: "${projectsPrefix}:distribution:tar", configuration: 'archives') + test project(path: "${projectsPrefix}:distribution:rpm", configuration: 'archives') + test project(path: "${projectsPrefix}:distribution:deb", configuration: 'archives') + // Collect all the plugins for (Project subproj : project.rootProject.subprojects) { if (subproj.path.startsWith(':plugins:')) { test project("${projectsPrefix}${subproj.path}") } } + + // The version of elasticsearch that we upgrade *from* + test "org.elasticsearch.distribution.deb:elasticsearch:$upgradeFromVersion@deb" + test "org.elasticsearch.distribution.rpm:elasticsearch:$upgradeFromVersion@rpm" } task checkPackages { @@ -58,14 +71,36 @@ task checkPackagesAllDistros { } task clean(type: Delete) { - group = 'Build' + group 'Build' delete buildDir } -task copyDepsToTestRoot(type: Copy) { - description "Dump bats test dependencies into the \$TESTROOT." - into "$buildDir/testroot" +File testRoot = new File("$buildDir/testroot") +task createTestRoot { + outputs.dir testRoot + doLast { + testRoot.mkdirs() + } +} + +task createVersionFile(type: FileContentsTask) { + dependsOn createTestRoot + file "${testRoot}/version" + contents = version +} + +task createUpgradeFromFile(type: FileContentsTask) { + dependsOn createTestRoot + file "${testRoot}/upgrade_from_version" + contents = upgradeFromVersion +} + +task prepareTestRoot(type: Copy) { + description 'Dump bats test dependencies into the $TESTROOT' + into testRoot from configurations.test + + dependsOn createVersionFile, createUpgradeFromFile } task checkVagrantVersion(type: Exec) { @@ -74,10 +109,10 @@ task checkVagrantVersion(type: Exec) { commandLine 'vagrant', '--version' standardOutput = new ByteArrayOutputStream() doLast { - def version = standardOutput.toString().trim() + String version = standardOutput.toString().trim() if ((version ==~ /Vagrant 1\.[789]\..+/) == false) { - throw new InvalidUserDataException('Illegal version of vagrant [' + - version + ']. Need [Vagrant 1.7+]') + throw new InvalidUserDataException( + "Illegal version of vagrant [${version}]. Need [Vagrant 1.7+]") } } } @@ -94,10 +129,10 @@ task vagrantSmokeTestAllDistros { // Each box gets it own set of tasks boxes.each { box -> - def boxTask = taskifyBoxName box - task "vagrantUp$boxTask"(type: VagrantCommandTask) { + String boxTask = taskifyBoxName box + task "vagrantUp${boxTask}"(type: VagrantCommandTask) { group 'Vagrant' - description "Startup a vagrant VM running $box" + description "Startup a vagrant VM running ${box}" boxName box /* Its important that we try to reprovision the box even if it already exists. That way updates to the vagrant configuration take automatically. @@ -114,34 +149,34 @@ boxes.each { box -> SKIPPED but that would require running vagrant status which is slow! */ dependsOn checkVagrantVersion } - task "vagrantHalt$boxTask"(type: VagrantCommandTask) { + task "vagrantHalt${boxTask}"(type: VagrantCommandTask) { group 'Vagrant' description "Shutdown the vagrant VM running $box" boxName box commandLine 'halt', box } - task "smokeTest$boxTask"(type: Exec) { + task "smokeTest${boxTask}"(type: Exec) { group 'Vagrant' - description "Smoke test the $box VM" - dependsOn "vagrantUp$boxTask" - finalizedBy "vagrantHalt$boxTask" + description "Smoke test the ${box} VM" + dependsOn "vagrantUp${boxTask}" + finalizedBy "vagrantHalt${boxTask}" commandLine 'vagrant', 'ssh', box, '--command', - "set -o pipefail && $smokeTestCommand | sed -ue \'s/^/ $box: /'" + "set -o pipefail && ${smokeTestCommand} | sed -ue \'s/^/ ${box}: /'" vagrantSmokeTestAllDistros.dependsOn name if (representativeBoxes.contains(box)) { vagrantSmokeTest.dependsOn name } } - task "check$boxTask"(type: BatsOverVagrantTask) { + task "check${boxTask}"(type: BatsOverVagrantTask) { group 'Package Verification' - description "Run packaging tests against $box" - dependsOn "vagrantUp$boxTask" - finalizedBy "vagrantHalt$boxTask" + description "Run packaging tests against ${box}" + dependsOn "vagrantUp${boxTask}" + finalizedBy "vagrantHalt${boxTask}" boxName box command testCommand - dependsOn copyDepsToTestRoot + dependsOn prepareTestRoot checkPackagesAllDistros.dependsOn name if (representativeBoxes.contains(box)) { checkPackages.dependsOn name @@ -151,6 +186,6 @@ boxes.each { box -> // Twists the box name into a sensible task name -def taskifyBoxName(box) { +String taskifyBoxName(box) { box.capitalize().replace('-', '') } diff --git a/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash index bd9e28e3f09..79e4426095a 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash @@ -239,6 +239,10 @@ fi install_and_check_plugin lang python jython-standalone-*.jar } +@test "[$GROUP] install mapper-attachments plugin" { + install_and_check_plugin mapper attachments +} + @test "[$GROUP] install murmur3 mapper plugin" { install_and_check_plugin mapper murmur3 } @@ -343,6 +347,10 @@ fi remove_plugin lang-python } +@test "[$GROUP] remove mapper-attachments plugin" { + remove_plugin mapper-attachments +} + @test "[$GROUP] remove murmur3 mapper plugin" { remove_plugin mapper-murmur3 } @@ -423,4 +431,3 @@ fi fi remove_jvm_example } -