honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix rpm and deb distributions 

and test them with vagrant
This commit is contained in:
Nik Everett 2015-11-05 15:47:54 -05:00
parent e837140385
commit 9b0a47d8e3
35 changed files with 526 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
TESTING.asciidoc
View File

@ -424,7 +424,7 @@ sudo bats $BATS/*rpm*.bats
If you wanted to retest all the release artifacts on a single VM you could:
-------------------------------------------------
gradle copyDepsToTestRoot
gradle prepareTestRoot
vagrant up trusty --provider virtualbox && vagrant ssh trusty
cd $TESTROOT
sudo bats $BATS/*.bats

8
Vagrantfile vendored
View File

@ -251,10 +251,10 @@ def provision(config,
rm -rf /tmp/bats
}
cat \<\<VARS > /etc/profile.d/elasticsearch_vars.sh
export ZIP=/elasticsearch/distribution/zip/build/releases
export TAR=/elasticsearch/distribution/tar/build/releases
export RPM=/elasticsearch/distribution/rpm/build/releases
export DEB=/elasticsearch/distribution/deb/build/releases
export ZIP=/elasticsearch/distribution/zip/build/distributions
export TAR=/elasticsearch/distribution/tar/build/distributions
export RPM=/elasticsearch/distribution/rpm/build/distributions
export DEB=/elasticsearch/distribution/deb/build/distributions
export TESTROOT=/elasticsearch/qa/vagrant/build/testroot
export BATS=/elasticsearch/qa/vagrant/src/test/resources/packaging/scripts
VARS

48
buildSrc/src/main/groovy/org/elasticsearch/gradle/EmptyDirTask.groovy Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.*
import org.gradle.internal.nativeintegration.filesystem.Chmod
import java.io.File
import javax.inject.Inject
/**
* Creates an empty directory.
*/
class EmptyDirTask extends DefaultTask {
@Input
Object dir
@Input
int dirMode = 0755
@TaskAction
void create() {
dir = dir as File
dir.mkdirs()
getChmod().chmod(dir, dirMode)
}
@Inject
Chmod getChmod() {
throw new UnsupportedOperationException()
}
}

50
buildSrc/src/main/groovy/org/elasticsearch/gradle/FileContentsTask.groovy Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.*
import java.io.File
/**
* Creates a file and sets it contents to something.
*/
class FileContentsTask extends DefaultTask {
/**
* The file to be built. Must be of type File to make @OutputFile happy.
*/
@OutputFile
File file
@Input
Object contents
/**
* The file to be built. Takes any objecct and coerces to a file.
*/
void setFile(Object file) {
this.file = file as File
}
@TaskAction
void setContents() {
file = file as File
file.text = contents.toString()
}
}

4
buildSrc/src/main/resources/deb/README Normal file
View File

@ -0,0 +1,4 @@
This directory contains templates that work around gradle-ospackage-plugin
trying to be helpful and adding templates for your os packaging scripts. We
have relatively nice scripts already so we just override the templates to be
mostly noops.

3
buildSrc/src/main/resources/deb/conffiles.ftl Normal file
View File

@ -0,0 +1,3 @@
<% files.each {file -> %><%= file
%>
<% } %>

2
buildSrc/src/main/resources/deb/postinst.ftl Normal file
View File

@ -0,0 +1,2 @@
#!/bin/sh -e
<% commands.each {command -> %><%= command %><% } %>

2
buildSrc/src/main/resources/deb/preinst.ftl Normal file
View File

@ -0,0 +1,2 @@
#!/bin/sh -e
<% commands.each {command -> %><%= command %><% } %>

291
distribution/build.gradle
View File

@ -20,7 +20,10 @@
import org.apache.tools.ant.filters.FixCrLfFilter
import org.elasticsearch.gradle.precommit.DependencyLicensesTask
import org.elasticsearch.gradle.test.RunTask
import org.elasticsearch.gradle.EmptyDirTask
import org.elasticsearch.gradle.MavenFilteringHack
import org.gradle.api.InvalidUserDataException
import org.gradle.internal.nativeintegration.filesystem.Chmod
// for deb/rpm
buildscript {
@ -68,21 +71,7 @@ subprojects {
* Properties to expand when copying packaging files *
*****************************************************************************/
project.ext {
expansions = [
'project.version': version,
'project.parent.artifactId': 'distributions',
// Default values for min/max heap memory allocated to elasticsearch java process
'packaging.elasticsearch.heap.min': '256m',
'packaging.elasticsearch.heap.max': '1g',
'project.build.finalName': "elasticsearch-${version}",
// Default configuration directory and file to use in bin/plugin script
'packaging.plugin.default.config.dir': '$ES_HOME/config',
'packaging.plugin.default.config.file': '$ES_HOME/config/elasticsearch.yml',
'packaging.env.file': '',
// TODO: do we really need this marker? the tgz and zip are exactly the same,
// we should not need to specify twice just to change this
'packaging.type': 'tar.gz',
]
expansions = expansionsForDistribution(project.name)
/*****************************************************************************
* Common files in all distributions *
@ -104,6 +93,7 @@ subprojects {
exclude 'bin/*.exe'
exclude 'config/**'
filesMatching('bin/*') { it.setMode(0755) }
MavenFilteringHack.filter(it, expansions)
}
}
}
@ -121,70 +111,182 @@ configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) {
with copySpec {
with commonFiles
from('../src/main/resources') {
include 'bin/*.bat'
include 'bin/*.bat'
filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf'))
}
MavenFilteringHack.filter(it, expansions)
}
from('../src/main/resources') {
include 'bin/*.exe'
}
include 'bin/*.exe'
}
}
}
}
/*****************************************************************************
* Deb and rpm configuration *
*****************************************************************************/
// ospackage supports adding empty dirs with directory() to rpm, but not deb...yet
// https://github.com/nebula-plugins/gradle-ospackage-plugin/issues/115
// however, even adding just for rpm doesn't seem to work...
// gradle may also get native support https://issues.gradle.org/browse/GRADLE-1671
// in the meantime, we hack this by copying an empty dir
// TODO: HACK DOES NOT WORK
/*ext.emptyDir = new File(project.buildDir, 'empty')
Closure emptyDirSpec() {
return {
from emptyDir
addParentDirs false
createDirectoryEntry true
}
}
task createEmptyDir << {
emptyDir.mkdirs()
}
buildRpm.dependsOn createEmptyDir
buildDeb.dependsOn createEmptyDir
*/
/*****************************************************************************
* Deb and rpm configuration *
*****************************************************************************/
*****************************************************************************
*
* The general strategy here is to build a directory on disk, packagingFiles
* that contains stuff that needs to be copied into the distributions. This is
* important for two reasons:
* 1. ospackage wants to copy the directory permissions that it sees off of the
* filesystem. If you ask it to create a directory that doesn't already
* exist on disk it petulantly creates it with 0755 permissions, no matter
* how hard you try to convince it otherwise.
* 2. Convincing ospackage to pick up an empty directory as part of a set of
* directories on disk is reasonably easy. Convincing it to just create an
* empty directory requires more wits than I have.
* 3. ospackage really wants to suck up some of the debian control scripts
* directly from the filesystem. It doesn't want to process them through
* MavenFilteringHack or any other copy-style action.
*/
configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) {
File packagingFiles = new File(buildDir, 'packaging')
project.ext.packagingFiles = packagingFiles
task processPackagingFiles(type: Copy) {
from '../src/main/packaging'
from 'src/main/packaging'
MavenFilteringHack.filter(it, expansions)
into packagingFiles
/* Explicitly declare the outputs so that gradle won't skip this task if
one of the other tasks like createEtc run first and create the packaging
directory as a side effect. */
outputs.dir("${packagingFiles}/scripts")
outputs.dir("${packagingFiles}/env")
outputs.dir("${packagingFiles}/systemd")
}
task createEtc(type: EmptyDirTask) {
dir "${packagingFiles}/etc/elasticsearch"
dirMode 0750
}
task createEtcScripts(type: EmptyDirTask) {
dependsOn createEtc
dir "${packagingFiles}/etc/elasticsearch/scripts"
dirMode 0750
}
task fillEtc(type: Copy) {
dependsOn createEtc, createEtcScripts
with configFiles
into "${packagingFiles}/etc/elasticsearch"
/* Explicitly declare the output files so this task doesn't consider itself
up to date when the directory is created, which it would by default. And
that'll happen when createEtc runs. */
outputs.file "${packagingFiles}/etc/elasticsearch/elasticsearch.yml"
outputs.file "${packagingFiles}/etc/elasticsearch/logging.yml"
}
task createPidDir(type: EmptyDirTask) {
dir "${packagingFiles}/var/run/elasticsearch"
}
task createLogDir(type: EmptyDirTask) {
dir "${packagingFiles}/var/log/elasticsearch"
}
task createDataDir(type: EmptyDirTask) {
dir "${packagingFiles}/var/lib/elasticsearch"
}
task createPluginsDir(type: EmptyDirTask) {
dir "${packagingFiles}/usr/share/elasticsearch/plugins"
}
/**
* Setup the build/packaging directory to be like the target filesystem
* because ospackage will use arbitrary permissions if you try to create a
* directory that doesn't exist on the filesystem.
*/
task preparePackagingFiles {
dependsOn processPackagingFiles, fillEtc, createPidDir, createLogDir,
createDataDir, createPluginsDir
}
apply plugin: 'nebula.ospackage-base'
ospackage {
packageName = 'elasticsearch'
// TODO: '-' is an illegal character in rpm version...redline croaks
version = '3.0.0'
packageName 'elasticsearch'
maintainer 'Elasticsearch Team <info@elastic.co>'
summary '''
Elasticsearch is a distributed RESTful search engine built for the cloud.
Reference documentation can be found at
https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html
and the 'Elasticsearch: The Definitive Guide' book can be found at
https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html
'''.stripIndent().replace('\n', ' ').trim()
url 'https://www.elastic.co/'
/* The version of the package can't contain -SNAPSHOT so we rip it off if
we see it. We'll add it back on to the file name though. */
version project.version.replace('-SNAPSHOT', '')
String scripts = "${packagingFiles}/scripts"
preInstall file("${scripts}/preinst")
postInstall file("${scripts}/postinst")
preUninstall file("${scripts}/prerm")
postUninstall file("${scripts}/postrm")
into '/usr/share/elasticsearch'
user 'root'
permissionGroup 'root'
with libFiles
with copySpec {
with commonFiles
// TODO: omit LICENSE.txt file on deb??
if (project.name == 'deb') {
// Deb gets a copyright file instead.
exclude 'LICENSE.txt'
}
}
into('/etc/elasticsearch') {
with configFiles
//into('scripts', emptyDirSpec())
createDirectoryEntry = true
includeEmptyDirs = true
configurationFile '/etc/elasticsearch/elasticsearch.yml'
configurationFile '/etc/elasticsearch/logging.yml'
into('/etc') {
from "${packagingFiles}/etc"
fileMode 0750
permissionGroup 'elasticsearch'
includeEmptyDirs true
createDirectoryEntry true
}
directory('/etc/elasticsearch/scripts')
into('/usr/lib/tmpfiles.d') {
from "${packagingFiles}/systemd/elasticsearch.conf"
}
configurationFile '/usr/lib/systemd/system/elasticsearch.service'
into('/usr/lib/systemd/system') {
from "${packagingFiles}/systemd/elasticsearch.service"
}
into('/usr/lib/sysctl.d') {
from "${packagingFiles}/systemd/sysctl/elasticsearch.conf"
}
configurationFile '/etc/init.d/elasticsearch'
into('/etc/init.d') {
from "${packagingFiles}/init.d/elasticsearch"
fileMode 0755
}
configurationFile project.expansions['path.env']
into(new File(project.expansions['path.env']).getParent()) {
from "${project.packagingFiles}/env/elasticsearch"
}
/**
* Suck up all the empty directories that we need to install into the path.
*/
Closure suckUpEmptyDirectories = { path ->
into(path) {
from "${packagingFiles}/${path}"
includeEmptyDirs true
createDirectoryEntry true
/* All of these empty directories have this ownership. We're just
lucky! */
user 'elasticsearch'
permissionGroup 'elasticsearch'
}
}
suckUpEmptyDirectories('/var/run')
suckUpEmptyDirectories('/var/log')
suckUpEmptyDirectories('/var/lib')
suckUpEmptyDirectories('/usr/share/elasticsearch')
}
// TODO: re-enable tests when we have real rpm and deb distros!
integTest.enabled = false
}
// TODO: dependency checks should really be when building the jar itself, which would remove the need
@ -199,3 +301,80 @@ DependencyLicensesTask.configure(project) {
RunTask.configure(project)
/**
* Build some variables that are replaced in the packages. This includes both
* scripts like bin/elasticsearch and bin/plugin that a user might run and also
* scripts like postinst which are run as part of the installation.
*
* <dl>
* <dt>package.name</dt>
* <dd>The name of the project. Its sprinkled throughout the scripts.</dd>
* <dt>package.version</dt>
* <dd>The version of the project. Its mostly used to find the exact jar name.
* </dt>
* <dt>path.conf</dt>
* <dd>The default directory from which to load configuration. This is used in
* the packaging scripts, but in that context it is always
* /etc/elasticsearch. Its also used in bin/plugin, where it is
* /etc/elasticsearch for the os packages but $ESHOME/config otherwise.</dd>
* <dt>path.env</dt>
* <dd>The env file sourced before bin/elasticsearch to set environment
* variables. Think /etc/defaults/elasticsearch.</dd>
* <dt>heap.min and heap.max</dt>
* <dd>Default min and max heap</dd>
* <dt>scripts.footer</dt>
* <dd>Footer appended to control scripts embedded in the distribution that is
* (almost) entirely there for cosmetic reasons.</dd>
* <dt>stopping.timeout</dt>
* <dd>RPM's init script needs to wait for elasticsearch to stop before
* returning from stop and it needs a maximum time to wait. This is it. One
* day. DEB retries forever.</dd>
* </dl>
*/
Map<String, String> expansionsForDistribution(distributionType) {
String footer = "# Built for ${project.name}-${project.version} " +
"(${distributionType})"
Map<String, Object> expansions = [
'project.name': project.name,
'project.version': version,
'path.conf': [
'tar': '$ES_HOME/config',
'zip': '$ES_HOME/config',
'def': '/etc/elasticsearch',
],
'path.env': [
'deb': '/etc/default/elasticsearch',
'rpm': '/etc/sysconfig/elasticsearch',
/* There isn't one of these files for tar or zip but its important to
make an empty string here so the script can properly skip it. */
'def': '',
],
'heap.min': '256m',
'heap.max': '1g',
'stopping.timeout': [
'rpm': 86400,
],
'scripts.footer': [
/* Debian needs exit 0 on these scripts so we add it here and preserve
the pretty footer. */
'deb': "exit 0\n${footer}",
'def': footer
],
]
Map<String, String> result = [:]
expansions = expansions.each { key, value ->
if (value instanceof Map) {
// 'def' is for default but its three characters like 'rpm' and 'deb'
value = value[distributionType] ?: value['def']
if (value == null) {
return
}
}
result[key] = value
}
return result
}

14
distribution/deb/build.gradle
View File

@ -18,7 +18,19 @@
*/
task buildDeb(type: Deb) {
dependsOn dependencyFiles
dependsOn dependencyFiles, preparePackagingFiles
// Follow elasticsearch's deb file naming convention
archiveName "${packageName}-${project.version}.deb"
packageGroup 'web'
requires 'libc6'
requires 'adduser'
into('/usr/share/lintian/overrides') {
from("${project.packagingFiles}/lintian/elasticsearch")
}
into('/usr/share/doc/elasticsearch') {
from "${project.packagingFiles}/copyright"
}
}
artifacts {

2
distribution/deb/src/main/packaging/init.d/elasticsearch
View File

@ -81,7 +81,7 @@ MAX_MAP_COUNT=262144
#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log
# Elasticsearch PID file directory
PID_DIR="${packaging.elasticsearch.pid.dir}"
PID_DIR="/var/run/elasticsearch"
# End of variables that can be overwritten in $DEFAULT

15
distribution/deb/src/main/packaging/packaging.properties
View File

@ -1,15 +0,0 @@
# Properties used to build to the DEB package
#
# Environment file
packaging.env.file=/etc/default/elasticsearch
# Default configuration directory and file to use in bin/plugin script
packaging.plugin.default.config.dir=${packaging.elasticsearch.conf.dir}
# Simple marker to check that properties are correctly overridden
packaging.type=deb
# Custom header for package scripts
packaging.scripts.header=#!/bin/sh${line.separator}set -e
packaging.scripts.footer=exit 0${line.separator}# Built for ${project.name}-${project.version} (${packaging.type})

5
distribution/deb/src/main/packaging/scripts/conffiles
View File

@ -1,5 +0,0 @@
${packaging.env.file}
${packaging.elasticsearch.conf.dir}/elasticsearch.yml
${packaging.elasticsearch.conf.dir}/logging.yml
/etc/init.d/elasticsearch
/usr/lib/systemd/system/elasticsearch.service

9
distribution/deb/src/main/packaging/scripts/control
View File

@ -1,9 +0,0 @@
Package: elasticsearch
Version: [[version]]
Architecture: all
Maintainer: Elasticsearch Team <info@elastic.co>
Depends: libc6, adduser
Section: web
Priority: optional
Homepage: https://www.elastic.co/
Description: Elasticsearch is a distributed RESTful search engine built for the cloud. Reference documentation can be found at https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html and the 'Elasticsearch: The Definitive Guide' book can be found at https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html

11
distribution/rpm/build.gradle
View File

@ -18,7 +18,16 @@
*/
task buildRpm(type: Rpm) {
dependsOn dependencyFiles
dependsOn dependencyFiles, preparePackagingFiles
// Follow elasticsearch's rpm file naming convention
archiveName = "${packageName}-${project.version}.rpm"
packageGroup 'Application/Internet'
prefix '/usr'
packager 'Elasticsearch'
release '1'
arch NOARCH
os LINUX
// TODO ospackage doesn't support icon but we used to have one
}
artifacts {

26
distribution/rpm/src/main/packaging/init.d/elasticsearch
View File

@ -3,7 +3,7 @@
# elasticsearch <summary>
#
# chkconfig: 2345 80 20
# description: Starts and stops a single elasticsearch instance on this system
# description: Starts and stops a single elasticsearch instance on this system
#
### BEGIN INIT INFO
@ -32,19 +32,19 @@ if [ -f /etc/rc.d/init.d/functions ]; then
fi
# Sets the default values for elasticsearch variables used in this script
ES_USER="${packaging.elasticsearch.user}"
ES_GROUP="${packaging.elasticsearch.group}"
ES_HOME="${packaging.elasticsearch.home.dir}"
MAX_OPEN_FILES=${packaging.os.max.open.files}
MAX_MAP_COUNT=${packaging.os.max.map.count}
LOG_DIR="${packaging.elasticsearch.log.dir}"
DATA_DIR="${packaging.elasticsearch.data.dir}"
CONF_DIR="${packaging.elasticsearch.conf.dir}"
ES_USER="elasticsearch"
ES_GROUP="elasticsearch"
ES_HOME="/usr/share/elasticsearch"
MAX_OPEN_FILES=65535
MAX_MAP_COUNT=262144
LOG_DIR="/var/log/elasticsearch"
DATA_DIR="/var/lib/elasticsearch"
CONF_DIR="${path.conf}"
PID_DIR="${packaging.elasticsearch.pid.dir}"
PID_DIR="/var/run/elasticsearch"
# Source the default env file
ES_ENV_FILE="${packaging.env.file}"
ES_ENV_FILE="${path.env}"
if [ -f "$ES_ENV_FILE" ]; then
. "$ES_ENV_FILE"
fi
@ -70,7 +70,7 @@ export JAVA_HOME
lockfile=/var/lock/subsys/$prog
# backwards compatibility for old config sysconfig files, pre 0.90.1
if [ -n $USER ] && [ -z $ES_USER ] ; then
if [ -n $USER ] && [ -z $ES_USER ] ; then
ES_USER=$USER
fi
@ -126,7 +126,7 @@ start() {
stop() {
echo -n $"Stopping $prog: "
# stop it here, often "killproc $prog"
killproc -p $pidfile -d ${packaging.elasticsearch.stopping.timeout} $prog
killproc -p $pidfile -d ${stopping.timeout} $prog
retval=$?
echo
[ $retval -eq 0 ] && rm -f $lockfile

18
distribution/rpm/src/main/packaging/packaging.properties
View File

@ -1,18 +0,0 @@
# Properties used to build to the RPM package
#
# Environment file
packaging.env.file=/etc/sysconfig/elasticsearch
# Default configuration directory and file to use in bin/plugin script
packaging.plugin.default.config.dir=${packaging.elasticsearch.conf.dir}
# Simple marker to check that properties are correctly overridden
packaging.type=rpm
# Custom header for package scripts
packaging.scripts.header=
packaging.scripts.footer=# Built for ${project.name}-${project.version} (${packaging.type})
# Maximum time to wait for elasticsearch to stop (default to 1 day)
packaging.elasticsearch.stopping.timeout=86400

34
distribution/src/main/packaging/env/elasticsearch vendored
View File

@ -3,21 +3,21 @@
################################
# Elasticsearch home directory
#ES_HOME=${packaging.elasticsearch.home.dir}
#ES_HOME=/usr/share/elasticsearch
# Elasticsearch configuration directory
#CONF_DIR=${packaging.elasticsearch.conf.dir}
#CONF_DIR=${path.conf}
# Elasticsearch data directory
#DATA_DIR=${packaging.elasticsearch.data.dir}
#DATA_DIR=/var/lib/elasticsearch
# Elasticsearch logs directory
#LOG_DIR=${packaging.elasticsearch.log.dir}
#LOG_DIR=/var/log/elasticsearch
# Elasticsearch PID directory
#PID_DIR=${packaging.elasticsearch.pid.dir}
#PID_DIR=/var/run/elasticsearch
# Heap size defaults to ${packaging.elasticsearch.heap.min} min, ${packaging.elasticsearch.heap.max} max
# Heap size defaults to ${heap.min} min, ${heap.max} max
# Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g
#ES_HEAP_SIZE=2g
@ -34,7 +34,7 @@
#ES_RESTART_ON_UPGRADE=true
# Path to the GC log file
#ES_GC_LOG_FILE=${packaging.elasticsearch.log.dir}/gc.log
#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log
################################
# Elasticsearch service
@ -43,15 +43,15 @@
# SysV init.d
#
# When executing the init script, this user will be used to run the elasticsearch service.
# The default value is '${packaging.elasticsearch.user}' and is declared in the init.d file.
# The default value is 'elasticsearch' and is declared in the init.d file.
# Note that this setting is only used by the init script. If changed, make sure that
# the configured user can read and write into the data, work, plugins and log directories.
# For systemd service, the user is usually configured in file ${packaging.elasticsearch.systemd.dir}/elasticsearch.service
#ES_USER=${packaging.elasticsearch.user}
#ES_GROUP=${packaging.elasticsearch.group}
# For systemd service, the user is usually configured in file /usr/lib/systemd/system/elasticsearch.service
#ES_USER=elasticsearch
#ES_GROUP=elasticsearch
# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process
ES_STARTUP_SLEEP_TIME=${packaging.elasticsearch.startup.sleep.time}
ES_STARTUP_SLEEP_TIME=5
################################
# System properties
@ -59,17 +59,17 @@ ES_STARTUP_SLEEP_TIME=${packaging.elasticsearch.startup.sleep.time}
# Specifies the maximum file descriptor number that can be opened by this process
# When using Systemd, this setting is ignored and the LimitNOFILE defined in
# ${packaging.elasticsearch.systemd.dir}/elasticsearch.service takes precedence
#MAX_OPEN_FILES=${packaging.os.max.open.files}
# /usr/lib/systemd/system/elasticsearch.service takes precedence
#MAX_OPEN_FILES=65535
# The maximum number of bytes of memory that may be locked into RAM
# Set to "unlimited" if you use the 'bootstrap.mlockall: true' option
# in elasticsearch.yml (ES_HEAP_SIZE must also be set).
# When using Systemd, the LimitMEMLOCK property must be set
# in ${packaging.elasticsearch.systemd.dir}/elasticsearch.service
# in /usr/lib/systemd/system/elasticsearch.service
#MAX_LOCKED_MEMORY=unlimited
# Maximum number of VMA (Virtual Memory Areas) a process can own
# When using Systemd, this setting is ignored and the 'vm.max_map_count'
# property is set at boot time in ${packaging.elasticsearch.systemd.sysctl.dir}/elasticsearch.conf
#MAX_MAP_COUNT=${packaging.os.max.map.count}
# property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf
#MAX_MAP_COUNT=262144

30
distribution/src/main/packaging/packaging.properties
View File

@ -1,30 +0,0 @@
# Common properties for building ZIP,GZ,RPM and DEB packages
#
# Properties defined here can be overridden with specific settings,
# like in rpm/packaging.properties and deb/packaging.properties.
# Environment file
packaging.env.file=
# Default configuration directory and file to use in bin/plugin script
packaging.plugin.default.config.dir=$ES_HOME/config
# Default values for min/max heap memory allocated to elasticsearch java process
packaging.elasticsearch.heap.min=256m
packaging.elasticsearch.heap.max=1g
# Specifies the maximum file descriptor number
packaging.os.max.open.files=65535
# Maximum number of VMA (Virtual Memory Areas) a process can own
packaging.os.max.map.count=262144
# Default number of seconds to wait before checking if Elasticsearch started successfully as a daemon process
packaging.elasticsearch.startup.sleep.time=5
# Simple marker to check that properties are correctly overridden
packaging.type=tar.gz
# Custom header for package scripts
packaging.scripts.header=
packaging.scripts.footer=

10
distribution/src/main/packaging/scripts/postinst
View File

@ -1,5 +1,3 @@
${packaging.scripts.header}
#
# This script is executed in the post-installation phase
#
@ -13,11 +11,11 @@ ${packaging.scripts.header}
# Sets the default values for elasticsearch variables used in this script
ES_USER="${packaging.elasticsearch.user}"
ES_GROUP="${packaging.elasticsearch.group}"
ES_USER="elasticsearch"
ES_GROUP="elasticsearch"
# Source the default env file
ES_ENV_FILE="${packaging.env.file}"
ES_ENV_FILE="${path.env}"
if [ -f "$ES_ENV_FILE" ]; then
. "$ES_ENV_FILE"
fi
@ -98,4 +96,4 @@ elif [ "$RESTART_ON_UPGRADE" = "true" ]; then
echo " OK"
fi
${packaging.scripts.footer}
${scripts.footer}

18
distribution/src/main/packaging/scripts/postrm
View File

@ -1,5 +1,3 @@
${packaging.scripts.header}
#
# This script is executed in the post-removal phase
#
@ -51,16 +49,16 @@ case "$1" in
esac
# Sets the default values for elasticsearch variables used in this script
ES_USER="${packaging.elasticsearch.user}"
ES_GROUP="${packaging.elasticsearch.group}"
LOG_DIR="${packaging.elasticsearch.log.dir}"
PLUGINS_DIR="${packaging.elasticsearch.plugins.dir}"
PID_DIR="${packaging.elasticsearch.pid.dir}"
DATA_DIR="${packaging.elasticsearch.data.dir}"
ES_USER="elasticsearch"
ES_GROUP="elasticsearch"
LOG_DIR="/var/log/elasticsearch"
PLUGINS_DIR="/usr/share/elasticsearch/plugins"
PID_DIR="/var/run/elasticsearch"
DATA_DIR="/var/lib/elasticsearch"
# Source the default env file
if [ "$SOURCE_ENV_FILE" = "true" ]; then
ES_ENV_FILE="${packaging.env.file}"
ES_ENV_FILE="${path.env}"
if [ -f "$ES_ENV_FILE" ]; then
. "$ES_ENV_FILE"
fi
@ -116,4 +114,4 @@ if [ "$REMOVE_USER_AND_GROUP" = "true" ]; then
fi
fi
${packaging.scripts.footer}
${scripts.footer}

10
distribution/src/main/packaging/scripts/preinst
View File

@ -1,5 +1,3 @@
${packaging.scripts.header}
#
# This script is executed in the pre-installation phase
#
@ -14,11 +12,11 @@ ${packaging.scripts.header}
# Sets the default values for elasticsearch variables used in this script
ES_USER="${packaging.elasticsearch.user}"
ES_GROUP="${packaging.elasticsearch.group}"
ES_USER="elasticsearch"
ES_GROUP="elasticsearch"
# Source the default env file
ES_ENV_FILE="${packaging.env.file}"
ES_ENV_FILE="${path.env}"
if [ -f "$ES_ENV_FILE" ]; then
. "$ES_ENV_FILE"
fi
@ -80,4 +78,4 @@ case "$1" in
;;
esac
${packaging.scripts.footer}
${scripts.footer}

4
distribution/src/main/packaging/scripts/prerm
View File

@ -1,5 +1,3 @@
${packaging.scripts.header}
#
# This script is executed in the pre-remove phase
#
@ -66,4 +64,4 @@ if [ "$STOP_REQUIRED" = "true" ]; then
echo " OK"
fi
${packaging.scripts.footer}
${scripts.footer}

2
distribution/src/main/packaging/systemd/elasticsearch.conf
View File

@ -1 +1 @@
d ${packaging.elasticsearch.pid.dir} 0755 ${packaging.elasticsearch.user} ${packaging.elasticsearch.group} - -
d /var/run/elasticsearch 0755 elasticsearch elasticsearch - -

28
distribution/src/main/packaging/systemd/elasticsearch.service
View File

@ -5,21 +5,21 @@ Wants=network-online.target
After=network-online.target
[Service]
Environment=ES_HOME=${packaging.elasticsearch.home.dir}
Environment=CONF_DIR=${packaging.elasticsearch.conf.dir}
Environment=DATA_DIR=${packaging.elasticsearch.data.dir}
Environment=LOG_DIR=${packaging.elasticsearch.log.dir}
Environment=PID_DIR=${packaging.elasticsearch.pid.dir}
EnvironmentFile=-${packaging.env.file}
Environment=ES_HOME=/usr/share/elasticsearch
Environment=CONF_DIR=${path.conf}
Environment=DATA_DIR=/var/lib/elasticsearch
Environment=LOG_DIR=/var/log/elasticsearch
Environment=PID_DIR=/var/run/elasticsearch
EnvironmentFile=-${path.env}
WorkingDirectory=${packaging.elasticsearch.home.dir}
WorkingDirectory=/usr/share/elasticsearch
User=${packaging.elasticsearch.user}
Group=${packaging.elasticsearch.group}
User=elasticsearch
Group=elasticsearch
ExecStartPre=${packaging.elasticsearch.bin.dir}/elasticsearch-systemd-pre-exec
ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec
ExecStart=${packaging.elasticsearch.bin.dir}/elasticsearch \
ExecStart=/usr/share/elasticsearch/bin/elasticsearch \
-Des.pidfile=${PID_DIR}/elasticsearch.pid \
-Des.default.path.home=${ES_HOME} \
-Des.default.path.logs=${LOG_DIR} \
@ -33,11 +33,11 @@ StandardOutput=null
StandardError=journal
# Specifies the maximum file descriptor number that can be opened by this process
LimitNOFILE=${packaging.os.max.open.files}
LimitNOFILE=65535
# Specifies the maximum number of bytes of memory that may be locked into RAM
# Set to "infinity" if you use the 'bootstrap.mlockall: true' option
# in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in ${packaging.env.file}
# in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in ${path.env}
#LimitMEMLOCK=infinity
# Disable timeout logic and wait until process is stopped
@ -55,4 +55,4 @@ SuccessExitStatus=143
[Install]
WantedBy=multi-user.target
# Built for ${project.name}-${project.version} (${packaging.type})
# Built for ${project.name}-${project.version} (${project.name})

2
distribution/src/main/packaging/systemd/sysctl/elasticsearch.conf
View File

@ -1 +1 @@
vm.max_map_count=${packaging.os.max.map.count}
vm.max_map_count=262144

9
distribution/src/main/resources/bin/elasticsearch
View File

@ -42,12 +42,9 @@
# Be aware that you will be entirely responsible for populating the needed
# environment variables.
# Maven will replace the project.name with elasticsearch below. If that
# hasn't been done, we assume that this is not a packaged version and the
# user has forgotten to run Maven to create a package.
IS_PACKAGED_VERSION='${project.parent.artifactId}'
if [ "$IS_PACKAGED_VERSION" != "distributions" ]; then
# Check to see if you are trying to run this without building it first. Gradle
# will replace the project.name with _something_.
if echo '${project.name}' | grep project.name > /dev/null ; then
cat >&2 << EOF
Error: You must build the project with Maven or download a pre-built package
before you can run Elasticsearch. See 'Building from Source' in README.textile

8
distribution/src/main/resources/bin/elasticsearch.in.bat
View File

@ -4,7 +4,7 @@ if DEFINED JAVA_HOME goto cont
:err
ECHO JAVA_HOME environment variable must be set! 1>&2
EXIT /B 1
EXIT /B 1
:cont
set SCRIPT_DIR=%~dp0
@ -14,11 +14,11 @@ for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
REM ***** JAVA options *****
if "%ES_MIN_MEM%" == "" (
set ES_MIN_MEM=${packaging.elasticsearch.heap.min}
set ES_MIN_MEM=${heap.min}
)
if "%ES_MAX_MEM%" == "" (
set ES_MAX_MEM=${packaging.elasticsearch.heap.max}
set ES_MAX_MEM=${heap.max}
)
if NOT "%ES_HEAP_SIZE%" == "" (
@ -93,7 +93,7 @@ set JAVA_OPTS=%JAVA_OPTS% -Djna.nosys=true
REM check in case a user was using this mechanism
if "%ES_CLASSPATH%" == "" (
set ES_CLASSPATH=%ES_HOME%/lib/${project.build.finalName}.jar;%ES_HOME%/lib/*
set ES_CLASSPATH=%ES_HOME%/lib/elasticsearch-${project.version}.jar;%ES_HOME%/lib/*
) else (
ECHO Error: Don't modify the classpath with ES_CLASSPATH, Best is to add 1>&2
ECHO additional elements via the plugin mechanism, or if code must really be 1>&2

6
distribution/src/main/resources/bin/elasticsearch.in.sh
View File

@ -10,13 +10,13 @@ EOF
exit 1
fi
ES_CLASSPATH="$ES_HOME/lib/${project.build.finalName}.jar:$ES_HOME/lib/*"
ES_CLASSPATH="$ES_HOME/lib/elasticsearch-${project.version}.jar:$ES_HOME/lib/*"
if [ "x$ES_MIN_MEM" = "x" ]; then
ES_MIN_MEM=${packaging.elasticsearch.heap.min}
ES_MIN_MEM=${heap.min}
fi
if [ "x$ES_MAX_MEM" = "x" ]; then
ES_MAX_MEM=${packaging.elasticsearch.heap.max}
ES_MAX_MEM=${heap.max}
fi
if [ "x$ES_HEAP_SIZE" != "x" ]; then
ES_MIN_MEM=$ES_HEAP_SIZE

6
distribution/src/main/resources/bin/plugin
View File

@ -25,12 +25,12 @@ ES_HOME=`cd "$ES_HOME"; pwd`
# Sets the default values for elasticsearch variables used in this script
if [ -z "$CONF_DIR" ]; then
CONF_DIR="${packaging.plugin.default.config.dir}"
CONF_DIR="${path.conf}"
fi
# The default env file is defined at building/packaging time.
# For a ${packaging.type} package, the value is "${packaging.env.file}".
ES_ENV_FILE="${packaging.env.file}"
# For a ${project.name} package, the value is "${path.env}".
ES_ENV_FILE="${path.env}"
# If an include is specified with the ES_INCLUDE environment variable, use it
if [ -n "$ES_INCLUDE" ]; then

4
distribution/src/main/resources/bin/service.bat
View File

@ -128,8 +128,8 @@ goto:eof
)
:foundJVM
if "%ES_MIN_MEM%" == "" set ES_MIN_MEM=${packaging.elasticsearch.heap.min}
if "%ES_MAX_MEM%" == "" set ES_MAX_MEM=${packaging.elasticsearch.heap.max}
if "%ES_MIN_MEM%" == "" set ES_MIN_MEM=${heap.min}
if "%ES_MAX_MEM%" == "" set ES_MAX_MEM=${heap.max}
if NOT "%ES_HEAP_SIZE%" == "" set ES_MIN_MEM=%ES_HEAP_SIZE%
if NOT "%ES_HEAP_SIZE%" == "" set ES_MAX_MEM=%ES_HEAP_SIZE%

13
distribution/tar/src/main/assemblies/targz-bin.xml
View File

@ -1,13 +0,0 @@
<?xml version="1.0"?>
<assembly>
<id>targz</id>
<formats>
<format>tar.gz</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<componentDescriptors>
<componentDescriptor>../src/main/assemblies/common-bin.xml</componentDescriptor>
</componentDescriptors>
</assembly>

13
distribution/zip/src/main/assemblies/zip-bin.xml
View File

@ -1,13 +0,0 @@
<?xml version="1.0"?>
<assembly>
<id>zip</id>
<formats>
<format>zip</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<componentDescriptors>
<componentDescriptor>../src/main/assemblies/common-bin.xml</componentDescriptor>
</componentDescriptors>
</assembly>

99
qa/vagrant/build.gradle
View File

@ -19,30 +19,43 @@
import org.elasticsearch.gradle.vagrant.VagrantCommandTask
import org.elasticsearch.gradle.vagrant.BatsOverVagrantTask
import org.elasticsearch.gradle.FileContentsTask
import org.gradle.api.InvalidUserDataException
def testScripts = '2*.bats' // Once the rpm and deb are build this should be *.bats
def testCommand = "cd \$TESTROOT && sudo bats --tap \$BATS/$testScripts"
def smokeTestCommand = 'echo I work'
def representativeBoxes = ['ubuntu-1404', 'centos-7']
def boxes = representativeBoxes + ['ubuntu-1204', 'ubuntu-1504', 'debian-8',
'centos-6', 'oel-7', 'fedora-22', 'opensuse-13', 'sles-12']
String testScripts = '*.bats'
String testCommand = "cd \$TESTROOT && sudo bats --tap \$BATS/$testScripts"
String smokeTestCommand = 'echo I work'
List<String> representativeBoxes = ['ubuntu-1404', 'centos-7']
List<String> boxes = representativeBoxes + ['ubuntu-1204', 'ubuntu-1504',
'debian-8', 'centos-6', 'oel-7', 'fedora-22', 'opensuse-13', 'sles-12']
/* The version of elasticsearch that we upgrade *from* as part of testing
* upgrades. */
String upgradeFromVersion = '2.0.0'
configurations {
test
}
repositories {
mavenCentral()
}
dependencies {
test project("${projectsPrefix}:distribution:tar")
// NOCOMMMIT: we need the rpm and deb!
// test project(path: ':distribution:rpm', configuration: 'archives')
// test project(path: ':distribution:deb', configuration: 'archives')
test project(path: "${projectsPrefix}:distribution:tar", configuration: 'archives')
test project(path: "${projectsPrefix}:distribution:rpm", configuration: 'archives')
test project(path: "${projectsPrefix}:distribution:deb", configuration: 'archives')
// Collect all the plugins
for (Project subproj : project.rootProject.subprojects) {
if (subproj.path.startsWith(':plugins:')) {
test project("${projectsPrefix}${subproj.path}")
}
}
// The version of elasticsearch that we upgrade *from*
test "org.elasticsearch.distribution.deb:elasticsearch:$upgradeFromVersion@deb"
test "org.elasticsearch.distribution.rpm:elasticsearch:$upgradeFromVersion@rpm"
}
task checkPackages {
@ -58,14 +71,36 @@ task checkPackagesAllDistros {
}
task clean(type: Delete) {
group = 'Build'
group 'Build'
delete buildDir
}
task copyDepsToTestRoot(type: Copy) {
description "Dump bats test dependencies into the \$TESTROOT."
into "$buildDir/testroot"
File testRoot = new File("$buildDir/testroot")
task createTestRoot {
outputs.dir testRoot
doLast {
testRoot.mkdirs()
}
}
task createVersionFile(type: FileContentsTask) {
dependsOn createTestRoot
file "${testRoot}/version"
contents = version
}
task createUpgradeFromFile(type: FileContentsTask) {
dependsOn createTestRoot
file "${testRoot}/upgrade_from_version"
contents = upgradeFromVersion
}
task prepareTestRoot(type: Copy) {
description 'Dump bats test dependencies into the $TESTROOT'
into testRoot
from configurations.test
dependsOn createVersionFile, createUpgradeFromFile
}
task checkVagrantVersion(type: Exec) {
@ -74,10 +109,10 @@ task checkVagrantVersion(type: Exec) {
commandLine 'vagrant', '--version'
standardOutput = new ByteArrayOutputStream()
doLast {
def version = standardOutput.toString().trim()
String version = standardOutput.toString().trim()
if ((version ==~ /Vagrant 1\.[789]\..+/) == false) {
throw new InvalidUserDataException('Illegal version of vagrant [' +
version + ']. Need [Vagrant 1.7+]')
throw new InvalidUserDataException(
"Illegal version of vagrant [${version}]. Need [Vagrant 1.7+]")
}
}
}
@ -94,10 +129,10 @@ task vagrantSmokeTestAllDistros {
// Each box gets it own set of tasks
boxes.each { box ->
def boxTask = taskifyBoxName box
task "vagrantUp$boxTask"(type: VagrantCommandTask) {
String boxTask = taskifyBoxName box
task "vagrantUp${boxTask}"(type: VagrantCommandTask) {
group 'Vagrant'
description "Startup a vagrant VM running $box"
description "Startup a vagrant VM running ${box}"
boxName box
/* Its important that we try to reprovision the box even if it already
exists. That way updates to the vagrant configuration take automatically.
@ -114,34 +149,34 @@ boxes.each { box ->
SKIPPED but that would require running vagrant status which is slow! */
dependsOn checkVagrantVersion
}
task "vagrantHalt$boxTask"(type: VagrantCommandTask) {
task "vagrantHalt${boxTask}"(type: VagrantCommandTask) {
group 'Vagrant'
description "Shutdown the vagrant VM running $box"
boxName box
commandLine 'halt', box
}
task "smokeTest$boxTask"(type: Exec) {
task "smokeTest${boxTask}"(type: Exec) {
group 'Vagrant'
description "Smoke test the $box VM"
dependsOn "vagrantUp$boxTask"
finalizedBy "vagrantHalt$boxTask"
description "Smoke test the ${box} VM"
dependsOn "vagrantUp${boxTask}"
finalizedBy "vagrantHalt${boxTask}"
commandLine 'vagrant', 'ssh', box, '--command',
"set -o pipefail && $smokeTestCommand | sed -ue \'s/^/ $box: /'"
"set -o pipefail && ${smokeTestCommand} | sed -ue \'s/^/ ${box}: /'"
vagrantSmokeTestAllDistros.dependsOn name
if (representativeBoxes.contains(box)) {
vagrantSmokeTest.dependsOn name
}
}
task "check$boxTask"(type: BatsOverVagrantTask) {
task "check${boxTask}"(type: BatsOverVagrantTask) {
group 'Package Verification'
description "Run packaging tests against $box"
dependsOn "vagrantUp$boxTask"
finalizedBy "vagrantHalt$boxTask"
description "Run packaging tests against ${box}"
dependsOn "vagrantUp${boxTask}"
finalizedBy "vagrantHalt${boxTask}"
boxName box
command testCommand
dependsOn copyDepsToTestRoot
dependsOn prepareTestRoot
checkPackagesAllDistros.dependsOn name
if (representativeBoxes.contains(box)) {
checkPackages.dependsOn name
@ -151,6 +186,6 @@ boxes.each { box ->
// Twists the box name into a sensible task name
def taskifyBoxName(box) {
String taskifyBoxName(box) {
box.capitalize().replace('-', '')
}

9
qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash
View File

@ -239,6 +239,10 @@ fi
install_and_check_plugin lang python jython-standalone-*.jar
}
@test "[$GROUP] install mapper-attachments plugin" {
install_and_check_plugin mapper attachments
}
@test "[$GROUP] install murmur3 mapper plugin" {
install_and_check_plugin mapper murmur3
}
@ -343,6 +347,10 @@ fi
remove_plugin lang-python
}
@test "[$GROUP] remove mapper-attachments plugin" {
remove_plugin mapper-attachments
}
@test "[$GROUP] remove murmur3 mapper plugin" {
remove_plugin mapper-murmur3
}
@ -423,4 +431,3 @@ fi
fi
remove_jvm_example
}