remove option to configure custom config file via CONF_FILE or -Des.default.conf

It is rarely used and was not consistently handled by different distributions anyway.
This commit also adds a test for specifying CONF_DIR when installing plugins and
starting elasticsearch.

relates to #12712 and #12954
closes #5329
closes #13715
This commit is contained in:
Britta Weber 2015-09-24 17:39:00 +02:00
parent aa19a4134d
commit 71aefd5a06
19 changed files with 163 additions and 116 deletions

View File

@ -26,7 +26,6 @@ import org.elasticsearch.common.PidFile;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.cli.CliTool;
import org.elasticsearch.common.cli.Terminal;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.CreationException;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.logging.ESLogger;
@ -249,13 +248,13 @@ final class Bootstrap {
Environment environment = initialSettings(foreground);
Settings settings = environment.settings();
setupLogging(settings, environment);
checkForCustomConfFile();
if (environment.pidFile() != null) {
PidFile.create(environment.pidFile(), true);
}
setupLogging(settings, environment);
if (System.getProperty("es.max-open-files", "false").equals("true")) {
ESLogger logger = Loggers.getLogger(Bootstrap.class);
logger.info("max_open_files [{}]", ProcessProbe.getInstance().getMaxFileDescriptorCount());
@ -330,4 +329,21 @@ final class Bootstrap {
System.err.flush();
}
}
private static void checkForCustomConfFile() {
String confFileSetting = System.getProperty("es.default.config");
checkUnsetAndMaybeExit(confFileSetting, "es.default.config");
confFileSetting = System.getProperty("es.config");
checkUnsetAndMaybeExit(confFileSetting, "es.config");
confFileSetting = System.getProperty("elasticsearch.config");
checkUnsetAndMaybeExit(confFileSetting, "elasticsearch.config");
}
private static void checkUnsetAndMaybeExit(String confFileSetting, String settingName) {
if (confFileSetting != null && confFileSetting.isEmpty() == false) {
ESLogger logger = Loggers.getLogger(Bootstrap.class);
logger.info("{} is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed.", settingName);
System.exit(1);
}
}
}

View File

@ -83,42 +83,20 @@ public class InternalSettingsPreparer {
initializeSettings(output, input, true);
Environment environment = new Environment(output.build());
// TODO: can we simplify all of this and have a single filename, which is looked up in the config dir?
boolean loadFromEnv = true;
if (useSystemProperties(input)) {
// if its default, then load it, but also load form env
if (Strings.hasText(System.getProperty("es.default.config"))) {
// TODO: we don't allow multiple config files, but having loadFromEnv true here allows just that
loadFromEnv = true;
output.loadFromPath(environment.configFile().resolve(System.getProperty("es.default.config")));
}
// TODO: these should be elseifs so that multiple files cannot be loaded
// if explicit, just load it and don't load from env
if (Strings.hasText(System.getProperty("es.config"))) {
loadFromEnv = false;
output.loadFromPath(environment.configFile().resolve(System.getProperty("es.config")));
}
if (Strings.hasText(System.getProperty("elasticsearch.config"))) {
loadFromEnv = false;
output.loadFromPath(environment.configFile().resolve(System.getProperty("elasticsearch.config")));
boolean settingsFileFound = false;
Set<String> foundSuffixes = new HashSet<>();
for (String allowedSuffix : ALLOWED_SUFFIXES) {
Path path = environment.configFile().resolve("elasticsearch" + allowedSuffix);
if (Files.exists(path)) {
if (!settingsFileFound) {
output.loadFromPath(path);
}
settingsFileFound = true;
foundSuffixes.add(allowedSuffix);
}
}
if (loadFromEnv) {
boolean settingsFileFound = false;
Set<String> foundSuffixes = new HashSet<>();
for (String allowedSuffix : ALLOWED_SUFFIXES) {
Path path = environment.configFile().resolve("elasticsearch" + allowedSuffix);
if (Files.exists(path)) {
if (!settingsFileFound) {
output.loadFromPath(path);
}
settingsFileFound = true;
foundSuffixes.add(allowedSuffix);
}
}
if (foundSuffixes.size() > 1) {
throw new SettingsException("multiple settings files found with suffixes: " + Strings.collectionToDelimitedString(foundSuffixes, ","));
}
if (foundSuffixes.size() > 1) {
throw new SettingsException("multiple settings files found with suffixes: " + Strings.collectionToDelimitedString(foundSuffixes, ","));
}
// re-initialize settings now that the config file has been loaded

View File

@ -76,6 +76,7 @@
<include>bin/elasticsearch</include>
<include>bin/elasticsearch.in.sh</include>
<include>bin/plugin</include>
<include>bin/elasticsearch-systemd-pre-exec</include>
</includes>
</resource>
</resources>
@ -110,7 +111,7 @@
<data>
<src>${project.build.directory}/generated-packaging/deb/bin</src>
<type>directory</type>
<includes>elasticsearch,elasticsearch.in.sh,plugin</includes>
<includes>elasticsearch,elasticsearch.in.sh,plugin,elasticsearch-systemd-pre-exec</includes>
<mapper>
<type>perm</type>
<prefix>${packaging.elasticsearch.bin.dir}</prefix>

View File

@ -74,9 +74,6 @@ DATA_DIR=/var/lib/$NAME
# Elasticsearch configuration directory
CONF_DIR=/etc/$NAME
# Elasticsearch configuration file (elasticsearch.yml)
CONF_FILE=$CONF_DIR/elasticsearch.yml
# Maximum number of VMA (Virtual Memory Areas) a process can own
MAX_MAP_COUNT=262144
@ -93,10 +90,16 @@ if [ -f "$DEFAULT" ]; then
. "$DEFAULT"
fi
# CONF_FILE setting was removed
if [ ! -z "$CONF_FILE" ]; then
echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
exit 1
fi
# Define other required variables
PID_FILE="$PID_DIR/$NAME.pid"
DAEMON=$ES_HOME/bin/elasticsearch
DAEMON_OPTS="-d -p $PID_FILE --default.config=$CONF_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR"
DAEMON_OPTS="-d -p $PID_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR"
export ES_HEAP_SIZE
export ES_HEAP_NEWSIZE

View File

@ -6,7 +6,6 @@ packaging.env.file=/etc/default/elasticsearch
# Default configuration directory and file to use in bin/plugin script
packaging.plugin.default.config.dir=${packaging.elasticsearch.conf.dir}
packaging.plugin.default.config.file=${packaging.elasticsearch.conf.dir}/elasticsearch.yml
# Simple marker to check that properties are correctly overridden
packaging.type=deb

View File

@ -79,6 +79,7 @@
<include>bin/elasticsearch</include>
<include>bin/elasticsearch.in.sh</include>
<include>bin/plugin</include>
<include>bin/elasticsearch-systemd-pre-exec</include>
</includes>
</resource>
</resources>
@ -127,6 +128,7 @@
<include>elasticsearch</include>
<include>elasticsearch.in.sh</include>
<include>plugin</include>
<include>elasticsearch-systemd-pre-exec</include>
</includes>
</source>
</sources>

View File

@ -40,7 +40,7 @@ MAX_MAP_COUNT=${packaging.os.max.map.count}
LOG_DIR="${packaging.elasticsearch.log.dir}"
DATA_DIR="${packaging.elasticsearch.data.dir}"
CONF_DIR="${packaging.elasticsearch.conf.dir}"
CONF_FILE="${packaging.elasticsearch.conf.dir}/elasticsearch.yml"
PID_DIR="${packaging.elasticsearch.pid.dir}"
# Source the default env file
@ -49,6 +49,12 @@ if [ -f "$ES_ENV_FILE" ]; then
. "$ES_ENV_FILE"
fi
# CONF_FILE setting was removed
if [ ! -z "$CONF_FILE" ]; then
echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
exit 1
fi
exec="$ES_HOME/bin/elasticsearch"
prog="elasticsearch"
pidfile="$PID_DIR/${prog}.pid"
@ -83,7 +89,6 @@ checkJava() {
start() {
checkJava
[ -x $exec ] || exit 5
[ -f $CONF_FILE ] || exit 6
if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then
echo "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set"
return 7

View File

@ -6,7 +6,6 @@ packaging.env.file=/etc/sysconfig/elasticsearch
# Default configuration directory and file to use in bin/plugin script
packaging.plugin.default.config.dir=${packaging.elasticsearch.conf.dir}
packaging.plugin.default.config.file=${packaging.elasticsearch.conf.dir}/elasticsearch.yml
# Simple marker to check that properties are correctly overridden
packaging.type=rpm

View File

@ -8,9 +8,6 @@
# Elasticsearch configuration directory
#CONF_DIR=${packaging.elasticsearch.conf.dir}
# Elasticsearch configuration file
#CONF_FILE=$CONF_DIR/elasticsearch.yml
# Elasticsearch data directory
#DATA_DIR=${packaging.elasticsearch.data.dir}

View File

@ -8,7 +8,6 @@ packaging.env.file=
# Default configuration directory and file to use in bin/plugin script
packaging.plugin.default.config.dir=$ES_HOME/config
packaging.plugin.default.config.file=$ES_HOME/config/elasticsearch.yml
# Default values for min/max heap memory allocated to elasticsearch java process
packaging.elasticsearch.heap.min=256m

View File

@ -7,7 +7,6 @@ After=network-online.target
[Service]
Environment=ES_HOME=${packaging.elasticsearch.home.dir}
Environment=CONF_DIR=${packaging.elasticsearch.conf.dir}
Environment=CONF_FILE=${packaging.elasticsearch.conf.dir}/elasticsearch.yml
Environment=DATA_DIR=${packaging.elasticsearch.data.dir}
Environment=LOG_DIR=${packaging.elasticsearch.log.dir}
Environment=PID_DIR=${packaging.elasticsearch.pid.dir}
@ -18,12 +17,13 @@ WorkingDirectory=${packaging.elasticsearch.home.dir}
User=${packaging.elasticsearch.user}
Group=${packaging.elasticsearch.group}
ExecStartPre=${packaging.elasticsearch.bin.dir}/elasticsearch-systemd-pre-exec
ExecStart=${packaging.elasticsearch.bin.dir}/elasticsearch \
-Des.pidfile=${PID_DIR}/elasticsearch.pid \
-Des.default.path.home=${ES_HOME} \
-Des.default.path.logs=${LOG_DIR} \
-Des.default.path.data=${DATA_DIR} \
-Des.default.config=${CONF_FILE} \
-Des.default.path.conf=${CONF_DIR}
# Connects standard output to /dev/null

View File

@ -42,10 +42,10 @@
# Be aware that you will be entirely responsible for populating the needed
# environment variables.
# Maven will replace the project.name with elasticsearch below. If that
# hasn't been done, we assume that this is not a packaged version and the
# user has forgotten to run Maven to create a package.
IS_PACKAGED_VERSION='${project.parent.artifactId}'
if [ "$IS_PACKAGED_VERSION" != "distributions" ]; then
cat >&2 << EOF

View File

@ -0,0 +1,7 @@
#!/bin/sh
# CONF_FILE setting was removed
if [ ! -z "$CONF_FILE" ]; then
echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
exit 1
fi

View File

@ -1,5 +1,6 @@
#!/bin/sh
CDPATH=""
SCRIPT="$0"
@ -21,17 +22,10 @@ ES_HOME=`dirname "$SCRIPT"`/..
# make ELASTICSEARCH_HOME absolute
ES_HOME=`cd "$ES_HOME"; pwd`
# Sets the default values for elasticsearch variables used in this script
if [ -z "$CONF_DIR" ]; then
CONF_DIR="${packaging.plugin.default.config.dir}"
if [ -z "$CONF_FILE" ]; then
CONF_FILE="$CONF_DIR/elasticsearch.yml"
fi
fi
if [ -z "$CONF_FILE" ]; then
CONF_FILE="${packaging.plugin.default.config.file}"
fi
# The default env file is defined at building/packaging time.
@ -66,6 +60,12 @@ if [ "x$JAVA_TOOL_OPTIONS" != "x" ]; then
unset JAVA_TOOL_OPTIONS
fi
# CONF_FILE setting was removed
if [ ! -z "$CONF_FILE" ]; then
echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
exit 1
fi
if [ -x "$JAVA_HOME/bin/java" ]; then
JAVA=$JAVA_HOME/bin/java
else
@ -105,16 +105,6 @@ if [ -e "$CONF_DIR" ]; then
esac
fi
if [ -e "$CONF_FILE" ]; then
case "$properties" in
*-Des.default.config=*|*-Des.config=*)
;;
*)
properties="$properties -Des.default.config=\"$CONF_FILE\""
;;
esac
fi
# full hostname passed through cut for portability on systems that do not support hostname -s
# export on separate line for shells that do not support combining definition and export
HOSTNAME=`hostname | cut -d. -f1`

View File

@ -5,6 +5,8 @@ TITLE Elasticsearch Service ${project.version}
if NOT DEFINED JAVA_HOME goto err
if not "%CONF_FILE%" == "" goto conffileset
set SCRIPT_DIR=%~dp0
for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
@ -147,9 +149,7 @@ if "%DATA_DIR%" == "" set DATA_DIR=%ES_HOME%\data
if "%CONF_DIR%" == "" set CONF_DIR=%ES_HOME%\config
if "%CONF_FILE%" == "" set CONF_FILE=%ES_HOME%\config\elasticsearch.yml
set ES_PARAMS=-Delasticsearch;-Des.path.home="%ES_HOME%";-Des.default.config="%CONF_FILE%";-Des.default.path.home="%ES_HOME%";-Des.default.path.logs="%LOG_DIR%";-Des.default.path.data="%DATA_DIR%";-Des.default.path.conf="%CONF_DIR%"
set ES_PARAMS=-Delasticsearch;-Des.path.home="%ES_HOME%";-Des.default.path.home="%ES_HOME%";-Des.default.path.logs="%LOG_DIR%";-Des.default.path.data="%DATA_DIR%";-Des.default.path.conf="%CONF_DIR%"
set JVM_OPTS=%JAVA_OPTS: =;%
@ -207,4 +207,8 @@ set /a conv=%conv% * 1024
set "%~2=%conv%"
goto:eof
:conffileset
echo CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed.
goto:eof
ENDLOCAL

View File

@ -22,7 +22,6 @@ Each package features a configuration file, which allows you to set the followin
`LOG_DIR`:: Log directory, defaults to `/var/log/elasticsearch`
`DATA_DIR`:: Data directory, defaults to `/var/lib/elasticsearch`
`CONF_DIR`:: Configuration file directory (which needs to include `elasticsearch.yml` and `logging.yml` files), defaults to `/etc/elasticsearch`
`CONF_FILE`:: Path to configuration file, defaults to `/etc/elasticsearch/elasticsearch.yml`
`ES_JAVA_OPTS`:: Any additional java options you may want to apply. This may be useful, if you need to set the `node.name` property, but do not want to change the `elasticsearch.yml` configuration file, because it is distributed via a provisioning system like puppet or chef. Example: `ES_JAVA_OPTS="-Des.node.name=search-01"`
`RESTART_ON_UPGRADE`:: Configure restart on package upgrade, defaults to `false`. This means you will have to restart your elasticsearch instance after installing a package manually. The reason for this is to ensure, that upgrades in a cluster do not result in a continuous shard reallocation resulting in high network traffic and reducing the response times of your cluster.
`ES_GC_LOG_FILE` :: The absolute log file path for creating a garbage collection logfile, which is done by the JVM. Note that this logfile can grow pretty quick and thus is disabled by default.
@ -72,9 +71,9 @@ sudo service elasticsearch start
[float]
===== Using systemd
==== Using systemd
Distributions like SUSE do not use the `chkconfig` tool to register services, but rather `systemd` and its command `/bin/systemctl` to start and stop services (at least in newer versions, otherwise use the `chkconfig` commands above). The configuration file is also placed at `/etc/sysconfig/elasticsearch`. After installing the RPM, you have to change the systemd configuration and then start up elasticsearch
Distributions like SUSE do not use the `chkconfig` tool to register services, but rather `systemd` and its command `/bin/systemctl` to start and stop services (at least in newer versions, otherwise use the `chkconfig` commands above). The configuration file is also placed at `/etc/sysconfig/elasticsearch` if the system is rpm based and `/etc/default/elasticsearch` if it is deb. After installing the RPM, you have to change the systemd configuration and then start up elasticsearch
[source,sh]
--------------------------------------------------

View File

@ -298,14 +298,6 @@ Enter value for [node.name]:
NOTE: Elasticsearch will not start if `${prompt.text}` or `${prompt.secret}`
is used in the settings and the process is run as a service or in the background.
The location of the configuration file can be set externally using a
system property:
[source,sh]
--------------------------------------------------
$ elasticsearch -Des.config=/path/to/config/file
--------------------------------------------------
[float]
[[configuration-index-settings]]
=== Index Settings

View File

@ -239,42 +239,13 @@ clean_before_test() {
start_elasticsearch_service() {
local desiredStatus=${1:-green}
if [ -f "/tmp/elasticsearch/bin/elasticsearch" ]; then
# su and the Elasticsearch init script work together to break bats.
# sudo isolates bats enough from the init script so everything continues
# to tick along
sudo -u elasticsearch bash <<BASH
# If jayatana is installed then we try to use it. Elasticsearch should ignore it even when we try.
# If it doesn't ignore it then Elasticsearch will fail to start because of security errors.
# This line is attempting to emulate the on login behavior of /usr/share/upstart/sessions/jayatana.conf
[ -f /usr/share/java/jayatanaag.jar ] && export JAVA_TOOL_OPTIONS="-javaagent:/usr/share/java/jayatanaag.jar"
# And now we can start Elasticsearch normally, in the background (-d) and with a pidfile (-p).
/tmp/elasticsearch/bin/elasticsearch -d -p /tmp/elasticsearch/elasticsearch.pid
BASH
elif is_systemd; then
run systemctl daemon-reload
[ "$status" -eq 0 ]
run systemctl enable elasticsearch.service
[ "$status" -eq 0 ]
run systemctl is-enabled elasticsearch.service
[ "$status" -eq 0 ]
run systemctl start elasticsearch.service
[ "$status" -eq 0 ]
elif is_sysvinit; then
run service elasticsearch start
[ "$status" -eq 0 ]
fi
run_elasticsearch_service 0
wait_for_elasticsearch_status $desiredStatus
if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then
pid=$(cat /tmp/elasticsearch/elasticsearch.pid)
[ "x$pid" != "x" ] && [ "$pid" -gt 0 ]
echo "Looking for elasticsearch pid...."
ps $pid
elif is_systemd; then
@ -290,6 +261,59 @@ BASH
fi
}
# Start elasticsearch
# $1 expected status code
# $2 additional command line args
run_elasticsearch_service() {
# Set the CONF_DIR setting in case we start as a service
if [ ! -z "$CONF_DIR" ] ; then
if is_dpkg ; then
echo "CONF_DIR=$CONF_DIR" >> /etc/default/elasticsearch;
elif is_rpm; then
echo "CONF_DIR=$CONF_DIR" >> /etc/sysconfig/elasticsearch;
fi
fi
if [ -f "/tmp/elasticsearch/bin/elasticsearch" ]; then
if [ -z "$CONF_DIR" ]; then
local CONF_DIR=""
fi
# we must capture the exit code to compare so we don't want to start as background process in case we expect something other than 0
BACKGROUND=""
if [ $1 = 0 ]; then
BACKGROUND="-d"
fi
# su and the Elasticsearch init script work together to break bats.
# sudo isolates bats enough from the init script so everything continues
# to tick along
run sudo -u elasticsearch bash <<BASH
# If jayatana is installed then we try to use it. Elasticsearch should ignore it even when we try.
# If it doesn't ignore it then Elasticsearch will fail to start because of security errors.
# This line is attempting to emulate the on login behavior of /usr/share/upstart/sessions/jayatana.conf
[ -f /usr/share/java/jayatanaag.jar ] && export JAVA_TOOL_OPTIONS="-javaagent:/usr/share/java/jayatanaag.jar"
# And now we can start Elasticsearch normally, in the background (-d) and with a pidfile (-p).
timeout 60s /tmp/elasticsearch/bin/elasticsearch $BACKGROUND -p /tmp/elasticsearch/elasticsearch.pid -Des.path.conf="$CONF_DIR" $2
BASH
[ "$status" -eq $1 ]
elif is_systemd; then
run systemctl daemon-reload
[ "$status" -eq 0 ]
run systemctl enable elasticsearch.service
[ "$status" -eq 0 ]
run systemctl is-enabled elasticsearch.service
[ "$status" -eq 0 ]
run systemctl start elasticsearch.service
[ "$status" -eq $1 ]
elif is_sysvinit; then
run service elasticsearch start
[ "$status" -eq $1 ]
fi
}
stop_elasticsearch_service() {
if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then
pid=$(cat /tmp/elasticsearch/elasticsearch.pid)
@ -325,7 +349,7 @@ wait_for_elasticsearch_status() {
if [ -e "$ESLOG/elasticsearch.log" ]; then
cat "$ESLOG/elasticsearch.log"
else
echo "The elasticsearch log doesn't exist. Maybe /vag/log/messages has something:"
echo "The elasticsearch log doesn't exist. Maybe /var/log/messages has something:"
tail -n20 /var/log/messages
fi
false

View File

@ -83,6 +83,35 @@ else
}
fi
@test "[$GROUP] install jvm-example plugin with a custom CONFIG_FILE and check failure" {
local relativePath=${1:-$(readlink -m jvm-example-*.zip)}
CONF_FILE="$ESCONFIG/elasticsearch.yml" run sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/plugin" install "file://$relativePath"
# this should fail because CONF_FILE is no longer supported
[ $status = 1 ]
CONF_FILE="$ESCONFIG/elasticsearch.yml" run sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/plugin" remove jvm-example
echo "status is $status"
[ $status = 1 ]
}
@test "[$GROUP] start elasticsearch with a custom CONFIG_FILE and check failure" {
local CONF_FILE="$ESCONFIG/elasticsearch.yml"
if is_dpkg; then
echo "CONF_FILE=$CONF_FILE" >> /etc/default/elasticsearch;
elif is_rpm; then
echo "CONF_FILE=$CONF_FILE" >> /etc/sysconfig/elasticsearch;
fi
run_elasticsearch_service 1 -Des.default.config="$CONF_FILE"
# remove settings again otherwise cleaning up before next testrun will fail
if is_dpkg ; then
sudo sed -i '/CONF_FILE/d' /etc/default/elasticsearch
elif is_rpm; then
sudo sed -i '/CONF_FILE/d' /etc/sysconfig/elasticsearch
fi
}
@test "[$GROUP] install jvm-example plugin with a custom path.plugins" {
# Clean up after the last time this test was run
rm -rf /tmp/plugins.*
@ -111,6 +140,9 @@ fi
move_config
CONF_DIR="$ESCONFIG" install_jvm_example
CONF_DIR="$ESCONFIG" start_elasticsearch_service
diff <(curl -s localhost:9200/_cat/configured_example | sed 's/ //g') <(echo "foo")
stop_elasticsearch_service
CONF_DIR="$ESCONFIG" remove_jvm_example
}