From c4c0db6f214bc93bd592632fea02f44f6975c87f Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 8 Feb 2020 18:50:14 -0500 Subject: [PATCH] Introduce jvm.options.d for customizing JVM options (#51882) This commit introduces the ability to override JVM options by adding custom JVM options files to a jvm.options.d directory. This simplifies administration of Elasticsearch by not requiring administrators to keep the root jvm.options file in sync with changes that we make to the root jvm.options file. Instead, they are not expected to modify this file but instead supply their own in jvm.options.d. In Docker installations, this means they can bind mount this directory in. In future versions of Elasticsearch, we can consider removing the root jvm.options file (instead, providing all options there as system JVM options). --- distribution/archives/build.gradle | 11 +- distribution/docker/src/docker/Dockerfile | 4 +- distribution/packages/build.gradle | 4 +- .../packages/src/common/scripts/postrm | 17 ++ .../packages/src/deb/lintian/elasticsearch | 1 + distribution/src/bin/elasticsearch | 3 +- .../src/bin/elasticsearch-service.bat | 8 +- distribution/src/bin/elasticsearch.bat | 3 +- .../tools/launchers/JvmOptionsParser.java | 145 ++++++++++-------- .../important-settings/heap-size.asciidoc | 5 +- docs/reference/setup/install/docker.asciidoc | 23 +-- docs/reference/setup/jvm-options.asciidoc | 29 ++-- .../packaging/test/ArchiveTests.java | 56 +++++++ .../packaging/test/DebPreservationTests.java | 9 +- .../packaging/test/PackageTests.java | 20 ++- .../packaging/test/RpmPreservationTests.java | 8 + .../packaging/util/Installation.java | 4 + .../packaging/util/Packages.java | 4 + 18 files changed, 242 insertions(+), 112 deletions(-) diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 26f45d3b57d..fb540486155 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -46,6 +46,11 @@ task createPluginsDir(type: EmptyDirTask) { dir = "${pluginsDir}" dirMode = 0755 } +ext.jvmOptionsDir = new File(buildDir, 'jvm-options-hack/jvm.options.d') +task createJvmOptionsDir(type: EmptyDirTask) { + dir = "${jvmOptionsDir}" + dirMode = 0750 +} CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss, boolean jdk) { return copySpec { @@ -57,6 +62,10 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla dirMode 0750 fileMode 0660 with configFiles(distributionType, oss, jdk) + from { + dirMode 0750 + jvmOptionsDir.getParent() + } } into('bin') { with binFiles(distributionType, oss, jdk) @@ -96,7 +105,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla // common config across all zip/tar tasks.withType(AbstractArchiveTask) { - dependsOn createLogsDir, createPluginsDir + dependsOn createLogsDir, createPluginsDir, createJvmOptionsDir String subdir = it.name.substring('build'.size()).replaceAll(/[A-Z]/) { '-' + it.toLowerCase() }.substring(1) destinationDir = file("${subdir}/build/distributions") baseName = "elasticsearch${subdir.contains('oss') ? '-oss' : ''}" diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 41113d1d596..ef576f32dc5 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -30,8 +30,8 @@ ${source_elasticsearch} RUN tar zxf /opt/${elasticsearch} --strip-components=1 RUN grep ES_DISTRIBUTION_TYPE=tar /usr/share/elasticsearch/bin/elasticsearch-env \ && sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env -RUN mkdir -p config data logs -RUN chmod 0775 config data logs +RUN mkdir -p config config/jvm.options.d data logs +RUN chmod 0775 config config/jvm.options.d data logs COPY config/elasticsearch.yml config/log4j2.properties config/ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index f4ad06d19b6..dcf87ef19d6 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -79,8 +79,9 @@ void addProcessFilesTask(String type, boolean oss, boolean jdk) { mkdir "${packagingFiles}/var/lib/elasticsearch" mkdir "${packagingFiles}/usr/share/elasticsearch/plugins" - // bare empty dir for /etc/elasticsearch + // bare empty dir for /etc/elasticsearch and /etc/elasticsearch/jvm.options.d mkdir "${packagingFiles}/elasticsearch" + mkdir "${packagingFiles}/elasticsearch/jvm.options.d" } } } @@ -197,6 +198,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) { includeEmptyDirs true createDirectoryEntry true include("elasticsearch") // empty dir, just to add directory entry + include("elasticsearch/jvm.options.d") // empty dir, just to add directory entry } from("${packagingFiles}/etc/elasticsearch") { into('/etc/elasticsearch') diff --git a/distribution/packages/src/common/scripts/postrm b/distribution/packages/src/common/scripts/postrm index e0e0bd237a6..0b77069603b 100644 --- a/distribution/packages/src/common/scripts/postrm +++ b/distribution/packages/src/common/scripts/postrm @@ -17,6 +17,7 @@ else fi REMOVE_DIRS=false +REMOVE_JVM_OPTIONS_DIRECTORY=false REMOVE_USER_AND_GROUP=false case "$1" in @@ -27,6 +28,8 @@ case "$1" in ;; purge) + REMOVE_DIRS=true + REMOVE_JVM_OPTIONS_DIRECTORY=true REMOVE_USER_AND_GROUP=true ;; failed-upgrade|abort-install|abort-upgrade|disappear|upgrade|disappear) @@ -80,6 +83,20 @@ if [ "$REMOVE_DIRS" = "true" ]; then rmdir --ignore-fail-on-non-empty /var/lib/elasticsearch fi + # delete the jvm.options.d directory if and only if empty + if [ -d "${ES_PATH_CONF}/jvm.options.d" ]; then + rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}/jvm.options.d" + fi + + # delete the jvm.options.d directory if we are purging + if [ "$REMOVE_JVM_OPTIONS_DIRECTORY" = "true" ]; then + if [ -d "${ES_PATH_CONF}/jvm.options.d" ]; then + echo -n "Deleting jvm.options.d directory..." + rm -rf "${ES_PATH_CONF}/jvm.options.d" + echo " OK" + fi + fi + # delete the conf directory if and only if empty if [ -d "${ES_PATH_CONF}" ]; then rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}" diff --git a/distribution/packages/src/deb/lintian/elasticsearch b/distribution/packages/src/deb/lintian/elasticsearch index 98038177a00..05c25a40ba5 100644 --- a/distribution/packages/src/deb/lintian/elasticsearch +++ b/distribution/packages/src/deb/lintian/elasticsearch @@ -16,6 +16,7 @@ missing-dep-on-jarwrapper # we prefer to not make our config and log files world readable non-standard-file-perm etc/default/elasticsearch 0660 != 0644 non-standard-dir-perm etc/elasticsearch/ 2750 != 0755 +non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755 non-standard-file-perm etc/elasticsearch/* non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755 non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755 diff --git a/distribution/src/bin/elasticsearch b/distribution/src/bin/elasticsearch index 1573d549bd4..e45bdb1ad22 100755 --- a/distribution/src/bin/elasticsearch +++ b/distribution/src/bin/elasticsearch @@ -46,8 +46,7 @@ then fi fi -ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options -ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_JVM_OPTIONS"` +ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"` # manual parsing to find out, if process should be detached if [[ $DAEMONIZE = false ]]; then diff --git a/distribution/src/bin/elasticsearch-service.bat b/distribution/src/bin/elasticsearch-service.bat index dd936138016..2fcad8bf49b 100644 --- a/distribution/src/bin/elasticsearch-service.bat +++ b/distribution/src/bin/elasticsearch-service.bat @@ -115,7 +115,7 @@ set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;% @setlocal -for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" || echo jvm_options_parser_failed"`) do set ES_JAVA_OPTS=%%a +for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_PATH_CONF!" || echo jvm_options_parser_failed"`) do set ES_JAVA_OPTS=%%a @endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%ES_JAVA_OPTS%" & set ES_JAVA_OPTS=%ES_JAVA_OPTS% if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" ( @@ -167,15 +167,15 @@ for %%a in ("%ES_JAVA_OPTS:;=","%") do ( @endlocal & set JVM_MS=%JVM_MS% & set JVM_MX=%JVM_MX% & set JVM_SS=%JVM_SS% & set OTHER_JAVA_OPTS=%OTHER_JAVA_OPTS% if "%JVM_MS%" == "" ( - echo minimum heap size not set; configure using -Xms via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS + echo minimum heap size not set; configure using -Xms via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS goto:eof ) if "%JVM_MX%" == "" ( - echo maximum heap size not set; configure using -Xmx via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS + echo maximum heap size not set; configure using -Xmx via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS goto:eof ) if "%JVM_SS%" == "" ( - echo thread stack size not set; configure using -Xss via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS + echo thread stack size not set; configure using -Xss via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS goto:eof ) set OTHER_JAVA_OPTS=%OTHER_JAVA_OPTS:"=% diff --git a/distribution/src/bin/elasticsearch.bat b/distribution/src/bin/elasticsearch.bat index 48a34fdd332..81cd6a4efd7 100644 --- a/distribution/src/bin/elasticsearch.bat +++ b/distribution/src/bin/elasticsearch.bat @@ -72,9 +72,8 @@ if not defined ES_TMPDIR ( for /f "tokens=* usebackq" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.TempDirectory"`) do set ES_TMPDIR=%%a ) -set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options @setlocal -for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" ^|^| echo jvm_options_parser_failed`) do set ES_JAVA_OPTS=%%a +for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_PATH_CONF!" ^|^| echo jvm_options_parser_failed`) do set ES_JAVA_OPTS=%%a @endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%ES_JAVA_OPTS%" & set ES_JAVA_OPTS=%ES_JAVA_OPTS% if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" ( diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java index 24fa6f632cd..4574c19074d 100644 --- a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java @@ -27,7 +27,9 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -42,6 +44,7 @@ import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; /** * Parses JVM options from a file and prints a single line with all JVM options to standard output. @@ -52,80 +55,88 @@ final class JvmOptionsParser { * The main entry point. The exit code is 0 if the JVM options were successfully parsed, otherwise the exit code is 1. If an improperly * formatted line is discovered, the line is output to standard error. * - * @param args the args to the program which should consist of a single option, the path to the JVM options + * @param args the args to the program which should consist of a single option, the path to ES_PATH_CONF */ public static void main(final String[] args) throws InterruptedException, IOException { if (args.length != 1) { - throw new IllegalArgumentException("expected one argument specifying path to jvm.options but was " + Arrays.toString(args)); - } - final List jvmOptions = new ArrayList<>(); - final SortedMap invalidLines = new TreeMap<>(); - try ( - InputStream is = Files.newInputStream(Paths.get(args[0])); - Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8); - BufferedReader br = new BufferedReader(reader) - ) { - parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, new JvmOptionConsumer() { - @Override - public void accept(final String jvmOption) { - jvmOptions.add(jvmOption); - } - }, new InvalidLineConsumer() { - @Override - public void accept(final int lineNumber, final String line) { - invalidLines.put(lineNumber, line); - } - }); + throw new IllegalArgumentException("expected one argument specifying path to ES_PATH_CONF but was " + Arrays.toString(args)); } - if (invalidLines.isEmpty()) { - // now append the JVM options from ES_JAVA_OPTS - final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS"); - if (environmentJvmOptions != null) { - jvmOptions.addAll( - Arrays.stream(environmentJvmOptions.split("\\s+")).filter(s -> s.trim().isEmpty() == false).collect(Collectors.toList()) - ); + final ArrayList jvmOptionsFiles = new ArrayList<>(); + jvmOptionsFiles.add(Paths.get(args[0], "jvm.options")); + + final Path jvmOptionsDirectory = Paths.get(args[0], "jvm.options.d"); + + if (Files.isDirectory(jvmOptionsDirectory)) { + try ( + DirectoryStream jvmOptionsDirectoryStream = Files.newDirectoryStream(Paths.get(args[0], "jvm.options.d"), "*.options") + ) { + // collect the matching JVM options files after sorting them by Path::compareTo + StreamSupport.stream(jvmOptionsDirectoryStream.spliterator(), false).sorted().forEach(jvmOptionsFiles::add); } - final Map substitutions = new HashMap<>(); - substitutions.put("ES_TMPDIR", System.getenv("ES_TMPDIR")); - if (null != System.getenv("ES_PATH_CONF")) { - substitutions.put("ES_PATH_CONF", System.getenv("ES_PATH_CONF")); - } - final List substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions)); - final List ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions); - final List systemJvmOptions = SystemJvmOptions.systemJvmOptions(); - final List finalJvmOptions = new ArrayList<>( - systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size() - ); - finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden - finalJvmOptions.addAll(substitutedJvmOptions); - finalJvmOptions.addAll(ergonomicJvmOptions); - final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(finalJvmOptions); - Launchers.outPrintln(spaceDelimitedJvmOptions); - Launchers.exit(0); - } else { - final String errorMessage = String.format( - Locale.ROOT, - "encountered [%d] error%s parsing [%s]", - invalidLines.size(), - invalidLines.size() == 1 ? "" : "s", - args[0] - ); - Launchers.errPrintln(errorMessage); - int count = 0; - for (final Map.Entry entry : invalidLines.entrySet()) { - count++; - final String message = String.format( - Locale.ROOT, - "[%d]: encountered improperly formatted JVM option line [%s] on line number [%d]", - count, - entry.getValue(), - entry.getKey() - ); - Launchers.errPrintln(message); - } - Launchers.exit(1); } + + final List jvmOptions = new ArrayList<>(); + + for (final Path jvmOptionsFile : jvmOptionsFiles) { + final SortedMap invalidLines = new TreeMap<>(); + try ( + InputStream is = Files.newInputStream(jvmOptionsFile); + Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8); + BufferedReader br = new BufferedReader(reader) + ) { + parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, jvmOptions::add, invalidLines::put); + } + if (invalidLines.isEmpty() == false) { + final String errorMessage = String.format( + Locale.ROOT, + "encountered [%d] error%s parsing [%s]", + invalidLines.size(), + invalidLines.size() == 1 ? "" : "s", + jvmOptionsFile + ); + Launchers.errPrintln(errorMessage); + int count = 0; + for (final Map.Entry entry : invalidLines.entrySet()) { + count++; + final String message = String.format( + Locale.ROOT, + "[%d]: encountered improperly formatted JVM option in [%s] on line number [%d]: [%s]", + count, + jvmOptionsFile, + entry.getKey(), + entry.getValue() + ); + Launchers.errPrintln(message); + } + Launchers.exit(1); + } + } + + // now append the JVM options from ES_JAVA_OPTS + final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS"); + if (environmentJvmOptions != null) { + jvmOptions.addAll( + Arrays.stream(environmentJvmOptions.split("\\s+")).filter(s -> s.trim().isEmpty() == false).collect(Collectors.toList()) + ); + } + final Map substitutions = new HashMap<>(); + substitutions.put("ES_TMPDIR", System.getenv("ES_TMPDIR")); + if (null != System.getenv("ES_PATH_CONF")) { + substitutions.put("ES_PATH_CONF", System.getenv("ES_PATH_CONF")); + } + final List substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions)); + final List ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions); + final List systemJvmOptions = SystemJvmOptions.systemJvmOptions(); + final List finalJvmOptions = new ArrayList<>( + systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size() + ); + finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden + finalJvmOptions.addAll(substitutedJvmOptions); + finalJvmOptions.addAll(ergonomicJvmOptions); + final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(finalJvmOptions); + Launchers.outPrintln(spaceDelimitedJvmOptions); + Launchers.exit(0); } static List substitutePlaceholders(final List jvmOptions, final Map substitutions) { diff --git a/docs/reference/setup/important-settings/heap-size.asciidoc b/docs/reference/setup/important-settings/heap-size.asciidoc index 37e417e086e..43211dcc16c 100644 --- a/docs/reference/setup/important-settings/heap-size.asciidoc +++ b/docs/reference/setup/important-settings/heap-size.asciidoc @@ -49,7 +49,7 @@ caches, but the less memory it leaves available for the operating system to use for the filesystem cache. Also, larger heaps can cause longer garbage collection pauses. -Here are examples of how to set the heap size via the jvm.options file: +Here is an example of how to set the heap size via a `jvm.options.d/` file: [source,txt] ------------------ @@ -60,8 +60,7 @@ Here are examples of how to set the heap size via the jvm.options file: <2> Set the maximum heap size to 2g. It is also possible to set the heap size via an environment variable. This can -be done by commenting out the `Xms` and `Xmx` settings in the -<> file and setting these values via `ES_JAVA_OPTS`: +be done by setting these values via `ES_JAVA_OPTS`: [source,sh] ------------------ diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc index b792ffd7a98..6ed2a3b3b7d 100644 --- a/docs/reference/setup/install/docker.asciidoc +++ b/docs/reference/setup/install/docker.asciidoc @@ -272,22 +272,23 @@ unless you are pinning one container per host. [[docker-set-heap-size]] ===== Set the heap size -Use the `ES_JAVA_OPTS` environment variable to set the heap size. -For example, to use 16GB, specify `-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with -`docker run`. Note that while the default configuration file `jvm.options` -sets a default heap of 1GB, any value you set in `ES_JAVA_OPTS` will -override it. +To configure the heap size, you can bind mount a <> +file under `/usr/share/elasticsearch/config/jvm.options.d` that includes your +desired <> settings. Note that while the default root +`jvm.options` file sets a default heap of 1 GB, any value you set in a +bind-mounted JVM options file will override it. + +While setting the heap size via bind-mounted JVM options is the recommended +method, you can also configure this by using the `ES_JAVA_OPTS` environment +variable to set the heap size. For example, to use 16 GB, specify +`-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with `docker run`. Note that while the +default root `jvm.options` file sets a default heap of 1 GB, any value you set +in `ES_JAVA_OPTS` will override it. IMPORTANT: You must <> even if you are https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory[limiting memory access] to the container. -While setting the heap size via an environment variable is the recommended -method, you can also configure this by bind-mounting your own `jvm.options` -file under `/usr/share/elasticsearch/config/`. The file that {es} provides -contains some important settings, so you should start by taking a copy of -`jvm.options` from an {es} container and editing it as you require. - ===== Pin deployments to a specific image version Pin your deployments to a specific version of the {es} Docker image. For diff --git a/docs/reference/setup/jvm-options.asciidoc b/docs/reference/setup/jvm-options.asciidoc index 61c24fc1e1b..3a619355eda 100644 --- a/docs/reference/setup/jvm-options.asciidoc +++ b/docs/reference/setup/jvm-options.asciidoc @@ -5,18 +5,20 @@ You should rarely need to change Java Virtual Machine (JVM) options. If you do, the most likely change is setting the <>. The remainder of this document explains in detail how to set JVM options. -The preferred method of setting JVM options (including system properties and JVM -flags) is via the `jvm.options` configuration file. The default location of this -file is `config/jvm.options` (when installing from the tar or zip distributions) -and `/etc/elasticsearch/jvm.options` (when installing from the Debian or RPM -packages). +The preferred method of setting or overriding JVM options is via JVM options +files. When installing from the tar or zip distributions, the root `jvm.options` +configuration file is `config/jvm.options` and custom JVM options files can be +added to `config/jvm.options.d/`. When installing from the Debian or RPM +packages, the root `jvm.options` configuration file is +``/etc/elasticsearch/jvm.options` and custom JVM options files can be added to +`/etc/elasticsearch/jvm.options.d/`. When using the <> you can bind mount custom JVM options files into +`/usr/share/elasticsearch/config/jvm.options.d/`. You should never need to +modify the root `jvm.options` file instead preferring to use custom JVM options +files. The processing ordering of custom JVM options is lexicographic. -NOTE: If you are using the <>, we -recommend that you <>. - -This file contains a line-delimited list of JVM arguments following -a special syntax: +JVM options files must have the suffix '.options' and contain a line-delimited +list of JVM arguments following a special syntax: * lines consisting of whitespace only are ignored * lines beginning with `#` are treated as comments and are ignored @@ -62,9 +64,6 @@ a special syntax: * all other lines are rejected -You can add custom JVM flags to this file and check this configuration into your -version control system. - An alternative mechanism for setting Java Virtual Machine options is via the `ES_JAVA_OPTS` environment variable. For instance: @@ -81,7 +80,7 @@ The JVM has a built-in mechanism for observing the `JAVA_TOOL_OPTIONS` environment variable. We intentionally ignore this environment variable in our packaging scripts. The primary reason for this is that on some OS (e.g., Ubuntu) there are agents installed by default via this environment variable that we do -not want interfering with Elasticsearch. +not want interfering with {es}. Additionally, some other Java programs support the `JAVA_OPTS` environment variable. This is *not* a mechanism built into the JVM but instead a convention diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index a023cf5969c..23303f911a4 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -261,6 +261,62 @@ public class ArchiveTests extends PackagingTestCase { } } + public void test71CustomJvmOptionsDirectoryFile() throws Exception { + final Path heapOptions = installation.config(Paths.get("jvm.options.d", "heap.options")); + try { + append(heapOptions, "-Xms512m\n-Xmx512m\n"); + + startElasticsearch(); + + final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); + assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); + + stopElasticsearch(); + } finally { + rm(heapOptions); + } + } + + public void test72CustomJvmOptionsDirectoryFilesAreProcessedInSortedOrder() throws Exception { + final Path firstOptions = installation.config(Paths.get("jvm.options.d", "first.options")); + final Path secondOptions = installation.config(Paths.get("jvm.options.d", "second.options")); + try { + /* + * We override the heap in the first file, and disable compressed oops, and override the heap in the second file. By doing this, + * we can test that both files are processed by the JVM options parser, and also that they are processed in lexicographic order. + */ + append(firstOptions, "-Xms384m\n-Xmx384m\n-XX:-UseCompressedOops\n"); + append(secondOptions, "-Xms512m\n-Xmx512m\n"); + + startElasticsearch(); + + final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); + assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); + assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\"")); + + stopElasticsearch(); + } finally { + rm(firstOptions); + rm(secondOptions); + } + } + + public void test73CustomJvmOptionsDirectoryFilesWithoutOptionsExtensionIgnored() throws Exception { + final Path jvmOptionsIgnored = installation.config(Paths.get("jvm.options.d", "jvm.options.ignored")); + try { + append(jvmOptionsIgnored, "-Xms512\n-Xmx512m\n"); + + startElasticsearch(); + + final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); + assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":1073741824")); + + stopElasticsearch(); + } finally { + rm(jvmOptionsIgnored); + } + } + public void test80RelativePathConf() throws Exception { final Path temp = getTempDir().resolve("esconf-alternate"); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DebPreservationTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DebPreservationTests.java index e7d895005a6..18777e1a4ee 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DebPreservationTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DebPreservationTests.java @@ -25,6 +25,7 @@ import org.junit.BeforeClass; import java.nio.file.Paths; import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileExists; +import static org.elasticsearch.packaging.util.FileUtils.append; import static org.elasticsearch.packaging.util.FileUtils.assertPathsDoNotExist; import static org.elasticsearch.packaging.util.FileUtils.assertPathsExist; import static org.elasticsearch.packaging.util.Packages.SYSVINIT_SCRIPT; @@ -53,15 +54,17 @@ public class DebPreservationTests extends PackagingTestCase { } public void test20Remove() throws Exception { + append(installation.config(Paths.get("jvm.options.d", "heap.options")), "# foo"); + remove(distribution()); // some config files were not removed - assertPathsExist( installation.config, installation.config("elasticsearch.yml"), installation.config("jvm.options"), - installation.config("log4j2.properties") + installation.config("log4j2.properties"), + installation.config(Paths.get("jvm.options.d", "heap.options")) ); if (distribution().isDefault()) { @@ -96,6 +99,8 @@ public class DebPreservationTests extends PackagingTestCase { } public void test30Purge() throws Exception { + append(installation.config(Paths.get("jvm.options.d", "heap.options")), "# foo"); + sh.run("dpkg --purge " + distribution().flavor.name); assertRemoved(distribution()); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java index c950969c12b..9524a3fe4b3 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java @@ -138,6 +138,22 @@ public class PackageTests extends PackagingTestCase { } } + public void test34CustomJvmOptionsDirectoryFile() throws Exception { + final Path heapOptions = installation.config(Paths.get("jvm.options.d", "heap.options")); + try { + append(heapOptions, "-Xms512m\n-Xmx512m\n"); + + startElasticsearch(); + + final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); + assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); + + stopElasticsearch(); + } finally { + rm(heapOptions); + } + } + public void test42BundledJdkRemoved() throws Exception { assumeThat(distribution().hasJdk, is(true)); @@ -301,7 +317,7 @@ public class PackageTests extends PackagingTestCase { }); } - public void test82SystemdMask() throws Exception { + public void test83SystemdMask() throws Exception { try { assumeTrue(isSystemd()); @@ -314,7 +330,7 @@ public class PackageTests extends PackagingTestCase { } } - public void test83serviceFileSetsLimits() throws Exception { + public void test84serviceFileSetsLimits() throws Exception { // Limits are changed on systemd platforms only assumeTrue(isSystemd()); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java index 6acc5df8759..5da7ec4366e 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.packaging.util.Shell; import org.junit.BeforeClass; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.stream.Stream; import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileDoesNotExist; @@ -84,6 +85,7 @@ public class RpmPreservationTests extends PackagingTestCase { ) .map(each -> installation.config(each)) .forEach(path -> append(path, "# foo")); + append(installation.config(Paths.get("jvm.options.d", "heap.options")), "# foo"); if (distribution().isDefault()) { Stream.of( "role_mapping.yml", @@ -122,6 +124,7 @@ public class RpmPreservationTests extends PackagingTestCase { "jvm.options", "log4j2.properties" ).forEach(this::assertConfFilePreserved); + assertThat(installation.config(Paths.get("jvm.options.d", "heap.options")), fileExists()); if (distribution().isDefault()) { Stream.of( @@ -136,7 +139,12 @@ public class RpmPreservationTests extends PackagingTestCase { private void assertConfFilePreserved(String configFile) { final Path original = installation.config(configFile); final Path saved = installation.config(configFile + ".rpmsave"); + assertConfFilePreserved(original ,saved); + } + + private void assertConfFilePreserved(final Path original, final Path saved) { assertThat(original, fileDoesNotExist()); assertThat(saved, fileExists()); } + } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java index e879fe24c8b..9144496f2d1 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java @@ -135,6 +135,10 @@ public class Installation { return config.resolve(configFileName); } + public Path config(Path configFileName) { + return config.resolve(configFileName); + } + public Executables executables() { return new Executables(); } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java index de5b9039034..df9d3fd0b85 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java @@ -182,6 +182,10 @@ public class Packages { assertThat(es.config, file(Directory, "root", "elasticsearch", p750)); assertThat(sh.run("find \"" + es.config + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("2750")); + final Path jvmOptionsDirectory = es.config.resolve("jvm.options.d"); + assertThat(jvmOptionsDirectory, file(Directory, "root", "elasticsearch", p750)); + assertThat(sh.run("find \"" + jvmOptionsDirectory + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("2750")); + Stream.of( "elasticsearch.keystore", "elasticsearch.yml",