Add JVM options configuration file

This commit adds a new configuration file jvm.options to centralize and
simplify management of JVM options. This separates the configuration of
the JVM from the packaging scripts (bin/elasticsearch*, bin/service.bat,
and init.d/elasticsearch) simplifying end-user operational management of
custom JVM options.

distribution/build.gradle

@@ -145,6 +145,7 @@ subprojects {
 
     configFiles = copySpec {
       from '../src/main/resources/config'
+      MavenFilteringHack.filter(it, expansions)
     }
 
     commonFiles = copySpec {
@@ -242,6 +243,7 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
       up to date when the directory is created, which it would by default. And
       that'll happen when createEtc runs. */
     outputs.file "${packagingFiles}/etc/elasticsearch/elasticsearch.yml"
+    outputs.file "${packagingFiles}/etc/elasticsearch/jvm.options"
     outputs.file "${packagingFiles}/etc/elasticsearch/logging.yml"
   }
 
@@ -305,6 +307,7 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
     }
 
     configurationFile '/etc/elasticsearch/elasticsearch.yml'
+    configurationFile '/etc/elasticsearch/jvm.options'
     configurationFile '/etc/elasticsearch/logging.yml'
     into('/etc') {
       from "${packagingFiles}/etc"

distribution/deb/src/main/packaging/init.d/elasticsearch

@@ -46,16 +46,6 @@ ES_GROUP=elasticsearch
 # Directory where the Elasticsearch binary distribution resides
 ES_HOME=/usr/share/$NAME
 
-# Heap size defaults to 256m min, 1g max
-# Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g
-#ES_HEAP_SIZE=2g
-
-# Heap new generation
-#ES_HEAP_NEWSIZE=
-
-# max direct memory
-#ES_DIRECT_SIZE=
-
 # Additional Java OPTS
 #ES_JAVA_OPTS=
 
@@ -101,11 +91,7 @@ PID_FILE="$PID_DIR/$NAME.pid"
 DAEMON=$ES_HOME/bin/elasticsearch
 DAEMON_OPTS="-d -p $PID_FILE -Ees.default.path.logs=$LOG_DIR -Ees.default.path.data=$DATA_DIR -Ees.default.path.conf=$CONF_DIR"
 
-export ES_HEAP_SIZE
-export ES_HEAP_NEWSIZE
-export ES_DIRECT_SIZE
 export ES_JAVA_OPTS
-export ES_GC_LOG_FILE
 export JAVA_HOME
 export ES_INCLUDE
 
@@ -129,11 +115,6 @@ case "$1" in
   start)
 	checkJava
 
-	if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then
-		log_failure_msg "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set"
-		exit 1
-	fi
-
 	log_daemon_msg "Starting $DESC"
 
 	pid=`pidofproc -p $PID_FILE elasticsearch`

distribution/rpm/src/main/packaging/init.d/elasticsearch

@@ -59,11 +59,7 @@ exec="$ES_HOME/bin/elasticsearch"
 prog="elasticsearch"
 pidfile="$PID_DIR/${prog}.pid"
 
-export ES_HEAP_SIZE
-export ES_HEAP_NEWSIZE
-export ES_DIRECT_SIZE
 export ES_JAVA_OPTS
-export ES_GC_LOG_FILE
 export ES_STARTUP_SLEEP_TIME
 export JAVA_HOME
 export ES_INCLUDE
@@ -91,10 +87,7 @@ checkJava() {
 start() {
     checkJava
     [ -x $exec ] || exit 5
-    if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then
+
-        echo "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set"
-        return 7
-    fi
     if [ -n "$MAX_OPEN_FILES" ]; then
         ulimit -n $MAX_OPEN_FILES
     fi
@@ -104,7 +97,6 @@ start() {
     if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then
         sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
     fi
-    export ES_GC_LOG_FILE
 
     # Ensure that the PID_DIR exists (it is cleaned at OS startup time)
     if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then

distribution/src/main/packaging/env/elasticsearch

@@ -20,25 +20,12 @@
 # Elasticsearch PID directory
 #PID_DIR=/var/run/elasticsearch
 
-# Heap size defaults to ${heap.min} min, ${heap.max} max
-# Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g
-#ES_HEAP_SIZE=2g
-
-# Heap new generation
-#ES_HEAP_NEWSIZE=
-
-# Maximum direct memory
-#ES_DIRECT_SIZE=
-
 # Additional Java OPTS
 #ES_JAVA_OPTS=
 
 # Configure restart on package upgrade (true, every other setting will lead to not restarting)
 #ES_RESTART_ON_UPGRADE=true
 
-# Path to the GC log file
-#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log
-
 ################################
 # Elasticsearch service
 ################################
@@ -67,7 +54,7 @@ ES_STARTUP_SLEEP_TIME=5
 
 # The maximum number of bytes of memory that may be locked into RAM
 # Set to "unlimited" if you use the 'bootstrap.mlockall: true' option
-# in elasticsearch.yml (ES_HEAP_SIZE  must also be set).
+# in elasticsearch.yml.
 # When using Systemd, the LimitMEMLOCK property must be set
 # in /usr/lib/systemd/system/elasticsearch.service
 #MAX_LOCKED_MEMORY=unlimited

distribution/src/main/resources/bin/elasticsearch

@@ -6,14 +6,14 @@
 # behavior, those variables are:
 #
 #   ES_CLASSPATH -- A Java classpath containing everything necessary to run.
-#   JAVA_OPTS    -- Additional arguments to the JVM for heap size, etc
+#   ES_JVM_OPTIONS -- Path to file containing JVM options
 #   ES_JAVA_OPTS -- External Java Opts on top of the defaults set
 #
+# Optionally, exact memory values can be set using the `ES_JAVA_OPTS`.
+# Note that the Xms and Xmx lines in the JVM options file must be
+# commented out. Sample format include "512m", and "10g".
 #
-# Optionally, exact memory values can be set using the following values, note,
+#   ES_JAVA_OPTS="-Xms8g -Xmx8g" ./bin/elasticsearch
-# they can still be set using the `ES_JAVA_OPTS`. Sample format include "512m", and "10g".
-#
-#   ES_HEAP_SIZE -- Sets both the minimum and maximum memory to allocate (recommended)
 #
 # As a convenience, a fragment of shell is sourced in order to set one or
 # more of these variables. This so-called `include' can be placed in a
@@ -44,6 +44,13 @@
 
 # Check to see if you are trying to run this without building it first. Gradle
 # will replace the project.name with _something_.
+
+jvm_options() {
+  if [ -f "$1" ]; then
+    echo "$(grep "^-" "$1" | tr '\n' ' ')"
+  fi
+}
+
 if echo '${project.name}' | grep project.name > /dev/null ; then
     cat >&2 << EOF
 Error: You must build the project with Maven or download a pre-built package
@@ -74,6 +81,17 @@ ES_HOME=`dirname "$SCRIPT"`/..
 # make ELASTICSEARCH_HOME absolute
 ES_HOME=`cd "$ES_HOME"; pwd`
 
+if [ -z "$ES_JVM_OPTIONS" ]; then
+    for jvm_options in "$ES_HOME"/config/jvm.options \
+                      /etc/elasticsearch/jvm.options; do
+        if [ -r "$jvm_options" ]; then
+            ES_JVM_OPTIONS=$jvm_options
+            break
+        fi
+    done
+fi
+
+ES_JAVA_OPTS="$(jvm_options "$ES_JVM_OPTIONS") $ES_JAVA_OPTS"
 
 # If an include wasn't specified in the environment, then search for one...
 if [ "x$ES_INCLUDE" = "x" ]; then
@@ -114,7 +132,7 @@ fi
 # works around https://bugs.launchpad.net/ubuntu/+source/jayatana/+bug/1441487
 if [ "x$JAVA_TOOL_OPTIONS" != "x" ]; then
     echo "Warning: Ignoring JAVA_TOOL_OPTIONS=$JAVA_TOOL_OPTIONS"
-    echo "Please pass JVM parameters via JAVA_OPTS instead"
+    echo "Please pass JVM parameters via ES_JAVA_OPTS instead"
     unset JAVA_TOOL_OPTIONS
 fi
 
@@ -126,10 +144,10 @@ export HOSTNAME
 # manual parsing to find out, if process should be detached
 daemonized=`echo $* | egrep -- '(^-d |-d$| -d |--daemonize$|--daemonize )'`
 if [ -z "$daemonized" ] ; then
-    exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" \
+    exec "$JAVA" $ES_JAVA_OPTS -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" \
           org.elasticsearch.bootstrap.Elasticsearch "$@"
 else
-    exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" \
+    exec "$JAVA" $ES_JAVA_OPTS -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" \
           org.elasticsearch.bootstrap.Elasticsearch "$@" <&- &
     retval=$?
     pid=$!

distribution/src/main/resources/bin/elasticsearch.in.sh

@@ -11,82 +11,3 @@ EOF
 fi
 
 ES_CLASSPATH="$ES_HOME/lib/elasticsearch-${project.version}.jar:$ES_HOME/lib/*"
-
-if [ "x$ES_MIN_MEM" = "x" ]; then
-    ES_MIN_MEM=${heap.min}
-fi
-if [ "x$ES_MAX_MEM" = "x" ]; then
-    ES_MAX_MEM=${heap.max}
-fi
-if [ "x$ES_HEAP_SIZE" != "x" ]; then
-    ES_MIN_MEM=$ES_HEAP_SIZE
-    ES_MAX_MEM=$ES_HEAP_SIZE
-fi
-
-# min and max heap sizes should be set to the same value to avoid
-# stop-the-world GC pauses during resize, and so that we can lock the
-# heap in memory on startup to prevent any of it from being swapped
-# out.
-JAVA_OPTS="$JAVA_OPTS -Xms${ES_MIN_MEM}"
-JAVA_OPTS="$JAVA_OPTS -Xmx${ES_MAX_MEM}"
-
-# new generation
-if [ "x$ES_HEAP_NEWSIZE" != "x" ]; then
-    JAVA_OPTS="$JAVA_OPTS -Xmn${ES_HEAP_NEWSIZE}"
-fi
-
-# max direct memory
-if [ "x$ES_DIRECT_SIZE" != "x" ]; then
-    JAVA_OPTS="$JAVA_OPTS -XX:MaxDirectMemorySize=${ES_DIRECT_SIZE}"
-fi
-
-# set to headless, just in case
-JAVA_OPTS="$JAVA_OPTS -Djava.awt.headless=true"
-
-# Force the JVM to use IPv4 stack
-if [ "x$ES_USE_IPV4" != "x" ]; then
-  JAVA_OPTS="$JAVA_OPTS -Djava.net.preferIPv4Stack=true"
-fi
-
-# Add gc options. ES_GC_OPTS is unsupported, for internal testing
-if [ "x$ES_GC_OPTS" = "x" ]; then
-  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseParNewGC"
-  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseConcMarkSweepGC"
-  ES_GC_OPTS="$ES_GC_OPTS -XX:CMSInitiatingOccupancyFraction=75"
-  ES_GC_OPTS="$ES_GC_OPTS -XX:+UseCMSInitiatingOccupancyOnly"
-fi
-
-JAVA_OPTS="$JAVA_OPTS $ES_GC_OPTS"
-
-# GC logging options
-if [ -n "$ES_GC_LOG_FILE" ]; then
-  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDetails"
-  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCTimeStamps"
-  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDateStamps"
-  JAVA_OPTS="$JAVA_OPTS -XX:+PrintClassHistogram"
-  JAVA_OPTS="$JAVA_OPTS -XX:+PrintTenuringDistribution"
-  JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCApplicationStoppedTime"
-  JAVA_OPTS="$JAVA_OPTS -Xloggc:$ES_GC_LOG_FILE"
-
-  # Ensure that the directory for the log file exists: the JVM will not create it.
-  mkdir -p "`dirname \"$ES_GC_LOG_FILE\"`"
-fi
-
-# Causes the JVM to dump its heap on OutOfMemory.
-JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError"
-# The path to the heap dump location, note directory must exists and have enough
-# space for a full heap dump.
-#JAVA_OPTS="$JAVA_OPTS -XX:HeapDumpPath=$ES_HOME/logs/heapdump.hprof"
-
-# Disables explicit GC
-JAVA_OPTS="$JAVA_OPTS -XX:+DisableExplicitGC"
-
-# Enable pre-touching of memory pages used by the JVM during hotspot
-# initialization
-JAVA_OPTS="$JAVA_OPTS -XX:+AlwaysPreTouch"
-
-# Ensure UTF-8 encoding by default (e.g. filenames)
-JAVA_OPTS="$JAVA_OPTS -Dfile.encoding=UTF-8"
-
-# Use our provided JNA always versus the system one
-JAVA_OPTS="$JAVA_OPTS -Djna.nosys=true"

distribution/src/main/resources/config/elasticsearch.yml

@@ -42,8 +42,9 @@
 #
 # bootstrap.mlockall: true
 #
-# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory
+# Make sure that the heap size is set to about half the memory available
-# available on the system and that the owner of the process is allowed to use this limit.
+# on the system and that the owner of the process is allowed to use this
+# limit.
 #
 # Elasticsearch performs poorly when the system is swapping the memory.
 #

distribution/src/main/resources/config/jvm.options

@@ -0,0 +1,73 @@
+### configuration
+
+## basic
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+-Djna.nosys=true
+
+## networking
+
+# force the JVM to use the IPv4 stack
+#-Djava.net.preferIPv4Stack=true
+
+### memory
+
+## heap configuration
+
+# minimum size of total heap space
+-Xms${heap.min}
+
+# maximum size of total heap space
+# it is recommended to set the minimum size and maximum size equal to
+# each other to prevent pauses upon resize, and so that memory locking
+# will lock the entire heap in physical memory
+-Xmx${heap.max}
+
+# size of young generation heap space
+#-Xmn${new.size}
+
+# maximum size of NIO direct-buffer allocations
+#-XX:MaxDirectMemorySize${max.direct.memory.size}
+
+## CMS configuration
+-XX:+UseParNewGC
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## optimizations
+
+# disable calls to System#gc
+-XX:+DisableExplicitGC
+
+# pre-touch memory pages used by the JVM during initialization
+-XX:+AlwaysPreTouch
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${heap.dump.path}
+
+## GC logging
+
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${loggc}

docs/reference/migration/migrate_5_0/packaging.asciidoc

@@ -21,4 +21,20 @@ a memory page during GC time. This will increase the startup time of
 Elasticsearch as well as increasing the initial resident memory usage of the
 Java process.
 
+==== JVM options
+
+Arguments to the Java Virtual Machine have been centralized and moved
+to a new configuration file jvm.options. This centralization allows for
+simpler end-user management of JVM options.
+
+This migration removes all previous mechanisms of setting JVM options
+via the environment variables `ES_MIN_MEM`, `ES_MAX_MEM`,
+`ES_HEAP_SIZE`, `ES_HEAP_NEWSIZE`, `ES_DIRECT_SIZE`, `ES_USE_IPV4`,
+`ES_GC_OPTS`, `ES_GC_LOG_FILE`, and `JAVA_OPTS`.
+
+The default location for this file is in config/jvm.options if installing
+from the tar or zip distributions, and /etc/elasticsearch/jvm.options if installing
+from the Debian or RPM packages. You can specify an alternative location by setting
+the environment variable `ES_JVM_OPTIONS` to the path to the file.
+
 

docs/reference/modules/scripting/security.asciidoc

@@ -99,12 +99,13 @@ The classloader whitelist can be customised by tweaking the local Java
 Security Policy either:
 
 * system wide: `$JAVA_HOME/lib/security/java.policy`,
-* for just the `elasticsearch` user: `/home/elasticsearch/.java.policy`, or
+* for just the `elasticsearch` user: `/home/elasticsearch/.java.policy`
-* from a file specified in the `JAVA_OPTS` environment variable with `-Djava.security.policy=someURL`:
+* by adding a system property to the <<sysconfig,es-java-opts>> configuration: `-Djava.security.policy=someURL`, or
+* via the `ES_JAVA_OPTS` environment variable with `-Djava.security.policy=someURL`:
 +
 [source,js]
 ---------------------------------
-export JAVA_OPTS="${JAVA_OPTS} -Djava.security.policy=file:///path/to/my.policy`
+export ES_JAVA_OPTS="${ES_JAVA_OPTS} -Djava.security.policy=file:///path/to/my.policy`
 ./bin/elasticsearch
 ---------------------------------
 

docs/reference/setup/install/sysconfig-file.asciidoc

@@ -11,18 +11,6 @@
 
   Set a custom Java path to be used.
 
-`ES_HEAP_SIZE`::
-
-  The heap size to start with.
-
-`ES_HEAP_NEWSIZE`::
-
-    The size of the new generation heap.
-
-`ES_DIRECT_SIZE`::
-
-    The maximum size of the direct memory.
-
 `MAX_OPEN_FILES`::
 
     Maximum number of open files, defaults to `65536`.
@@ -30,8 +18,7 @@
 `MAX_LOCKED_MEMORY`::
 
     Maximum locked memory size. Set to `unlimited if you use the
-    `bootstrap.mlockall` option in elasticsearch.yml. You must also set
+    `bootstrap.mlockall` option in elasticsearch.yml.
-    ES_HEAP_SIZE.
 
 `MAX_MAP_COUNT`::
 
@@ -66,10 +53,3 @@
     package manually. The reason for this is to ensure, that upgrades in a
     cluster do not result in a continuous shard reallocation resulting in high
     network traffic and reducing the response times of your cluster.
-
-`ES_GC_LOG_FILE` ::
-
-    The absolute log file path for creating a garbage collection logfile,
-    which is done by the JVM. Note that this logfile can grow pretty quick and
-    thus is disabled by default.
-

docs/reference/setup/install/windows.asciidoc

@@ -141,18 +141,6 @@ The Elasticsearch service can be configured prior to installation by setting the
 
   The installation directory of the desired JVM to run the service under.
 
-`ES_HEAP_SIZE`::
-
-  The heap size to start with.
-
-`ES_MIN_MEM`::
-
-  The initial memory allocation pool for the JVM (`Xms`).  Defaults to `256m`, overriden by `%ES_HEAP_SIZE%`.
-
-`ES_MAX_MEM`::
-
-  The maxmimum memory allocation pool for the JVM (`Xmx`).  Defaults to `1g`, overriden by `%ES_HEAP_SIZE%`.
-
 `LOG_DIR`::
 
     Log directory, defaults to `%ES_HOME%\logs`.

docs/reference/setup/sysconfig/configuring.asciidoc

@@ -104,14 +104,23 @@ LimitMEMLOCK=infinity
 [[es-java-opts]]
 ==== Setting JVM system properties
 
-Any Java system properties or arguments which should be passed to the JVM
+The preferred method of setting Java Virtual Machine options (including
-should be specified in the `ES_JAVA_OPTS` environment variable, for instance:
+system properties and JVM flags) is via the jvm.options configuration
+file. The default location of this file is config/jvm.options (when
+installing from the tar or zip distributions) and
+/etc/elasticsearch/jvm.options (when installing from the Debian or RPM
+packages). This file contains a line-delimited list of JVM arguments,
+which must begin with `-`. You can add custom JVM flags to this file and
+check this configuration into your version control system.
+
+An alternative mechanism for setting Java Virtual Machine options is
+via the `ES_JAVA_OPTS` environment variable. For instance:
 
 [source,sh]
 ---------------------------------
-EXPORT ES_JAVA_OPTS="$ES_JAVA_OPTS -Djava.io.tmpdir=/path/to/temp/dir"
+export ES_JAVA_OPTS="$ES_JAVA_OPTS -Djava.io.tmpdir=/path/to/temp/dir"
 ./bin/elasticsearch
 ---------------------------------
 
-When using the RPM or Debian packages, ES_JAVA_OPTS can be specified in the
+When using the RPM or Debian packages, `ES_JAVA_OPTS` can be specified in the
 <<sysconfig,system configuration file>>.

docs/reference/setup/sysconfig/heap_size.asciidoc

@@ -1,36 +1,65 @@
 [[heap-size]]
-=== Set JVM heap size with ES_HEAP_SIZE
+=== Set JVM heap size via jvm.options
 
 In development mode, Elasticsearch tells the JVM to use a heap with a minimum
-size of 256MB and a maximum size of 1GB. When moving to production, it is
+size of 256 MB and a maximum size of 1 GB. When moving to production, it is
-important to configure the `ES_HEAP_SIZE environment variable to ensure that
+important to configure heap size to ensure that Elasticsearch has enough
-Elasticsearch has enough heap available.
+heap available.
 
-Elasticsearch will assign the entire heap specified in `ES_HEAP_SIZE on startup.
+Elasticsearch will assign the entire heap specified in <<sysconfig,es-java-opts>>
+via the Xms (minimum heap size) and Xmx (maximum heap size) settings.
 
-The value for this setting depends on the amount of RAM available on your server. A good rule of thumb is:
+The value for these setting depends on the amount of RAM available on
+your server. Good rules of thumb are:
+
+* Set the minimum heap size (Xms) and maximum heap size (Xmx) to be
+  equal to each other.
 
 * The more heap available to Elasticsearch, the more memory it can use for
-  caching.
+  caching. But note that too much heap can subject you to long garbage
+  collection pauses.
 
-* Set the `ES_HEAP_SIZE` to no more than 50% of your RAM, to ensure that there
+* Set Xmx to no more than 50% of your physical RAM, to ensure that there
-  is enough RAM left for kernel file system caches.
+  is enough physical RAM left for kernel file system caches.
 
-* Don’t set `ES_HEAP_SIZE` to more than about 31GB to ensure that the JVM uses
+* Don’t set Xmx to above the cutoff that the JVM uses for compressed
-  compressed object pointers (compressed-oops). You can verify that you are
+  object pointers (compressed oops); the exact cutoff varies but is
-  under the limit by looking for a line in the logs like the following:
+  near 32 GB. You can verify that you are under the limit by looking
+  for a line in the logs like the following:
 
     heap size [1.9gb], compressed ordinary object pointers [true]
 
-Here are examples of how to set the ES_HEAP_SIZE:
+* Even better, try to stay below the threshold for zero-based
+  compressed oops; the exact cutoff varies but 26 GB is safe on most
+  systems, but can be as large as 30 GB on some system. You can verify
+  that you are under the limit by starting Elasticsearch with the JVM
+  options "-XX:+UnlockDiagnosticVMOptions -XX:+PrintCompressedOopsMode"
+  and looking for a line like the following:
+
+    heap address: 0x000000011be00000, size: 27648 MB, zero based Compressed Oops
+
+  showing that zero-based compressed oops are enabled instead of
+
+    heap address: 0x0000000118400000, size: 28672 MB, Compressed Oops with base: 0x00000001183ff000
+
+Here are examples of how to set the heap size via the jvm.options file:
+
+[source]
+------------------
+Xms2g <1>
+Xmx2g <2>
+------------------
+<1> Set the minimum heap size to 2g.
+<2> Set the maximum heap size to 2g.
+
+It is also possible to set the heap size via an environment variable.
+This can be done by commenting out the `Xms` and `Xmx` settings
+in the jvm.options file and setting these values via `ES_JAVA_OPTS`:
 
 [source,sh]
 ------------------
-ES_HEAP_SIZE=2g    ./bin/elasticsearch <1>
+ES_JAVA_OPTS="-Xms2g -Xmx2g" ./bin/elasticsearch <3>
-ES_HEAP_SIZE=4000m ./bin/elasticsearch <2>
+ES_JAVA_OPTS="-Xmx4000mb -Xmx4000mb" ./bin/elasticsearch <4>
 ------------------
-<1> Set the min/max heap size to 2GB.
+<3> Set the minimum and maximum heap size to 2 GB.
-<2> Set the min/max heap size to 4,000MB.
+<4> Set the minimum and maximum heap size to 4000 MB.
-
-When using the Debian or RPM package, ES_HEAP_SIZE can be configured in the
-<<sysconfig,system configuration file>>.

docs/reference/setup/sysconfig/swap.asciidoc

@@ -83,8 +83,7 @@ export ES_JAVA_OPTS="$ES_JAVA_OPTS -Djava.io.tmpdir=/path/to/temp/dir"
 
 The second option is to completely disable swap. Usually Elasticsearch
 is the only service running on a box, and its memory usage is controlled
-by the `ES_HEAP_SIZE` environment variable.  There should be no need
+by the JVM options.  There should be no need to have swap enabled.
-to have swap enabled.
 
 On Linux systems, you can disable swap temporarily
 by running: `sudo swapoff -a`. To disable it permanently, you will need

qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats

@@ -95,5 +95,26 @@ setup() {
     start_elasticsearch_service
     run_elasticsearch_tests
     stop_elasticsearch_service
+}
+
+@test "[TAR]" start Elasticsearch with custom JVM options {
+    local es_java_opts=$ES_JAVA_OPTS
+    local es_jvm_options=$ES_JVM_OPTIONS
+    local temp=`mktemp -d`
+    touch "$temp/jvm.options"
+    chown -R elasticsearch:elasticsearch "$temp"
+    echo "-Xms264m" >> "$temp/jvm.options"
+    echo "-Xmx264m" >> "$temp/jvm.options"
+    export ES_JVM_OPTIONS="$temp/jvm.options"
+    export ES_JAVA_OPTS="-XX:-UseCompressedOops"
+    start_elasticsearch_service
+    curl -s -XGET localhost:9200/_nodes | fgrep '"heap_init_in_bytes":276824064'
+    curl -s -XGET localhost:9200/_nodes | fgrep '"using_compressed_ordinary_object_pointers":"false"'
+    stop_elasticsearch_service
+    export ES_JVM_OPTIONS=$es_jvm_options
+    export ES_JAVA_OPTS=$es_java_opts
+}
+
+@test "[TAR]" remove tar {
     rm -rf "/tmp/elasticsearch"
 }

qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats

@@ -128,6 +128,7 @@ setup() {
     # The configuration files are still here
     assert_file_exist "/etc/elasticsearch"
     assert_file_exist "/etc/elasticsearch/elasticsearch.yml"
+    assert_file_exist "/etc/elasticsearch/jvm.options"
     assert_file_exist "/etc/elasticsearch/logging.yml"
 
     # The env file is still here
@@ -148,6 +149,7 @@ setup() {
     # all remaining files are deleted by the purge
     assert_file_not_exist "/etc/elasticsearch"
     assert_file_not_exist "/etc/elasticsearch/elasticsearch.yml"
+    assert_file_not_exist "/etc/elasticsearch/jvm.options"
     assert_file_not_exist "/etc/elasticsearch/logging.yml"
 
     assert_file_not_exist "/etc/default/elasticsearch"

qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats

@@ -116,6 +116,7 @@ setup() {
 
     assert_file_not_exist "/etc/elasticsearch"
     assert_file_not_exist "/etc/elasticsearch/elasticsearch.yml"
+    assert_file_not_exist  "/etc/elasticsearch/jvm.options"
     assert_file_not_exist "/etc/elasticsearch/logging.yml"
 
     assert_file_not_exist "/etc/init.d/elasticsearch"

qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash

@@ -348,6 +348,8 @@ run_elasticsearch_service() {
 # This line is attempting to emulate the on login behavior of /usr/share/upstart/sessions/jayatana.conf
 [ -f /usr/share/java/jayatanaag.jar ] && export JAVA_TOOL_OPTIONS="-javaagent:/usr/share/java/jayatanaag.jar"
 # And now we can start Elasticsearch normally, in the background (-d) and with a pidfile (-p).
+export ES_JVM_OPTIONS=$ES_JVM_OPTIONS
+export ES_JAVA_OPTS=$ES_JAVA_OPTS
 $timeoutCommand/tmp/elasticsearch/bin/elasticsearch $background -p /tmp/elasticsearch/elasticsearch.pid $ES_PATH_CONF $commandLineArgs
 BASH
         [ "$status" -eq "$expectedStatus" ]