Add temporary directory cleanup workarounds (#32615)

On some Linux distributions tmpfiles.d cleans files and
directories under /tmp if they haven't been accessed for
10 days.

This can cause problems for ML as ML is currently the only
component that uses the temp directory more than a few
seconds after startup. If you didn't open an ML job for
10 days and then tried to open one then the temp directory
would have been deleted.

This commit prevents the problem occurring in the case of
Elasticsearch being managed by systemd, as systemd private
temp directories are not subject to periodic cleanup (by
default).

Additionally there are now some docs to warn people about
the risk and suggest a manual mitigation for .tar.gz users.

distribution/packages/src/common/systemd/elasticsearch.service

@@ -6,6 +6,7 @@ After=network-online.target
 
 [Service]
 RuntimeDirectory=elasticsearch
+PrivateTmp=true
 Environment=ES_HOME=/usr/share/elasticsearch
 Environment=ES_PATH_CONF=${path.conf}
 Environment=PID_DIR=/var/run/elasticsearch

docs/reference/setup/important-settings.asciidoc

@@ -14,6 +14,7 @@ The following settings *must* be considered before going to production:
 * <<heap-size,Heap size>>
 * <<heap-dump-path,Heap dump path>>
 * <<gc-logging,GC logging>>
+* <<es-tmpdir,Temp directory>>
 
 include::important-settings/path-settings.asciidoc[]
 

docs/reference/setup/important-settings/es-tmpdir.asciidoc

@@ -0,0 +1,23 @@
+[[es-tmpdir]]
+=== Temp directory
+
+By default, Elasticsearch uses a private temporary directory that the startup
+script creates immediately below the system temporary directory.
+
+On some Linux distributions a system utility will clean files and directories
+from `/tmp` if they have not been recently accessed. This can lead to the
+private temporary directory being removed while Elasticsearch is running if
+features that require the temporary directory are not used for a long time.
+This causes problems if a feature that requires the temporary directory is
+subsequently used.
+
+If you install Elasticsearch using the `.deb` or `.rpm` packages and run it
+under `systemd` then the private temporary directory that Elasticsearch uses
+is excluded from periodic cleanup.
+
+However, if you intend to run the `.tar.gz` distribution on Linux for an
+extended period then you should consider creating a dedicated temporary
+directory for Elasticsearch that is not under a path that will have old files
+and directories cleaned from it. This directory should have permissions set
+so that only the user that Elasticsearch runs as can access it. Then set the
+`$ES_TMPDIR` environment variable to point to it before starting Elasticsearch.

qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats

@@ -189,7 +189,10 @@ setup() {
 
 @test "[SYSTEMD] start Elasticsearch with custom JVM options" {
     assert_file_exist $ESENVFILE
-    local temp=`mktemp -d`
+    # The custom config directory is not under /tmp or /var/tmp because
+    # systemd's private temp directory functionaly means different
+    # processes can have different views of what's in these directories
+    local temp=`mktemp -p /etc -d`
     cp "$ESCONFIG"/elasticsearch.yml "$temp"
     cp "$ESCONFIG"/log4j2.properties "$temp"
     touch "$temp/jvm.options"

qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash

@@ -92,11 +92,14 @@ fi
 
 @test "[$GROUP] install a sample plugin with a symlinked plugins path" {
     # Clean up after the last time this test was run
-    rm -rf /tmp/plugins.*
-    rm -rf /tmp/old_plugins.*
+    rm -rf /var/plugins.*
+    rm -rf /var/old_plugins.*
 
     rm -rf "$ESPLUGINS"
-    local es_plugins=$(mktemp -d -t 'plugins.XXXX')
+    # The custom plugins directory is not under /tmp or /var/tmp because
+    # systemd's private temp directory functionaly means different
+    # processes can have different views of what's in these directories
+    local es_plugins=$(mktemp -p /var -d -t 'plugins.XXXX')
     chown -R elasticsearch:elasticsearch "$es_plugins"
     ln -s "$es_plugins" "$ESPLUGINS"
 

qa/vagrant/src/test/resources/packaging/utils/utils.bash

@@ -555,7 +555,10 @@ run_elasticsearch_tests() {
 # Move the config directory to another directory and properly chown it.
 move_config() {
     local oldConfig="$ESCONFIG"
-    export ESCONFIG="${1:-$(mktemp -d -t 'config.XXXX')}"
+    # The custom config directory is not under /tmp or /var/tmp because
+    # systemd's private temp directory functionaly means different
+    # processes can have different views of what's in these directories
+    export ESCONFIG="${1:-$(mktemp -p /etc -d -t 'config.XXXX')}"
     echo "Moving configuration directory from $oldConfig to $ESCONFIG"
 
     # Move configuration files to the new configuration directory