Bundle java in distributions (#38013)

* Bundle java in distributions

Setting up a jdk is currently a required external step when installing
elasticsearch. This is particularly problematic for the rpm/deb packages
as installing a jdk in the same package installation command does not
guarantee any order, so must be done in separate steps. Additionally,
JAVA_HOME must be set and often causes problems in selecting a correct
jdk when, for example, the system java is an older unsupported version.

This commit bundles platform specific openjdks into each distribution.
In addition to eliminating the issues above, it also presents future
possible improvements like using jlink to build jdk images only
containing modules that elasticsearch uses.

closes #31845

Vagrantfile

@@ -350,6 +350,8 @@ def sh_install_deps(config,
 if [ -z "\\\$JAVA_HOME" ]; then
   export JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))
 fi
+export SYSTEM_JAVA_HOME=\\\$JAVA_HOME
+unset JAVA_HOME
 JAVA
     ensure tar
     ensure curl
@@ -388,6 +390,7 @@ Defaults   env_keep += "BATS_TESTS"
 Defaults   env_keep += "PACKAGING_ARCHIVES"
 Defaults   env_keep += "PACKAGING_TESTS"
 Defaults   env_keep += "JAVA_HOME"
+Defaults   env_keep += "SYSTEM_JAVA_HOME"
 SUDOERS_VARS
     chmod 0440 /etc/sudoers.d/elasticsearch_vars
   SHELL
@@ -408,6 +411,9 @@ def windows_common(config, name)
   config.vm.provision 'set env variables', type: 'shell', inline: <<-SHELL
     $ErrorActionPreference = "Stop"
     [Environment]::SetEnvironmentVariable("PACKAGING_ARCHIVES", "C:/project/build/packaging/archives", "Machine")
+    $javaHome = [Environment]::GetEnvironmentVariable("JAVA_HOME", "Machine")
+    [Environment]::SetEnvironmentVariable("SYSTEM_JAVA_HOME", $javaHome, "Machine")
     [Environment]::SetEnvironmentVariable("PACKAGING_TESTS", "C:/project/build/packaging/tests", "Machine")
+    [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine")
   SHELL
 end

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -213,6 +213,7 @@ class BuildPlugin implements Plugin<Project> {
             project.rootProject.ext.runtimeJavaHome = runtimeJavaHome
             project.rootProject.ext.compilerJavaVersion = compilerJavaVersionEnum
             project.rootProject.ext.runtimeJavaVersion = runtimeJavaVersionEnum
+            project.rootProject.ext.isRuntimeJavaHomeSet = compilerJavaHome.equals(runtimeJavaHome) == false
             project.rootProject.ext.javaVersions = javaVersions
             project.rootProject.ext.buildChecksDone = true
             project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
@@ -231,6 +232,7 @@ class BuildPlugin implements Plugin<Project> {
         project.ext.runtimeJavaHome = project.rootProject.ext.runtimeJavaHome
         project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion
         project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
+        project.ext.isRuntimeJavaHomeSet = project.rootProject.ext.isRuntimeJavaHomeSet
         project.ext.javaVersions = project.rootProject.ext.javaVersions
         project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm
         project.ext.gradleJavaVersion = project.rootProject.ext.gradleJavaVersion

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -681,7 +681,13 @@ class ClusterFormationTasks {
     static Task configureExecTask(String name, Project project, Task setup, NodeInfo node, Object[] execArgs) {
         return project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { Exec exec ->
             exec.workingDir node.cwd
-            exec.environment 'JAVA_HOME', node.getJavaHome()
+            // TODO: this must change to 7.0.0 after bundling java has been backported
+            if (project.isRuntimeJavaHomeSet || node.nodeVersion.before(Version.fromString("8.0.0"))) {
+                exec.environment.put('JAVA_HOME', project.runtimeJavaHome)
+            } else {
+                // force JAVA_HOME to *not* be set
+                exec.environment.remove('JAVA_HOME')
+            }
             if (Os.isFamily(Os.FAMILY_WINDOWS)) {
                 exec.executable 'cmd'
                 exec.args '/C', 'call'
@@ -698,8 +704,12 @@ class ClusterFormationTasks {
     static Task configureStartTask(String name, Project project, Task setup, NodeInfo node) {
         // this closure is converted into ant nodes by groovy's AntBuilder
         Closure antRunner = { AntBuilder ant ->
-            ant.exec(executable: node.executable, spawn: node.config.daemonize, dir: node.cwd, taskname: 'elasticsearch') {
+            ant.exec(executable: node.executable, spawn: node.config.daemonize, newenvironment: true,
+                     dir: node.cwd, taskname: 'elasticsearch') {
                 node.env.each { key, value -> env(key: key, value: value) }
+                if (project.isRuntimeJavaHomeSet || node.nodeVersion.before(Version.fromString("8.0.0"))) {
+                    env(key: 'JAVA_HOME', value: project.runtimeJavaHome)
+                }
                 node.args.each { arg(value: it) }
             }
         }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -23,7 +23,6 @@ import com.sun.jna.Native
 import com.sun.jna.WString
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.Version
-import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
 
 import java.nio.file.Files
@@ -240,11 +239,6 @@ class NodeInfo {
         return Native.toString(shortPath).substring(4)
     }
 
-    /** Return the java home used by this node. */
-    String getJavaHome() {
-        return javaVersion == null ? project.runtimeJavaHome : project.javaVersions.get(javaVersion)
-    }
-
     /** Returns debug string for the command that started this node. */
     String getCommandString() {
         String esCommandString = "\nNode ${nodeNum} configuration:\n"
@@ -252,7 +246,6 @@ class NodeInfo {
         esCommandString += "|  cwd: ${cwd}\n"
         esCommandString += "|  command: ${executable} ${args.join(' ')}\n"
         esCommandString += '|  environment:\n'
-        esCommandString += "|    JAVA_HOME: ${javaHome}\n"
         env.each { k, v -> esCommandString += "|    ${k}: ${v}\n" }
         if (config.daemonize) {
             esCommandString += "|\n|  [${wrapperScript.name}]\n"

buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/VersionProperties.java

@@ -10,6 +10,7 @@ import java.util.Properties;
  * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions.
  */
 public class VersionProperties {
+
     public static String getElasticsearch() {
         return elasticsearch;
     }
@@ -18,17 +19,25 @@ public class VersionProperties {
         return lucene;
     }
 
+    public static String getBundledJdk() {
+        return bundledJdk;
+    }
+
     public static Map<String, String> getVersions() {
         return versions;
     }
 
     private static final String elasticsearch;
     private static final String lucene;
+    private static final String bundledJdk;
     private static final Map<String, String> versions = new HashMap<String, String>();
+
     static {
         Properties props = getVersionProperties();
         elasticsearch = props.getProperty("elasticsearch");
         lucene = props.getProperty("lucene");
+        bundledJdk = props.getProperty("bundled_jdk");
+
         for (String property : props.stringPropertyNames()) {
             versions.put(property, props.getProperty(property));
         }

buildSrc/version.properties

@@ -1,6 +1,8 @@
 elasticsearch     = 7.1.0
 lucene            = 8.0.0-snapshot-ff9509a8df
 
+bundled_jdk       = 11.0.2+9
+
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0

distribution/archives/build.gradle

@@ -45,7 +45,7 @@ task createPluginsDir(type: EmptyDirTask) {
   dirMode 0755
 }
 
-CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, boolean oss) {
+CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss) {
   return copySpec {
     into("elasticsearch-${version}") {
       into('lib') {
@@ -59,6 +59,11 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, boolean os
       into('bin') {
         with binFiles(distributionType, oss)
       }
+      if (platform != null) {
+        into('jdk') {
+          with jdkFiles(platform)
+        }
+      }
       into('') {
         from {
           dirMode 0755
@@ -102,19 +107,19 @@ Closure commonZipConfig = {
 
 task buildIntegTestZip(type: Zip) {
   configure(commonZipConfig)
-  with archiveFiles(transportModulesFiles, 'zip', true)
+  with archiveFiles(transportModulesFiles, 'zip', null, true)
 }
 
 task buildWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'windows-x86_64'
-  with archiveFiles(modulesFiles(false), 'zip', false)
+  with archiveFiles(modulesFiles(false), 'zip', 'windows', false)
 }
 
 task buildOssWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'windows-x86_64'
-  with archiveFiles(modulesFiles(true), 'zip', true)
+  with archiveFiles(modulesFiles(true), 'zip', 'windows', true)
 }
 
 Closure commonTarConfig = {
@@ -127,25 +132,25 @@ Closure commonTarConfig = {
 task buildDarwinTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'darwin-x86_64'
-  with archiveFiles(modulesFiles(false), 'tar', false)
+  with archiveFiles(modulesFiles(false), 'tar', 'darwin', false)
 }
 
 task buildOssDarwinTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'darwin-x86_64'
-  with archiveFiles(modulesFiles(true), 'tar', true)
+  with archiveFiles(modulesFiles(true), 'tar', 'darwin', true)
 }
 
 task buildLinuxTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(false), 'tar', false)
+  with archiveFiles(modulesFiles(false), 'tar', 'linux', false)
 }
 
 task buildOssLinuxTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(true), 'tar', true)
+  with archiveFiles(modulesFiles(true), 'tar', 'linux', true)
 }
 
 Closure tarExists = { it -> new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() }

distribution/build.gradle

@@ -20,11 +20,14 @@
 import org.elasticsearch.gradle.ConcatFilesTask
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.NoticeTask
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.test.RunTask
 import org.apache.tools.ant.filters.FixCrLfFilter
 
 import java.nio.file.Files
 import java.nio.file.Path
+import java.util.regex.Matcher
+import java.util.regex.Pattern
 
 /*****************************************************************************
  *                  Third party dependencies report                          *
@@ -210,6 +213,51 @@ xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
   copyLog4jProperties(buildDefaultLog4jConfig, xpackModule)
 }
 
+/*****************************************************************************
+ *                                    JDKs                                   *
+ *****************************************************************************/
+// extract the bundled jdk version, broken into elements as: [feature, interim, update, build]
+// Note the "patch" version is not yet handled here, as it has not yet been used by java.
+Pattern JDK_VERSION = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d+)\\+(\\d+)")
+Matcher jdkVersionMatcher = JDK_VERSION.matcher(VersionProperties.bundledJdk)
+if (jdkVersionMatcher.matches() == false) {
+  throw new IllegalArgumentException("Malformed jdk version [" + VersionProperties.bundledJdk + "]")
+}
+String jdkVersion = jdkVersionMatcher.group(1) + '.' + jdkVersionMatcher.group(2) + '.' + jdkVersionMatcher.group(3)
+String jdkMajor = jdkVersionMatcher.group(1)
+String jdkBuild = jdkVersionMatcher.group(4)
+
+repositories {
+  ivy {
+    url "https://download.java.net"
+    patternLayout {
+      artifact "java/GA/jdk${jdkMajor}/${jdkBuild}/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"
+    }
+  }
+}
+for (String platform : ['linux', 'darwin', 'windows']) {
+  String jdkConfigName = "jdk_${platform}"
+  Configuration jdkConfig = configurations.create(jdkConfigName)
+  String extension = platform.equals('windows') ? 'zip' : 'tar.gz'
+  dependencies.add(jdkConfigName, "jdk:${platform.equals('darwin') ? 'osx' : platform}:${jdkVersion}@${extension}")
+
+  int rootNdx = platform.equals('darwin') ? 2 : 1
+  Closure removeRootDir = {
+    it.eachFile { FileCopyDetails details ->
+      details.relativePath = new RelativePath(true, details.relativePath.segments[rootNdx..-1] as String[])
+    }
+    it.includeEmptyDirs false
+  }
+  project.task("extract${platform.capitalize()}Jdk", type: Copy) {
+    into "${buildDir}/jdks/openjdk-${jdkVersion}_${platform}"
+    if (extension.equals('zip')) {
+      from({ zipTree(jdkConfig.singleFile) }, removeRootDir)
+    } else {
+      from({ tarTree(resources.gzip(jdkConfig.singleFile)) }, removeRootDir)
+    }
+  }
+}
+
 // make sure we have a clean task since we aren't a java project, but we have tasks that
 // put stuff in the build dir
 task clean(type: Delete) {
@@ -332,6 +380,17 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         from buildDefaultNotice
       }
     }
+
+    jdkFiles = { platform ->
+      copySpec {
+        from project(':distribution').tasks.getByName("extract${platform.capitalize()}Jdk")
+        eachFile { FileCopyDetails details ->
+          if (details.relativePath.segments[-2] == 'bin') {
+            details.mode = 0755
+          }
+        }
+      }
+    }
   }
 
 }

distribution/packages/build.gradle

@@ -135,6 +135,9 @@ Closure commonPackageConfig(String type, boolean oss) {
       into('modules') {
         with modulesFiles(oss)
       }
+      into('jdk') {
+        with jdkFiles('linux')
+      }
       // we need to specify every intermediate directory in these paths so the package managers know they are explicitly
       // intended to manage them; otherwise they may be left behind on uninstallation. duplicate calls of the same
       // directory are fine

distribution/packages/src/common/scripts/preinst

@@ -15,19 +15,6 @@ err_exit() {
     exit 1
 }
 
-# Check for these at preinst time due to failures in postinst if they do not exist
-if [ -x "$JAVA_HOME/bin/java" ]; then
-    JAVA="$JAVA_HOME/bin/java"
-elif command -v java; then
-    JAVA=`command -v java`
-else
-    JAVA=""
-fi
-
-if [ -z "$JAVA" ]; then
-    err_exit "could not find java; set JAVA_HOME"
-fi
-
 case "$1" in
 
     # Debian ####################################################

distribution/src/bin/elasticsearch-env

@@ -36,17 +36,19 @@ ES_HOME=`dirname "$ES_HOME"`
 ES_CLASSPATH="$ES_HOME/lib/*"
 
 # now set the path to java
-if [ -x "$JAVA_HOME/bin/java" ]; then
+if [ ! -z "$JAVA_HOME" ]; then
   JAVA="$JAVA_HOME/bin/java"
 else
-  set +e
+  if [ "$(uname -s)" = "Darwin" ]; then
-  JAVA=`which java`
+    # OSX has a different structure
-  echo "warning: Falling back to java on path. This behavior is deprecated. Specify JAVA_HOME"
+    JAVA="$ES_HOME/jdk/Contents/Home/bin/java"
-  set -e
+  else
+    JAVA="$ES_HOME/jdk/bin/java"
+  fi
 fi
 
 if [ ! -x "$JAVA" ]; then
-  echo "could not find java; set JAVA_HOME" >&2
+  echo "could not find java in JAVA_HOME or bundled at $JAVA" >&2
   exit 1
 fi
 

qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java

@@ -30,12 +30,11 @@ import org.elasticsearch.packaging.util.Shell;
 import org.elasticsearch.packaging.util.Shell.Result;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
 import java.util.stream.Stream;
 
-import static java.util.stream.Collectors.joining;
 import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER;
 import static org.elasticsearch.packaging.util.Archives.installArchive;
 import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation;
@@ -46,6 +45,7 @@ import static org.elasticsearch.packaging.util.FileUtils.append;
 import static org.elasticsearch.packaging.util.FileUtils.cp;
 import static org.elasticsearch.packaging.util.FileUtils.getTempDir;
 import static org.elasticsearch.packaging.util.FileUtils.mkdir;
+import static org.elasticsearch.packaging.util.FileUtils.mv;
 import static org.elasticsearch.packaging.util.FileUtils.rm;
 import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
 import static org.hamcrest.CoreMatchers.containsString;
@@ -77,46 +77,23 @@ public abstract class ArchiveTestCase extends PackagingTestCase {
         assertThat(r.stdout, isEmptyString());
     }
 
-    public void test30AbortWhenJavaMissing() {
+    public void test30NoJava() {
         assumeThat(installation, is(notNullValue()));
 
         final Installation.Executables bin = installation.executables();
         final Shell sh = new Shell();
 
-        Platforms.onWindows(() -> {
+        final Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated");
-            // on windows, removing java from PATH and removing JAVA_HOME is less involved than changing the permissions of the java
-            // executable. we also don't check permissions in the windows scripts anyway
-            final String originalPath = sh.run("$Env:PATH").stdout.trim();
-            final String newPath = Arrays.stream(originalPath.split(";"))
-                .filter(path -> path.contains("Java") == false)
-                .collect(joining(";"));
-
-            // note the lack of a $ when clearing the JAVA_HOME env variable - with a $ it deletes the java home directory
-            // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/providers/environment-provider?view=powershell-6
-            //
-            // this won't persist to another session so we don't have to reset anything
-            final Result runResult = sh.runIgnoreExitCode(
-                "$Env:PATH = '" + newPath + "'; " +
-                "Remove-Item Env:JAVA_HOME; " +
-                bin.elasticsearch
-            );
 
+        try {
+            mv(installation.bundledJdk, relocatedJdk);
+            // ask for elasticsearch version to quickly exit if java is actually found (ie test failure)
+            final Result runResult = sh.runIgnoreExitCode(bin.elasticsearch.toString() + " -v");
             assertThat(runResult.exitCode, is(1));
-            assertThat(runResult.stderr, containsString("could not find java; set JAVA_HOME"));
+            assertThat(runResult.stderr, containsString("could not find java in JAVA_HOME or bundled"));
-        });
+        } finally {
-
+            mv(relocatedJdk, installation.bundledJdk);
-        Platforms.onLinux(() -> {
+        }
-            final String javaPath = sh.run("command -v java").stdout.trim();
-
-            try {
-                sh.run("chmod -x '" + javaPath + "'");
-                final Result runResult = sh.runIgnoreExitCode(bin.elasticsearch.toString());
-                assertThat(runResult.exitCode, is(1));
-                assertThat(runResult.stderr, containsString("could not find java; set JAVA_HOME"));
-            } finally {
-                sh.run("chmod +x '" + javaPath + "'");
-            }
-        });
     }
 
     public void test40CreateKeystoreManually() {
@@ -160,15 +137,51 @@ public abstract class ArchiveTestCase extends PackagingTestCase {
 
         Archives.runElasticsearch(installation);
 
-        final String gcLogName = Platforms.LINUX
+        assertTrue("gc logs exist", Files.exists(installation.logs.resolve("gc.log")));
-            ? "gc.log.0.current"
-            : "gc.log";
-        assertTrue("gc logs exist", Files.exists(installation.logs.resolve(gcLogName)));
         ServerUtils.runElasticsearchTests();
 
         Archives.stopElasticsearch(installation);
     }
 
+    public void assertRunsWithJavaHome() throws IOException {
+        Shell sh = new Shell();
+
+        Platforms.onLinux(() -> {
+            String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim();
+            sh.getEnv().put("JAVA_HOME", systemJavaHome);
+        });
+        Platforms.onWindows(() -> {
+            final String systemJavaHome = sh.run("$Env:SYSTEM_JAVA_HOME").stdout.trim();
+            sh.getEnv().put("JAVA_HOME", systemJavaHome);
+        });
+
+        Archives.runElasticsearch(installation, sh);
+        ServerUtils.runElasticsearchTests();
+        Archives.stopElasticsearch(installation);
+
+        String systemJavaHome = sh.getEnv().get("JAVA_HOME");
+        Path log = installation.logs.resolve("elasticsearch.log");
+        assertThat(new String(Files.readAllBytes(log), StandardCharsets.UTF_8), containsString(systemJavaHome));
+    }
+
+    public void test51JavaHomeOverride() throws IOException {
+        assumeThat(installation, is(notNullValue()));
+
+        assertRunsWithJavaHome();
+    }
+
+    public void test52BundledJdkRemoved() throws IOException {
+        assumeThat(installation, is(notNullValue()));
+
+        Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated");
+        try {
+            mv(installation.bundledJdk, relocatedJdk);
+            assertRunsWithJavaHome();
+        } finally {
+            mv(relocatedJdk, installation.bundledJdk);
+        }
+    }
+
     public void test60AutoCreateKeystore() {
         assumeThat(installation, is(notNullValue()));
 

qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java

@@ -25,9 +25,10 @@ import org.elasticsearch.packaging.util.Shell.Result;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -38,8 +39,8 @@ import static org.elasticsearch.packaging.util.Packages.assertInstalled;
 import static org.elasticsearch.packaging.util.Packages.assertRemoved;
 import static org.elasticsearch.packaging.util.Packages.install;
 import static org.elasticsearch.packaging.util.Packages.remove;
-import static org.elasticsearch.packaging.util.Packages.runInstallCommand;
 import static org.elasticsearch.packaging.util.Packages.startElasticsearch;
+import static org.elasticsearch.packaging.util.Packages.stopElasticsearch;
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
 import static org.elasticsearch.packaging.util.Platforms.getOsRelease;
 import static org.elasticsearch.packaging.util.Platforms.isSystemd;
@@ -60,23 +61,6 @@ public abstract class PackageTestCase extends PackagingTestCase {
         assumeTrue("only compatible distributions", distribution().packaging.compatible);
     }
 
-    public void test05InstallFailsWhenJavaMissing() {
-        final Shell sh = new Shell();
-        final Result javaHomeOutput = sh.run("echo $JAVA_HOME");
-
-        final Path javaHome = Paths.get(javaHomeOutput.stdout.trim());
-        final Path originalJavaPath = javaHome.resolve("bin").resolve("java");
-        final Path relocatedJavaPath = javaHome.resolve("bin").resolve("java.relocated");
-        try {
-            mv(originalJavaPath, relocatedJavaPath);
-            final Result installResult = runInstallCommand(distribution());
-            assertThat(installResult.exitCode, is(1));
-            assertThat(installResult.stderr, containsString("could not find java; set JAVA_HOME"));
-        } finally {
-            mv(relocatedJavaPath, originalJavaPath);
-        }
-    }
-
     public void test10InstallPackage() {
         assertRemoved(distribution());
         installation = install(distribution());
@@ -98,6 +82,43 @@ public abstract class PackageTestCase extends PackagingTestCase {
         assertThat(sh.run("ps aux").stdout, not(containsString("org.elasticsearch.bootstrap.Elasticsearch")));
     }
 
+    public void assertRunsWithJavaHome() throws IOException {
+        Shell sh = new Shell();
+
+        String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim();
+        byte[] originalEnvFile = Files.readAllBytes(installation.envFile);
+        try {
+            Files.write(installation.envFile, ("JAVA_HOME=" + systemJavaHome + "\n").getBytes(StandardCharsets.UTF_8),
+                StandardOpenOption.APPEND);
+            startElasticsearch();
+            runElasticsearchTests();
+            stopElasticsearch();
+        } finally {
+            Files.write(installation.envFile, originalEnvFile);
+        }
+
+        Path log = installation.logs.resolve("elasticsearch.log");
+        assertThat(new String(Files.readAllBytes(log), StandardCharsets.UTF_8), containsString(systemJavaHome));
+    }
+
+    public void test31JavaHomeOverride() throws IOException {
+        assumeThat(installation, is(notNullValue()));
+
+        assertRunsWithJavaHome();
+    }
+
+    public void test42BundledJdkRemoved() throws IOException {
+        assumeThat(installation, is(notNullValue()));
+
+        Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated");
+        try {
+            mv(installation.bundledJdk, relocatedJdk);
+            assertRunsWithJavaHome();
+        } finally {
+            mv(relocatedJdk, installation.bundledJdk);
+        }
+    }
+
     public void test40StartServer() throws IOException {
         assumeThat(installation, is(notNullValue()));
 

qa/vagrant/src/main/java/org/elasticsearch/packaging/test/WindowsServiceTestCase.java

@@ -35,9 +35,9 @@ import java.nio.file.Path;
 import java.util.Arrays;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeTrue;
-import static java.util.stream.Collectors.joining;
 import static org.elasticsearch.packaging.util.Archives.installArchive;
 import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation;
+import static org.elasticsearch.packaging.util.FileUtils.mv;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -65,22 +65,15 @@ public abstract class WindowsServiceTestCase extends PackagingTestCase {
     }
 
     private Result runWithoutJava(String script) {
-        // on windows, removing java from PATH and removing JAVA_HOME is less involved than changing the permissions of the java
+        final Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated");
-        // executable. we also don't check permissions in the windows scripts anyway
-        final String originalPath = sh.run("$Env:PATH").stdout.trim();
-        final String newPath = Arrays.stream(originalPath.split(";"))
-            .filter(path -> path.contains("Java") == false)
-            .collect(joining(";"));
 
-        // note the lack of a $ when clearing the JAVA_HOME env variable - with a $ it deletes the java home directory
+        try {
-        // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/providers/environment-provider?view=powershell-6
+            mv(installation.bundledJdk, relocatedJdk);
-        //
+            // ask for elasticsearch version to quickly exit if java is actually found (ie test failure)
-        // this won't persist to another session so we don't have to reset anything
+            return sh.runIgnoreExitCode(script);
-        return sh.runIgnoreExitCode(
+        } finally {
-            "$Env:PATH = '" + newPath + "'; " +
+            mv(relocatedJdk, installation.bundledJdk);
-                "Remove-Item Env:JAVA_HOME; " +
+        }
-                script
-        );
     }
 
     private void assertService(String id, String status, String displayName) {
@@ -135,7 +128,7 @@ public abstract class WindowsServiceTestCase extends PackagingTestCase {
     public void test13InstallMissingJava() throws IOException {
         Result result = runWithoutJava(serviceScript + " install");
         assertThat(result.exitCode, equalTo(1));
-        assertThat(result.stderr, containsString("could not find java; set JAVA_HOME"));
+        assertThat(result.stderr, containsString("could not find java in JAVA_HOME or bundled"));
     }
 
     public void test14RemoveNotInstalled() {

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java

@@ -30,6 +30,7 @@ public class Installation {
     public final Path home;
     public final Path bin; // this isn't a first-class installation feature but we include it for convenience
     public final Path lib; // same
+    public final Path bundledJdk;
     public final Path config;
     public final Path data;
     public final Path logs;
@@ -42,7 +43,7 @@ public class Installation {
         this.home = home;
         this.bin = home.resolve("bin");
         this.lib = home.resolve("lib");
-
+        this.bundledJdk = home.resolve("jdk");
         this.config = config;
         this.data = data;
         this.logs = logs;

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java

@@ -283,4 +283,13 @@ public class Packages {
             sh.run("service elasticsearch status");
         }
     }
+
+    public static void stopElasticsearch() throws IOException {
+        final Shell sh = new Shell();
+        if (isSystemd()) {
+            sh.run("systemctl stop elasticsearch.service");
+        } else {
+            sh.run("service elasticsearch stop");
+        }
+    }
 }

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java

@@ -105,6 +105,7 @@ public class Shell {
         ProcessBuilder builder = new ProcessBuilder();
         builder.command(command);
 
+
         if (workingDirectory != null) {
             setWorkingDirectory(builder, workingDirectory);
         }

qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats

@@ -61,7 +61,9 @@ setup() {
 }
 
 @test "[UPGRADE] start old version" {
+    export JAVA_HOME=$SYSTEM_JAVA_HOME
     start_elasticsearch_service
+    unset JAVA_HOME
 }
 
 @test "[UPGRADE] check elasticsearch version is old version" {

qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash

@@ -162,22 +162,6 @@ fi
     remove_plugin_example
 }
 
-@test "[$GROUP] fail if java executable is not found" {
-  [ "$GROUP" == "TAR PLUGINS" ] || skip "Test case only supported by TAR PLUGINS"
-  local JAVA=$(which java)
-
-  sudo chmod -x $JAVA
-  run "$ESHOME/bin/elasticsearch-plugin"
-  sudo chmod +x $JAVA
-
-  [ "$status" -eq 1 ]
-  local expected="could not find java; set JAVA_HOME"
-  [[ "$output" == *"$expected"* ]] || {
-    echo "Expected error message [$expected] but found: $output"
-    false
-  }
-}
-
 # Note that all of the tests from here to the end of the file expect to be run
 # in sequence and don't take well to being run one at a time.
 @test "[$GROUP] install a sample plugin" {

qa/vagrant/src/test/resources/packaging/utils/packages.bash

@@ -100,7 +100,7 @@ install_package() {
     fi
 
     # pass through java home to package
-    echo "JAVA_HOME=\"$JAVA_HOME\"" >> $(env_file)
+    echo "JAVA_HOME=\"$SYSTEM_JAVA_HOME\"" >> $(env_file)
 }
 
 # Checks that all directories & files are correctly installed after a deb or

server/src/main/java/org/elasticsearch/node/Node.java

@@ -289,6 +289,7 @@ public class Node implements Closeable {
                 Constants.JVM_NAME,
                 Constants.JAVA_VERSION,
                 Constants.JVM_VERSION);
+            logger.info("JVM Home [{}]", System.getProperty("java.home"));
             logger.info("JVM arguments {}", Arrays.toString(jvmInfo.getInputArguments()));
             if (Build.CURRENT.isProductionRelease() == false) {
                 logger.warn(

x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDocCsvSpecIT.java

@@ -69,7 +69,7 @@ public class JdbcDocCsvSpecIT extends SpecBaseIntegrationTestCase {
         // uncomment this to printout the result set and create new CSV tests
         //
         //JdbcTestUtils.logLikeCLI(elastic, log);
-        JdbcAssert.assertResultSets(expected, elastic, log, true, false);
+        JdbcAssert.assertResultSets(expected, elastic, log, true, true);
     }
 
     @Override

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java

@@ -66,6 +66,6 @@ public abstract class CsvSpecTestCase extends SpecBaseIntegrationTestCase {
     @Override
     protected void assertResults(ResultSet expected, ResultSet elastic) throws SQLException {
         Logger log = logEsResultSet() ? logger : null;
-        JdbcAssert.assertResultSets(expected, elastic, log, false, false);
+        JdbcAssert.assertResultSets(expected, elastic, log, false, true);
     }
 }