From 465343f12a54111709ca7c7cd776edcaba1e0c2b Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 8 Mar 2019 11:04:18 -0800 Subject: [PATCH] Bundle java in distributions (#38013) * Bundle java in distributions Setting up a jdk is currently a required external step when installing elasticsearch. This is particularly problematic for the rpm/deb packages as installing a jdk in the same package installation command does not guarantee any order, so must be done in separate steps. Additionally, JAVA_HOME must be set and often causes problems in selecting a correct jdk when, for example, the system java is an older unsupported version. This commit bundles platform specific openjdks into each distribution. In addition to eliminating the issues above, it also presents future possible improvements like using jlink to build jdk images only containing modules that elasticsearch uses. closes #31845 --- Vagrantfile | 6 ++ .../elasticsearch/gradle/BuildPlugin.groovy | 2 + .../gradle/test/ClusterFormationTasks.groovy | 14 ++- .../elasticsearch/gradle/test/NodeInfo.groovy | 7 -- .../gradle/VersionProperties.java | 9 ++ buildSrc/version.properties | 2 + distribution/archives/build.gradle | 21 +++-- distribution/build.gradle | 59 ++++++++++++ distribution/packages/build.gradle | 3 + .../packages/src/common/scripts/preinst | 13 --- distribution/src/bin/elasticsearch-env | 14 +-- distribution/src/bin/elasticsearch-env.bat | 6 +- .../packaging/test/ArchiveTestCase.java | 91 +++++++++++-------- .../packaging/test/PackageTestCase.java | 59 ++++++++---- .../test/WindowsServiceTestCase.java | 27 ++---- .../packaging/util/Installation.java | 3 +- .../packaging/util/Packages.java | 9 ++ .../elasticsearch/packaging/util/Shell.java | 1 + .../resources/packaging/tests/80_upgrade.bats | 2 + .../tests/module_and_plugin_test_cases.bash | 16 ---- .../resources/packaging/utils/packages.bash | 2 +- .../java/org/elasticsearch/node/Node.java | 1 + .../sql/qa/single_node/JdbcDocCsvSpecIT.java | 2 +- .../xpack/sql/qa/jdbc/CsvSpecTestCase.java | 2 +- 24 files changed, 237 insertions(+), 134 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index e4ae8e09060..4624172b02a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -350,6 +350,8 @@ def sh_install_deps(config, if [ -z "\\\$JAVA_HOME" ]; then export JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java)))) fi +export SYSTEM_JAVA_HOME=\\\$JAVA_HOME +unset JAVA_HOME JAVA ensure tar ensure curl @@ -388,6 +390,7 @@ Defaults env_keep += "BATS_TESTS" Defaults env_keep += "PACKAGING_ARCHIVES" Defaults env_keep += "PACKAGING_TESTS" Defaults env_keep += "JAVA_HOME" +Defaults env_keep += "SYSTEM_JAVA_HOME" SUDOERS_VARS chmod 0440 /etc/sudoers.d/elasticsearch_vars SHELL @@ -408,6 +411,9 @@ def windows_common(config, name) config.vm.provision 'set env variables', type: 'shell', inline: <<-SHELL $ErrorActionPreference = "Stop" [Environment]::SetEnvironmentVariable("PACKAGING_ARCHIVES", "C:/project/build/packaging/archives", "Machine") + $javaHome = [Environment]::GetEnvironmentVariable("JAVA_HOME", "Machine") + [Environment]::SetEnvironmentVariable("SYSTEM_JAVA_HOME", $javaHome, "Machine") [Environment]::SetEnvironmentVariable("PACKAGING_TESTS", "C:/project/build/packaging/tests", "Machine") + [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine") SHELL end diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 1bd06914313..97074965a76 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -213,6 +213,7 @@ class BuildPlugin implements Plugin { project.rootProject.ext.runtimeJavaHome = runtimeJavaHome project.rootProject.ext.compilerJavaVersion = compilerJavaVersionEnum project.rootProject.ext.runtimeJavaVersion = runtimeJavaVersionEnum + project.rootProject.ext.isRuntimeJavaHomeSet = compilerJavaHome.equals(runtimeJavaHome) == false project.rootProject.ext.javaVersions = javaVersions project.rootProject.ext.buildChecksDone = true project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion @@ -231,6 +232,7 @@ class BuildPlugin implements Plugin { project.ext.runtimeJavaHome = project.rootProject.ext.runtimeJavaHome project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion + project.ext.isRuntimeJavaHomeSet = project.rootProject.ext.isRuntimeJavaHomeSet project.ext.javaVersions = project.rootProject.ext.javaVersions project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm project.ext.gradleJavaVersion = project.rootProject.ext.gradleJavaVersion diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 9cebed6c416..49f8706ab9c 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -681,7 +681,13 @@ class ClusterFormationTasks { static Task configureExecTask(String name, Project project, Task setup, NodeInfo node, Object[] execArgs) { return project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { Exec exec -> exec.workingDir node.cwd - exec.environment 'JAVA_HOME', node.getJavaHome() + // TODO: this must change to 7.0.0 after bundling java has been backported + if (project.isRuntimeJavaHomeSet || node.nodeVersion.before(Version.fromString("8.0.0"))) { + exec.environment.put('JAVA_HOME', project.runtimeJavaHome) + } else { + // force JAVA_HOME to *not* be set + exec.environment.remove('JAVA_HOME') + } if (Os.isFamily(Os.FAMILY_WINDOWS)) { exec.executable 'cmd' exec.args '/C', 'call' @@ -698,8 +704,12 @@ class ClusterFormationTasks { static Task configureStartTask(String name, Project project, Task setup, NodeInfo node) { // this closure is converted into ant nodes by groovy's AntBuilder Closure antRunner = { AntBuilder ant -> - ant.exec(executable: node.executable, spawn: node.config.daemonize, dir: node.cwd, taskname: 'elasticsearch') { + ant.exec(executable: node.executable, spawn: node.config.daemonize, newenvironment: true, + dir: node.cwd, taskname: 'elasticsearch') { node.env.each { key, value -> env(key: key, value: value) } + if (project.isRuntimeJavaHomeSet || node.nodeVersion.before(Version.fromString("8.0.0"))) { + env(key: 'JAVA_HOME', value: project.runtimeJavaHome) + } node.args.each { arg(value: it) } } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy index 63af1dda03c..ae365038ccf 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy @@ -23,7 +23,6 @@ import com.sun.jna.Native import com.sun.jna.WString import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.Version -import org.gradle.api.InvalidUserDataException import org.gradle.api.Project import java.nio.file.Files @@ -240,11 +239,6 @@ class NodeInfo { return Native.toString(shortPath).substring(4) } - /** Return the java home used by this node. */ - String getJavaHome() { - return javaVersion == null ? project.runtimeJavaHome : project.javaVersions.get(javaVersion) - } - /** Returns debug string for the command that started this node. */ String getCommandString() { String esCommandString = "\nNode ${nodeNum} configuration:\n" @@ -252,7 +246,6 @@ class NodeInfo { esCommandString += "| cwd: ${cwd}\n" esCommandString += "| command: ${executable} ${args.join(' ')}\n" esCommandString += '| environment:\n' - esCommandString += "| JAVA_HOME: ${javaHome}\n" env.each { k, v -> esCommandString += "| ${k}: ${v}\n" } if (config.daemonize) { esCommandString += "|\n| [${wrapperScript.name}]\n" diff --git a/buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/VersionProperties.java b/buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/VersionProperties.java index 23ac9458b96..cdb0f01cf75 100644 --- a/buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/VersionProperties.java +++ b/buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/VersionProperties.java @@ -10,6 +10,7 @@ import java.util.Properties; * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions. */ public class VersionProperties { + public static String getElasticsearch() { return elasticsearch; } @@ -18,17 +19,25 @@ public class VersionProperties { return lucene; } + public static String getBundledJdk() { + return bundledJdk; + } + public static Map getVersions() { return versions; } private static final String elasticsearch; private static final String lucene; + private static final String bundledJdk; private static final Map versions = new HashMap(); + static { Properties props = getVersionProperties(); elasticsearch = props.getProperty("elasticsearch"); lucene = props.getProperty("lucene"); + bundledJdk = props.getProperty("bundled_jdk"); + for (String property : props.stringPropertyNames()) { versions.put(property, props.getProperty(property)); } diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 5ca70a21127..4e6563ff4a2 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,6 +1,8 @@ elasticsearch = 7.1.0 lucene = 8.0.0-snapshot-ff9509a8df +bundled_jdk = 11.0.2+9 + # optional dependencies spatial4j = 0.7 jts = 1.15.0 diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 98b2f6bd006..cc09d741a4e 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -45,7 +45,7 @@ task createPluginsDir(type: EmptyDirTask) { dirMode 0755 } -CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, boolean oss) { +CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss) { return copySpec { into("elasticsearch-${version}") { into('lib') { @@ -59,6 +59,11 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, boolean os into('bin') { with binFiles(distributionType, oss) } + if (platform != null) { + into('jdk') { + with jdkFiles(platform) + } + } into('') { from { dirMode 0755 @@ -102,19 +107,19 @@ Closure commonZipConfig = { task buildIntegTestZip(type: Zip) { configure(commonZipConfig) - with archiveFiles(transportModulesFiles, 'zip', true) + with archiveFiles(transportModulesFiles, 'zip', null, true) } task buildWindowsZip(type: Zip) { configure(commonZipConfig) archiveClassifier = 'windows-x86_64' - with archiveFiles(modulesFiles(false), 'zip', false) + with archiveFiles(modulesFiles(false), 'zip', 'windows', false) } task buildOssWindowsZip(type: Zip) { configure(commonZipConfig) archiveClassifier = 'windows-x86_64' - with archiveFiles(modulesFiles(true), 'zip', true) + with archiveFiles(modulesFiles(true), 'zip', 'windows', true) } Closure commonTarConfig = { @@ -127,25 +132,25 @@ Closure commonTarConfig = { task buildDarwinTar(type: Tar) { configure(commonTarConfig) archiveClassifier = 'darwin-x86_64' - with archiveFiles(modulesFiles(false), 'tar', false) + with archiveFiles(modulesFiles(false), 'tar', 'darwin', false) } task buildOssDarwinTar(type: Tar) { configure(commonTarConfig) archiveClassifier = 'darwin-x86_64' - with archiveFiles(modulesFiles(true), 'tar', true) + with archiveFiles(modulesFiles(true), 'tar', 'darwin', true) } task buildLinuxTar(type: Tar) { configure(commonTarConfig) archiveClassifier = 'linux-x86_64' - with archiveFiles(modulesFiles(false), 'tar', false) + with archiveFiles(modulesFiles(false), 'tar', 'linux', false) } task buildOssLinuxTar(type: Tar) { configure(commonTarConfig) archiveClassifier = 'linux-x86_64' - with archiveFiles(modulesFiles(true), 'tar', true) + with archiveFiles(modulesFiles(true), 'tar', 'linux', true) } Closure tarExists = { it -> new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() } diff --git a/distribution/build.gradle b/distribution/build.gradle index 461031d1997..295dc35d270 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -20,11 +20,14 @@ import org.elasticsearch.gradle.ConcatFilesTask import org.elasticsearch.gradle.MavenFilteringHack import org.elasticsearch.gradle.NoticeTask +import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.test.RunTask import org.apache.tools.ant.filters.FixCrLfFilter import java.nio.file.Files import java.nio.file.Path +import java.util.regex.Matcher +import java.util.regex.Pattern /***************************************************************************** * Third party dependencies report * @@ -210,6 +213,51 @@ xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule -> copyLog4jProperties(buildDefaultLog4jConfig, xpackModule) } +/***************************************************************************** + * JDKs * + *****************************************************************************/ +// extract the bundled jdk version, broken into elements as: [feature, interim, update, build] +// Note the "patch" version is not yet handled here, as it has not yet been used by java. +Pattern JDK_VERSION = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d+)\\+(\\d+)") +Matcher jdkVersionMatcher = JDK_VERSION.matcher(VersionProperties.bundledJdk) +if (jdkVersionMatcher.matches() == false) { + throw new IllegalArgumentException("Malformed jdk version [" + VersionProperties.bundledJdk + "]") +} +String jdkVersion = jdkVersionMatcher.group(1) + '.' + jdkVersionMatcher.group(2) + '.' + jdkVersionMatcher.group(3) +String jdkMajor = jdkVersionMatcher.group(1) +String jdkBuild = jdkVersionMatcher.group(4) + +repositories { + ivy { + url "https://download.java.net" + patternLayout { + artifact "java/GA/jdk${jdkMajor}/${jdkBuild}/GPL/openjdk-[revision]_[module]-x64_bin.[ext]" + } + } +} +for (String platform : ['linux', 'darwin', 'windows']) { + String jdkConfigName = "jdk_${platform}" + Configuration jdkConfig = configurations.create(jdkConfigName) + String extension = platform.equals('windows') ? 'zip' : 'tar.gz' + dependencies.add(jdkConfigName, "jdk:${platform.equals('darwin') ? 'osx' : platform}:${jdkVersion}@${extension}") + + int rootNdx = platform.equals('darwin') ? 2 : 1 + Closure removeRootDir = { + it.eachFile { FileCopyDetails details -> + details.relativePath = new RelativePath(true, details.relativePath.segments[rootNdx..-1] as String[]) + } + it.includeEmptyDirs false + } + project.task("extract${platform.capitalize()}Jdk", type: Copy) { + into "${buildDir}/jdks/openjdk-${jdkVersion}_${platform}" + if (extension.equals('zip')) { + from({ zipTree(jdkConfig.singleFile) }, removeRootDir) + } else { + from({ tarTree(resources.gzip(jdkConfig.singleFile)) }, removeRootDir) + } + } +} + // make sure we have a clean task since we aren't a java project, but we have tasks that // put stuff in the build dir task clean(type: Delete) { @@ -332,6 +380,17 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { from buildDefaultNotice } } + + jdkFiles = { platform -> + copySpec { + from project(':distribution').tasks.getByName("extract${platform.capitalize()}Jdk") + eachFile { FileCopyDetails details -> + if (details.relativePath.segments[-2] == 'bin') { + details.mode = 0755 + } + } + } + } } } diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 6d4f8fc8893..b0b7a1d4b82 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -135,6 +135,9 @@ Closure commonPackageConfig(String type, boolean oss) { into('modules') { with modulesFiles(oss) } + into('jdk') { + with jdkFiles('linux') + } // we need to specify every intermediate directory in these paths so the package managers know they are explicitly // intended to manage them; otherwise they may be left behind on uninstallation. duplicate calls of the same // directory are fine diff --git a/distribution/packages/src/common/scripts/preinst b/distribution/packages/src/common/scripts/preinst index ffe71459c69..66e5038a55d 100644 --- a/distribution/packages/src/common/scripts/preinst +++ b/distribution/packages/src/common/scripts/preinst @@ -15,19 +15,6 @@ err_exit() { exit 1 } -# Check for these at preinst time due to failures in postinst if they do not exist -if [ -x "$JAVA_HOME/bin/java" ]; then - JAVA="$JAVA_HOME/bin/java" -elif command -v java; then - JAVA=`command -v java` -else - JAVA="" -fi - -if [ -z "$JAVA" ]; then - err_exit "could not find java; set JAVA_HOME" -fi - case "$1" in # Debian #################################################### diff --git a/distribution/src/bin/elasticsearch-env b/distribution/src/bin/elasticsearch-env index b1eb3c9d085..0b5bd6b6d58 100644 --- a/distribution/src/bin/elasticsearch-env +++ b/distribution/src/bin/elasticsearch-env @@ -36,17 +36,19 @@ ES_HOME=`dirname "$ES_HOME"` ES_CLASSPATH="$ES_HOME/lib/*" # now set the path to java -if [ -x "$JAVA_HOME/bin/java" ]; then +if [ ! -z "$JAVA_HOME" ]; then JAVA="$JAVA_HOME/bin/java" else - set +e - JAVA=`which java` - echo "warning: Falling back to java on path. This behavior is deprecated. Specify JAVA_HOME" - set -e + if [ "$(uname -s)" = "Darwin" ]; then + # OSX has a different structure + JAVA="$ES_HOME/jdk/Contents/Home/bin/java" + else + JAVA="$ES_HOME/jdk/bin/java" + fi fi if [ ! -x "$JAVA" ]; then - echo "could not find java; set JAVA_HOME" >&2 + echo "could not find java in JAVA_HOME or bundled at $JAVA" >&2 exit 1 fi diff --git a/distribution/src/bin/elasticsearch-env.bat b/distribution/src/bin/elasticsearch-env.bat index f874b85c86d..4101d186398 100644 --- a/distribution/src/bin/elasticsearch-env.bat +++ b/distribution/src/bin/elasticsearch-env.bat @@ -20,12 +20,12 @@ rem now set the path to java if defined JAVA_HOME ( set JAVA="%JAVA_HOME%\bin\java.exe" ) else ( - echo warning: Falling back to java on path. This behavior is deprecated. Specify JAVA_HOME - for %%I in (java.exe) do set JAVA="%%~$PATH:I" + set JAVA="%ES_HOME%\jdk\bin\java.exe" + set JAVA_HOME="%ES_HOME%\jdk" ) if not exist %JAVA% ( - echo could not find java; set JAVA_HOME 1>&2 + echo "could not find java in JAVA_HOME or bundled at %ES_HOME%\jdk" >&2 exit /b 1 ) diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index b3fc7f0cae9..133237ecf7e 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -30,12 +30,11 @@ import org.elasticsearch.packaging.util.Shell; import org.elasticsearch.packaging.util.Shell.Result; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; import java.util.stream.Stream; -import static java.util.stream.Collectors.joining; import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER; import static org.elasticsearch.packaging.util.Archives.installArchive; import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation; @@ -46,6 +45,7 @@ import static org.elasticsearch.packaging.util.FileUtils.append; import static org.elasticsearch.packaging.util.FileUtils.cp; import static org.elasticsearch.packaging.util.FileUtils.getTempDir; import static org.elasticsearch.packaging.util.FileUtils.mkdir; +import static org.elasticsearch.packaging.util.FileUtils.mv; import static org.elasticsearch.packaging.util.FileUtils.rm; import static org.elasticsearch.packaging.util.ServerUtils.makeRequest; import static org.hamcrest.CoreMatchers.containsString; @@ -77,46 +77,23 @@ public abstract class ArchiveTestCase extends PackagingTestCase { assertThat(r.stdout, isEmptyString()); } - public void test30AbortWhenJavaMissing() { + public void test30NoJava() { assumeThat(installation, is(notNullValue())); final Installation.Executables bin = installation.executables(); final Shell sh = new Shell(); - Platforms.onWindows(() -> { - // on windows, removing java from PATH and removing JAVA_HOME is less involved than changing the permissions of the java - // executable. we also don't check permissions in the windows scripts anyway - final String originalPath = sh.run("$Env:PATH").stdout.trim(); - final String newPath = Arrays.stream(originalPath.split(";")) - .filter(path -> path.contains("Java") == false) - .collect(joining(";")); - - // note the lack of a $ when clearing the JAVA_HOME env variable - with a $ it deletes the java home directory - // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/providers/environment-provider?view=powershell-6 - // - // this won't persist to another session so we don't have to reset anything - final Result runResult = sh.runIgnoreExitCode( - "$Env:PATH = '" + newPath + "'; " + - "Remove-Item Env:JAVA_HOME; " + - bin.elasticsearch - ); + final Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated"); + try { + mv(installation.bundledJdk, relocatedJdk); + // ask for elasticsearch version to quickly exit if java is actually found (ie test failure) + final Result runResult = sh.runIgnoreExitCode(bin.elasticsearch.toString() + " -v"); assertThat(runResult.exitCode, is(1)); - assertThat(runResult.stderr, containsString("could not find java; set JAVA_HOME")); - }); - - Platforms.onLinux(() -> { - final String javaPath = sh.run("command -v java").stdout.trim(); - - try { - sh.run("chmod -x '" + javaPath + "'"); - final Result runResult = sh.runIgnoreExitCode(bin.elasticsearch.toString()); - assertThat(runResult.exitCode, is(1)); - assertThat(runResult.stderr, containsString("could not find java; set JAVA_HOME")); - } finally { - sh.run("chmod +x '" + javaPath + "'"); - } - }); + assertThat(runResult.stderr, containsString("could not find java in JAVA_HOME or bundled")); + } finally { + mv(relocatedJdk, installation.bundledJdk); + } } public void test40CreateKeystoreManually() { @@ -160,15 +137,51 @@ public abstract class ArchiveTestCase extends PackagingTestCase { Archives.runElasticsearch(installation); - final String gcLogName = Platforms.LINUX - ? "gc.log.0.current" - : "gc.log"; - assertTrue("gc logs exist", Files.exists(installation.logs.resolve(gcLogName))); + assertTrue("gc logs exist", Files.exists(installation.logs.resolve("gc.log"))); ServerUtils.runElasticsearchTests(); Archives.stopElasticsearch(installation); } + public void assertRunsWithJavaHome() throws IOException { + Shell sh = new Shell(); + + Platforms.onLinux(() -> { + String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim(); + sh.getEnv().put("JAVA_HOME", systemJavaHome); + }); + Platforms.onWindows(() -> { + final String systemJavaHome = sh.run("$Env:SYSTEM_JAVA_HOME").stdout.trim(); + sh.getEnv().put("JAVA_HOME", systemJavaHome); + }); + + Archives.runElasticsearch(installation, sh); + ServerUtils.runElasticsearchTests(); + Archives.stopElasticsearch(installation); + + String systemJavaHome = sh.getEnv().get("JAVA_HOME"); + Path log = installation.logs.resolve("elasticsearch.log"); + assertThat(new String(Files.readAllBytes(log), StandardCharsets.UTF_8), containsString(systemJavaHome)); + } + + public void test51JavaHomeOverride() throws IOException { + assumeThat(installation, is(notNullValue())); + + assertRunsWithJavaHome(); + } + + public void test52BundledJdkRemoved() throws IOException { + assumeThat(installation, is(notNullValue())); + + Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated"); + try { + mv(installation.bundledJdk, relocatedJdk); + assertRunsWithJavaHome(); + } finally { + mv(relocatedJdk, installation.bundledJdk); + } + } + public void test60AutoCreateKeystore() { assumeThat(installation, is(notNullValue())); diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java index eff2f386897..2baeda19e3d 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java @@ -25,9 +25,10 @@ import org.elasticsearch.packaging.util.Shell.Result; import org.junit.Before; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -38,8 +39,8 @@ import static org.elasticsearch.packaging.util.Packages.assertInstalled; import static org.elasticsearch.packaging.util.Packages.assertRemoved; import static org.elasticsearch.packaging.util.Packages.install; import static org.elasticsearch.packaging.util.Packages.remove; -import static org.elasticsearch.packaging.util.Packages.runInstallCommand; import static org.elasticsearch.packaging.util.Packages.startElasticsearch; +import static org.elasticsearch.packaging.util.Packages.stopElasticsearch; import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation; import static org.elasticsearch.packaging.util.Platforms.getOsRelease; import static org.elasticsearch.packaging.util.Platforms.isSystemd; @@ -60,23 +61,6 @@ public abstract class PackageTestCase extends PackagingTestCase { assumeTrue("only compatible distributions", distribution().packaging.compatible); } - public void test05InstallFailsWhenJavaMissing() { - final Shell sh = new Shell(); - final Result javaHomeOutput = sh.run("echo $JAVA_HOME"); - - final Path javaHome = Paths.get(javaHomeOutput.stdout.trim()); - final Path originalJavaPath = javaHome.resolve("bin").resolve("java"); - final Path relocatedJavaPath = javaHome.resolve("bin").resolve("java.relocated"); - try { - mv(originalJavaPath, relocatedJavaPath); - final Result installResult = runInstallCommand(distribution()); - assertThat(installResult.exitCode, is(1)); - assertThat(installResult.stderr, containsString("could not find java; set JAVA_HOME")); - } finally { - mv(relocatedJavaPath, originalJavaPath); - } - } - public void test10InstallPackage() { assertRemoved(distribution()); installation = install(distribution()); @@ -98,6 +82,43 @@ public abstract class PackageTestCase extends PackagingTestCase { assertThat(sh.run("ps aux").stdout, not(containsString("org.elasticsearch.bootstrap.Elasticsearch"))); } + public void assertRunsWithJavaHome() throws IOException { + Shell sh = new Shell(); + + String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim(); + byte[] originalEnvFile = Files.readAllBytes(installation.envFile); + try { + Files.write(installation.envFile, ("JAVA_HOME=" + systemJavaHome + "\n").getBytes(StandardCharsets.UTF_8), + StandardOpenOption.APPEND); + startElasticsearch(); + runElasticsearchTests(); + stopElasticsearch(); + } finally { + Files.write(installation.envFile, originalEnvFile); + } + + Path log = installation.logs.resolve("elasticsearch.log"); + assertThat(new String(Files.readAllBytes(log), StandardCharsets.UTF_8), containsString(systemJavaHome)); + } + + public void test31JavaHomeOverride() throws IOException { + assumeThat(installation, is(notNullValue())); + + assertRunsWithJavaHome(); + } + + public void test42BundledJdkRemoved() throws IOException { + assumeThat(installation, is(notNullValue())); + + Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated"); + try { + mv(installation.bundledJdk, relocatedJdk); + assertRunsWithJavaHome(); + } finally { + mv(relocatedJdk, installation.bundledJdk); + } + } + public void test40StartServer() throws IOException { assumeThat(installation, is(notNullValue())); diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/WindowsServiceTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/WindowsServiceTestCase.java index 750e36bd7cf..b9536f86184 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/WindowsServiceTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/WindowsServiceTestCase.java @@ -35,9 +35,9 @@ import java.nio.file.Path; import java.util.Arrays; import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeTrue; -import static java.util.stream.Collectors.joining; import static org.elasticsearch.packaging.util.Archives.installArchive; import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation; +import static org.elasticsearch.packaging.util.FileUtils.mv; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -65,22 +65,15 @@ public abstract class WindowsServiceTestCase extends PackagingTestCase { } private Result runWithoutJava(String script) { - // on windows, removing java from PATH and removing JAVA_HOME is less involved than changing the permissions of the java - // executable. we also don't check permissions in the windows scripts anyway - final String originalPath = sh.run("$Env:PATH").stdout.trim(); - final String newPath = Arrays.stream(originalPath.split(";")) - .filter(path -> path.contains("Java") == false) - .collect(joining(";")); + final Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated"); - // note the lack of a $ when clearing the JAVA_HOME env variable - with a $ it deletes the java home directory - // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/providers/environment-provider?view=powershell-6 - // - // this won't persist to another session so we don't have to reset anything - return sh.runIgnoreExitCode( - "$Env:PATH = '" + newPath + "'; " + - "Remove-Item Env:JAVA_HOME; " + - script - ); + try { + mv(installation.bundledJdk, relocatedJdk); + // ask for elasticsearch version to quickly exit if java is actually found (ie test failure) + return sh.runIgnoreExitCode(script); + } finally { + mv(relocatedJdk, installation.bundledJdk); + } } private void assertService(String id, String status, String displayName) { @@ -135,7 +128,7 @@ public abstract class WindowsServiceTestCase extends PackagingTestCase { public void test13InstallMissingJava() throws IOException { Result result = runWithoutJava(serviceScript + " install"); assertThat(result.exitCode, equalTo(1)); - assertThat(result.stderr, containsString("could not find java; set JAVA_HOME")); + assertThat(result.stderr, containsString("could not find java in JAVA_HOME or bundled")); } public void test14RemoveNotInstalled() { diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java index 41b4fb97556..0e29baaa2c8 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java @@ -30,6 +30,7 @@ public class Installation { public final Path home; public final Path bin; // this isn't a first-class installation feature but we include it for convenience public final Path lib; // same + public final Path bundledJdk; public final Path config; public final Path data; public final Path logs; @@ -42,7 +43,7 @@ public class Installation { this.home = home; this.bin = home.resolve("bin"); this.lib = home.resolve("lib"); - + this.bundledJdk = home.resolve("jdk"); this.config = config; this.data = data; this.logs = logs; diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java index f2226bfb0c4..5adc67df2d2 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java @@ -283,4 +283,13 @@ public class Packages { sh.run("service elasticsearch status"); } } + + public static void stopElasticsearch() throws IOException { + final Shell sh = new Shell(); + if (isSystemd()) { + sh.run("systemctl stop elasticsearch.service"); + } else { + sh.run("service elasticsearch stop"); + } + } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java index 5853bc2daa1..3e4ac7869d4 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java @@ -105,6 +105,7 @@ public class Shell { ProcessBuilder builder = new ProcessBuilder(); builder.command(command); + if (workingDirectory != null) { setWorkingDirectory(builder, workingDirectory); } diff --git a/qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats b/qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats index af0c1280b2d..466dac1b7b7 100644 --- a/qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats +++ b/qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats @@ -61,7 +61,9 @@ setup() { } @test "[UPGRADE] start old version" { + export JAVA_HOME=$SYSTEM_JAVA_HOME start_elasticsearch_service + unset JAVA_HOME } @test "[UPGRADE] check elasticsearch version is old version" { diff --git a/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash index d6fdb4d7c63..8cf5d0cc349 100644 --- a/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash +++ b/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash @@ -162,22 +162,6 @@ fi remove_plugin_example } -@test "[$GROUP] fail if java executable is not found" { - [ "$GROUP" == "TAR PLUGINS" ] || skip "Test case only supported by TAR PLUGINS" - local JAVA=$(which java) - - sudo chmod -x $JAVA - run "$ESHOME/bin/elasticsearch-plugin" - sudo chmod +x $JAVA - - [ "$status" -eq 1 ] - local expected="could not find java; set JAVA_HOME" - [[ "$output" == *"$expected"* ]] || { - echo "Expected error message [$expected] but found: $output" - false - } -} - # Note that all of the tests from here to the end of the file expect to be run # in sequence and don't take well to being run one at a time. @test "[$GROUP] install a sample plugin" { diff --git a/qa/vagrant/src/test/resources/packaging/utils/packages.bash b/qa/vagrant/src/test/resources/packaging/utils/packages.bash index 751e45f784a..a38f36c3d14 100644 --- a/qa/vagrant/src/test/resources/packaging/utils/packages.bash +++ b/qa/vagrant/src/test/resources/packaging/utils/packages.bash @@ -100,7 +100,7 @@ install_package() { fi # pass through java home to package - echo "JAVA_HOME=\"$JAVA_HOME\"" >> $(env_file) + echo "JAVA_HOME=\"$SYSTEM_JAVA_HOME\"" >> $(env_file) } # Checks that all directories & files are correctly installed after a deb or diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 19af7a467a7..8a1d5208c17 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -289,6 +289,7 @@ public class Node implements Closeable { Constants.JVM_NAME, Constants.JAVA_VERSION, Constants.JVM_VERSION); + logger.info("JVM Home [{}]", System.getProperty("java.home")); logger.info("JVM arguments {}", Arrays.toString(jvmInfo.getInputArguments())); if (Build.CURRENT.isProductionRelease() == false) { logger.warn( diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDocCsvSpecIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDocCsvSpecIT.java index f89f801d282..6cd53d22a17 100644 --- a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDocCsvSpecIT.java +++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDocCsvSpecIT.java @@ -69,7 +69,7 @@ public class JdbcDocCsvSpecIT extends SpecBaseIntegrationTestCase { // uncomment this to printout the result set and create new CSV tests // //JdbcTestUtils.logLikeCLI(elastic, log); - JdbcAssert.assertResultSets(expected, elastic, log, true, false); + JdbcAssert.assertResultSets(expected, elastic, log, true, true); } @Override diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java index 47e0e9c8f90..7029c469d2f 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java @@ -66,6 +66,6 @@ public abstract class CsvSpecTestCase extends SpecBaseIntegrationTestCase { @Override protected void assertResults(ResultSet expected, ResultSet elastic) throws SQLException { Logger log = logEsResultSet() ? logger : null; - JdbcAssert.assertResultSets(expected, elastic, log, false, false); + JdbcAssert.assertResultSets(expected, elastic, log, false, true); } }