From 1c26926deaef609322e6d1191523eea75935af91 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 23 Sep 2020 16:55:47 -0700 Subject: [PATCH] Avoid using bundled jdk on unsupported platforms (#62793) We use the bundled jdk for unit, integ and packaging tests. Since upgrading to jdk 15, centos-6 and oracle enterprise linux 6 have failed due to versions of glibc no longer supported by the jdk. This commit adds detection of the old glibc versions to gradle, and utilizes that when deciding which jdk to use for tests. relates #62709 closes #62635 --- .../gradle/info/BuildParams.java | 9 ++++++ .../gradle/info/GlobalBuildInfoPlugin.java | 29 +++++++++++++++++ .../gradle/test/DistroTestPlugin.java | 24 +++++++++----- .../testclusters/ElasticsearchNode.java | 4 ++- gradle/runtime-jdk-provision.gradle | 31 ++++++++++++------- .../packaging/util/Distribution.java | 5 ++- .../packaging/util/Platforms.java | 6 ++++ 7 files changed, 87 insertions(+), 21 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/BuildParams.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/BuildParams.java index 71f7fd29189..071c846034f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/BuildParams.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/BuildParams.java @@ -49,6 +49,7 @@ public class BuildParams { private static Integer defaultParallel; private static Boolean isSnapshotBuild; private static BwcVersions bwcVersions; + private static Boolean isBundledJdkSupported; /** * Initialize global build parameters. This method accepts and a initialization function which in turn accepts a @@ -134,6 +135,10 @@ public class BuildParams { return value(BuildParams.isSnapshotBuild); } + public static boolean isBundledJdkSupported() { + return value(BuildParams.isBundledJdkSupported); + } + private static T value(T object) { if (object == null) { String callingMethod = Thread.currentThread().getStackTrace()[2].getMethodName(); @@ -246,5 +251,9 @@ public class BuildParams { public void setBwcVersions(BwcVersions bwcVersions) { BuildParams.bwcVersions = requireNonNull(bwcVersions); } + + public void setIsBundledJdkSupported(boolean isBundledJdkSupported) { + BuildParams.isBundledJdkSupported = isBundledJdkSupported; + } } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java index 78ba16e11ca..223009c9b63 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java @@ -19,6 +19,7 @@ package org.elasticsearch.gradle.info; import org.apache.commons.io.IOUtils; +import org.apache.tools.ant.taskdefs.condition.Os; import org.elasticsearch.gradle.BwcVersions; import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.util.Util; @@ -67,6 +68,7 @@ public class GlobalBuildInfoPlugin implements Plugin { private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class); private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "server/src/main/java/org/elasticsearch/Version.java"; private static Integer _defaultParallel = null; + private static Boolean _isBundledJdkSupported = null; private final JavaInstallationRegistry javaInstallationRegistry; private final ObjectFactory objects; @@ -119,6 +121,7 @@ public class GlobalBuildInfoPlugin implements Plugin { params.setDefaultParallel(findDefaultParallel(project)); params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false)); params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true)); + params.setIsBundledJdkSupported(findIfBundledJdkSupported(project)); if (isInternal) { params.setBwcVersions(resolveBwcVersions(rootDir)); } @@ -276,6 +279,32 @@ public class GlobalBuildInfoPlugin implements Plugin { return versionedJavaHome; } + private static boolean findIfBundledJdkSupported(Project project) { + if (_isBundledJdkSupported == null) { + if (Os.isFamily(Os.FAMILY_UNIX) == false || Os.isFamily(Os.FAMILY_MAC)) { + _isBundledJdkSupported = true; + } else { + // check if glibc version can support java 15+ + ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + project.exec(spec -> { + spec.setCommandLine("getconf", "GNU_LIBC_VERSION"); + spec.setStandardOutput(stdout); + }); + String version = stdout.toString().trim(); + final int[] glibcVersion; + try { + String[] parts = version.split(" ")[1].split("\\."); + glibcVersion = new int[] { Integer.parseInt(parts[0]), Integer.parseInt(parts[1]) }; + } catch (Exception e) { + throw new IllegalStateException("Could not parse glibc version from " + version, e); + } + // as of java 15, java requires GLIBC 2.14+ + _isBundledJdkSupported = glibcVersion[0] == 2 && glibcVersion[1] >= 14 || glibcVersion[0] > 2; + } + } + return _isBundledJdkSupported; + } + private static String getJavaHomeEnvVarName(String version) { return "JAVA" + version + "_HOME"; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java index 4b301e7fb1d..f641e9227bf 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java @@ -77,6 +77,7 @@ public class DistroTestPlugin implements Plugin { private static final String DISTRIBUTION_SYSPROP = "tests.distribution"; private static final String BWC_DISTRIBUTION_SYSPROP = "tests.bwc-distribution"; private static final String EXAMPLE_PLUGIN_SYSPROP = "tests.example-plugin"; + private static final String IS_BUNDLED_JDK_SUPPORTED = "tests.is_bundled_jdk_supported"; @Override public void apply(Project project) { @@ -112,8 +113,8 @@ public class DistroTestPlugin implements Plugin { depsTasks.put(taskname, depsTask); TaskProvider destructiveTask = configureTestTask(project, taskname, distribution, t -> { t.onlyIf(t2 -> distribution.isDocker() == false || dockerSupport.get().getDockerAvailability().isAvailable); - addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath); - addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString()); + addSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath); + addSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString()); t.exclude("**/PackageUpgradeTests.class"); }, depsTask); @@ -151,8 +152,8 @@ public class DistroTestPlugin implements Plugin { upgradeDepsTask.configure(t -> t.dependsOn(distribution, bwcDistro)); depsTasks.put(upgradeTaskname, upgradeDepsTask); TaskProvider upgradeTest = configureTestTask(project, upgradeTaskname, distribution, t -> { - addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath); - addDistributionSysprop(t, BWC_DISTRIBUTION_SYSPROP, bwcDistro::getFilepath); + addSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath); + addSysprop(t, BWC_DISTRIBUTION_SYSPROP, bwcDistro::getFilepath); t.include("**/PackageUpgradeTests.class"); }, upgradeDepsTask); versionTasks.get(version.toString()).configure(t -> t.dependsOn(upgradeTest)); @@ -161,6 +162,12 @@ public class DistroTestPlugin implements Plugin { } } + project.getTasks() + .withType( + Test.class, + t -> addSysprop(t, IS_BUNDLED_JDK_SUPPORTED, () -> Boolean.toString(BuildParams.isBundledJdkSupported())) + ); + // setup jdks used by no-jdk tests, and by gradle executing TaskProvider linuxGradleJdk = createJdk(project, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, "linux", "x64"); TaskProvider linuxSystemJdk = createJdk(project, "system", SYSTEM_JDK_VENDOR, SYSTEM_JDK_VERSION, "linux", "x64"); @@ -281,9 +288,12 @@ public class DistroTestPlugin implements Plugin { // setup VM used by these tests VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class); vagrant.setBox(box); - vagrant.vmEnv("SYSTEM_JAVA_HOME", convertPath(project, vagrant, systemJdkProvider, "", "")); - vagrant.vmEnv("JAVA_HOME", ""); // make sure any default java on the system is ignored + vagrant.vmEnv("JAVA_HOME", convertPath(project, vagrant, gradleJdkProvider, "", "")); // make sure any default java on the system is + // ignored + // also set RUNTIME_JAVA_HOME, not because it is used, but to ensure the bundled jdk version is not loaded by gradle on legacy + // systems + vagrant.vmEnv("RUNTIME_JAVA_HOME", convertPath(project, vagrant, gradleJdkProvider, "", "")); vagrant.vmEnv("PATH", convertPath(project, vagrant, gradleJdkProvider, "/bin:$PATH", "\\bin;$Env:PATH")); // pass these along to get correct build scans if (System.getenv("JENKINS_URL") != null) { @@ -482,7 +492,7 @@ public class DistroTestPlugin implements Plugin { + distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk(), distro.getArchitecture()); } - private static void addDistributionSysprop(Test task, String sysprop, Supplier valueSupplier) { + private static void addSysprop(Test task, String sysprop, Supplier valueSupplier) { SystemPropertyCommandLineArgumentProvider props = task.getExtensions().getByType(SystemPropertyCommandLineArgumentProvider.class); props.systemProperty(sysprop, valueSupplier); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 1ee7f98dd25..fd16263cdf6 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -772,7 +772,9 @@ public class ElasticsearchNode implements TestClusterConfiguration { private java.util.Optional getRequiredJavaHome() { // If we are testing the current version of Elasticsearch, use the configured runtime Java - if (getTestDistribution() == TestDistribution.INTEG_TEST || getVersion().equals(VersionProperties.getElasticsearchVersion())) { + if (getTestDistribution() == TestDistribution.INTEG_TEST + || getVersion().equals(VersionProperties.getElasticsearchVersion()) + || BuildParams.isBundledJdkSupported() == false) { return java.util.Optional.of(BuildParams.getRuntimeJavaHome()).map(File::getAbsolutePath); } else if (getVersion().before("7.0.0")) { return java.util.Optional.of(bwcJdk.getJavaHomePath().toString()); diff --git a/gradle/runtime-jdk-provision.gradle b/gradle/runtime-jdk-provision.gradle index 4c346c450dc..07f306de306 100644 --- a/gradle/runtime-jdk-provision.gradle +++ b/gradle/runtime-jdk-provision.gradle @@ -5,22 +5,29 @@ import org.elasticsearch.gradle.info.BuildParams apply plugin: 'elasticsearch.jdk-download' -jdks { - provisioned_runtime { - vendor = VersionProperties.bundledJdkVendor - version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase()) - platform = OS.current().name().toLowerCase() - architecture = Architecture.current().name().toLowerCase() +if (BuildParams.getIsRuntimeJavaHomeSet()) { + configure(allprojects - project(':build-tools')) { + project.tasks.withType(Test).configureEach { Test test -> + if (BuildParams.getIsRuntimeJavaHomeSet()) { + test.executable = "${BuildParams.runtimeJavaHome}/bin/java" + } + } + } +} else if (BuildParams.isBundledJdkSupported()) { + jdks { + provisioned_runtime { + vendor = VersionProperties.bundledJdkVendor + version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase()) + platform = OS.current().name().toLowerCase() + architecture = Architecture.current().name().toLowerCase() + } } -} -configure(allprojects - project(':build-tools')) { - project.tasks.withType(Test).configureEach { Test test -> - if (BuildParams.getIsRuntimeJavaHomeSet()) { - test.executable = "${BuildParams.runtimeJavaHome}/bin/java" - } else { + configure(allprojects - project(':build-tools')) { + project.tasks.withType(Test).configureEach { Test test -> test.dependsOn(rootProject.jdks.provisioned_runtime) test.executable = rootProject.jdks.provisioned_runtime.getBinJavaPath() } } } +// if neither condition above is executed, tests will use the gradle jvm diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java index 992335c9c2e..ba087bd92f1 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -48,7 +48,10 @@ public class Distribution { this.platform = filename.contains("windows") ? Platform.WINDOWS : Platform.LINUX; this.flavor = filename.contains("oss") ? Flavor.OSS : Flavor.DEFAULT; - this.hasJdk = filename.contains("no-jdk") == false; + // even if a bundled jdk exists in the distribution, it is not supported on some legacy platforms. + // the distribution here acts like the bundled jdk doesn't exist because many tests use this flag + // to determine whether to test certain aspects of the bundled jdk behavior + this.hasJdk = filename.contains("no-jdk") == false && Platforms.IS_BUNDLED_JDK_SUPPORTED; String version = filename.split("-", 3)[1]; if (filename.contains("-SNAPSHOT")) { version += "-SNAPSHOT"; diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java index fa324690bf6..767e1366874 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java @@ -19,6 +19,8 @@ package org.elasticsearch.packaging.util; +import org.elasticsearch.common.Booleans; + import java.nio.file.Paths; import static org.elasticsearch.packaging.util.FileUtils.slurp; @@ -30,6 +32,10 @@ public class Platforms { public static final boolean DARWIN = OS_NAME.startsWith("Mac OS X"); public static final PlatformAction NO_ACTION = () -> {}; + public static final boolean IS_BUNDLED_JDK_SUPPORTED = Booleans.parseBoolean( + System.getProperty("tests.is_bundled_jdk_supported", "true") + ); + public static String getOsRelease() { if (LINUX) { return slurp(Paths.get("/etc/os-release"));