honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Avoid using bundled jdk on unsupported platforms (#62793) 

We use the bundled jdk for unit, integ and packaging tests. Since
upgrading to jdk 15, centos-6 and oracle enterprise linux 6 have failed
due to versions of glibc no longer supported by the jdk. This commit
adds detection of the old glibc versions to gradle, and utilizes that
when deciding which jdk to use for tests.

relates #62709
closes #62635
This commit is contained in:
Ryan Ernst 2020-09-23 16:55:47 -07:00 committed by GitHub
parent f971146de4
commit 1c26926dea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 87 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
buildSrc/src/main/java/org/elasticsearch/gradle/info/BuildParams.java
View File

@ -49,6 +49,7 @@ public class BuildParams {
private static Integer defaultParallel;
private static Boolean isSnapshotBuild;
private static BwcVersions bwcVersions;
private static Boolean isBundledJdkSupported;
/**
* Initialize global build parameters. This method accepts and a initialization function which in turn accepts a
@ -134,6 +135,10 @@ public class BuildParams {
return value(BuildParams.isSnapshotBuild);
}
public static boolean isBundledJdkSupported() {
return value(BuildParams.isBundledJdkSupported);
}
private static <T> T value(T object) {
if (object == null) {
String callingMethod = Thread.currentThread().getStackTrace()[2].getMethodName();
@ -246,5 +251,9 @@ public class BuildParams {
public void setBwcVersions(BwcVersions bwcVersions) {
BuildParams.bwcVersions = requireNonNull(bwcVersions);
}
public void setIsBundledJdkSupported(boolean isBundledJdkSupported) {
BuildParams.isBundledJdkSupported = isBundledJdkSupported;
}
}
}

29
buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.gradle.info;
import org.apache.commons.io.IOUtils;
import org.apache.tools.ant.taskdefs.condition.Os;
import org.elasticsearch.gradle.BwcVersions;
import org.elasticsearch.gradle.OS;
import org.elasticsearch.gradle.util.Util;
@ -67,6 +68,7 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class);
private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "server/src/main/java/org/elasticsearch/Version.java";
private static Integer _defaultParallel = null;
private static Boolean _isBundledJdkSupported = null;
private final JavaInstallationRegistry javaInstallationRegistry;
private final ObjectFactory objects;
@ -119,6 +121,7 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
params.setDefaultParallel(findDefaultParallel(project));
params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false));
params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true));
params.setIsBundledJdkSupported(findIfBundledJdkSupported(project));
if (isInternal) {
params.setBwcVersions(resolveBwcVersions(rootDir));
}
@ -276,6 +279,32 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
return versionedJavaHome;
}
private static boolean findIfBundledJdkSupported(Project project) {
if (_isBundledJdkSupported == null) {
if (Os.isFamily(Os.FAMILY_UNIX) == false || Os.isFamily(Os.FAMILY_MAC)) {
_isBundledJdkSupported = true;
} else {
// check if glibc version can support java 15+
ByteArrayOutputStream stdout = new ByteArrayOutputStream();
project.exec(spec -> {
spec.setCommandLine("getconf", "GNU_LIBC_VERSION");
spec.setStandardOutput(stdout);
});
String version = stdout.toString().trim();
final int[] glibcVersion;
try {
String[] parts = version.split(" ")[1].split("\\.");
glibcVersion = new int[] { Integer.parseInt(parts[0]), Integer.parseInt(parts[1]) };
} catch (Exception e) {
throw new IllegalStateException("Could not parse glibc version from " + version, e);
}
// as of java 15, java requires GLIBC 2.14+
_isBundledJdkSupported = glibcVersion[0] == 2 && glibcVersion[1] >= 14 || glibcVersion[0] > 2;
}
}
return _isBundledJdkSupported;
}
private static String getJavaHomeEnvVarName(String version) {
return "JAVA" + version + "_HOME";
}

24
buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
View File

@ -77,6 +77,7 @@ public class DistroTestPlugin implements Plugin<Project> {
private static final String DISTRIBUTION_SYSPROP = "tests.distribution";
private static final String BWC_DISTRIBUTION_SYSPROP = "tests.bwc-distribution";
private static final String EXAMPLE_PLUGIN_SYSPROP = "tests.example-plugin";
private static final String IS_BUNDLED_JDK_SUPPORTED = "tests.is_bundled_jdk_supported";
@Override
public void apply(Project project) {
@ -112,8 +113,8 @@ public class DistroTestPlugin implements Plugin<Project> {
depsTasks.put(taskname, depsTask);
TaskProvider<Test> destructiveTask = configureTestTask(project, taskname, distribution, t -> {
t.onlyIf(t2 -> distribution.isDocker() == false || dockerSupport.get().getDockerAvailability().isAvailable);
addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath);
addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString());
addSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath);
addSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString());
t.exclude("**/PackageUpgradeTests.class");
}, depsTask);
@ -151,8 +152,8 @@ public class DistroTestPlugin implements Plugin<Project> {
upgradeDepsTask.configure(t -> t.dependsOn(distribution, bwcDistro));
depsTasks.put(upgradeTaskname, upgradeDepsTask);
TaskProvider<Test> upgradeTest = configureTestTask(project, upgradeTaskname, distribution, t -> {
addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath);
addDistributionSysprop(t, BWC_DISTRIBUTION_SYSPROP, bwcDistro::getFilepath);
addSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath);
addSysprop(t, BWC_DISTRIBUTION_SYSPROP, bwcDistro::getFilepath);
t.include("**/PackageUpgradeTests.class");
}, upgradeDepsTask);
versionTasks.get(version.toString()).configure(t -> t.dependsOn(upgradeTest));
@ -161,6 +162,12 @@ public class DistroTestPlugin implements Plugin<Project> {
}
}
project.getTasks()
.withType(
Test.class,
t -> addSysprop(t, IS_BUNDLED_JDK_SUPPORTED, () -> Boolean.toString(BuildParams.isBundledJdkSupported()))
);
// setup jdks used by no-jdk tests, and by gradle executing
TaskProvider<Copy> linuxGradleJdk = createJdk(project, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, "linux", "x64");
TaskProvider<Copy> linuxSystemJdk = createJdk(project, "system", SYSTEM_JDK_VENDOR, SYSTEM_JDK_VERSION, "linux", "x64");
@ -281,9 +288,12 @@ public class DistroTestPlugin implements Plugin<Project> {
// setup VM used by these tests
VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class);
vagrant.setBox(box);
vagrant.vmEnv("SYSTEM_JAVA_HOME", convertPath(project, vagrant, systemJdkProvider, "", ""));
vagrant.vmEnv("JAVA_HOME", ""); // make sure any default java on the system is ignored
vagrant.vmEnv("JAVA_HOME", convertPath(project, vagrant, gradleJdkProvider, "", "")); // make sure any default java on the system is
// ignored
// also set RUNTIME_JAVA_HOME, not because it is used, but to ensure the bundled jdk version is not loaded by gradle on legacy
// systems
vagrant.vmEnv("RUNTIME_JAVA_HOME", convertPath(project, vagrant, gradleJdkProvider, "", ""));
vagrant.vmEnv("PATH", convertPath(project, vagrant, gradleJdkProvider, "/bin:$PATH", "\\bin;$Env:PATH"));
// pass these along to get correct build scans
if (System.getenv("JENKINS_URL") != null) {
@ -482,7 +492,7 @@ public class DistroTestPlugin implements Plugin<Project> {
+ distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk(), distro.getArchitecture());
}
private static void addDistributionSysprop(Test task, String sysprop, Supplier<String> valueSupplier) {
private static void addSysprop(Test task, String sysprop, Supplier<String> valueSupplier) {
SystemPropertyCommandLineArgumentProvider props = task.getExtensions().getByType(SystemPropertyCommandLineArgumentProvider.class);
props.systemProperty(sysprop, valueSupplier);
}

4
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
View File

@ -772,7 +772,9 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private java.util.Optional<String> getRequiredJavaHome() {
// If we are testing the current version of Elasticsearch, use the configured runtime Java
if (getTestDistribution() == TestDistribution.INTEG_TEST || getVersion().equals(VersionProperties.getElasticsearchVersion())) {
if (getTestDistribution() == TestDistribution.INTEG_TEST
|| getVersion().equals(VersionProperties.getElasticsearchVersion())
|| BuildParams.isBundledJdkSupported() == false) {
return java.util.Optional.of(BuildParams.getRuntimeJavaHome()).map(File::getAbsolutePath);
} else if (getVersion().before("7.0.0")) {
return java.util.Optional.of(bwcJdk.getJavaHomePath().toString());

31
gradle/runtime-jdk-provision.gradle
View File

@ -5,22 +5,29 @@ import org.elasticsearch.gradle.info.BuildParams
apply plugin: 'elasticsearch.jdk-download'
jdks {
provisioned_runtime {
vendor = VersionProperties.bundledJdkVendor
version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase())
platform = OS.current().name().toLowerCase()
architecture = Architecture.current().name().toLowerCase()
if (BuildParams.getIsRuntimeJavaHomeSet()) {
configure(allprojects - project(':build-tools')) {
project.tasks.withType(Test).configureEach { Test test ->
if (BuildParams.getIsRuntimeJavaHomeSet()) {
test.executable = "${BuildParams.runtimeJavaHome}/bin/java"
}
}
}
} else if (BuildParams.isBundledJdkSupported()) {
jdks {
provisioned_runtime {
vendor = VersionProperties.bundledJdkVendor
version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase())
platform = OS.current().name().toLowerCase()
architecture = Architecture.current().name().toLowerCase()
}
}
}
configure(allprojects - project(':build-tools')) {
project.tasks.withType(Test).configureEach { Test test ->
if (BuildParams.getIsRuntimeJavaHomeSet()) {
test.executable = "${BuildParams.runtimeJavaHome}/bin/java"
} else {
configure(allprojects - project(':build-tools')) {
project.tasks.withType(Test).configureEach { Test test ->
test.dependsOn(rootProject.jdks.provisioned_runtime)
test.executable = rootProject.jdks.provisioned_runtime.getBinJavaPath()
}
}
}
// if neither condition above is executed, tests will use the gradle jvm

5
qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java
View File

@ -48,7 +48,10 @@ public class Distribution {
this.platform = filename.contains("windows") ? Platform.WINDOWS : Platform.LINUX;
this.flavor = filename.contains("oss") ? Flavor.OSS : Flavor.DEFAULT;
this.hasJdk = filename.contains("no-jdk") == false;
// even if a bundled jdk exists in the distribution, it is not supported on some legacy platforms.
// the distribution here acts like the bundled jdk doesn't exist because many tests use this flag
// to determine whether to test certain aspects of the bundled jdk behavior
this.hasJdk = filename.contains("no-jdk") == false && Platforms.IS_BUNDLED_JDK_SUPPORTED;
String version = filename.split("-", 3)[1];
if (filename.contains("-SNAPSHOT")) {
version += "-SNAPSHOT";

6
qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.packaging.util;
import org.elasticsearch.common.Booleans;
import java.nio.file.Paths;
import static org.elasticsearch.packaging.util.FileUtils.slurp;
@ -30,6 +32,10 @@ public class Platforms {
public static final boolean DARWIN = OS_NAME.startsWith("Mac OS X");
public static final PlatformAction NO_ACTION = () -> {};
public static final boolean IS_BUNDLED_JDK_SUPPORTED = Booleans.parseBoolean(
System.getProperty("tests.is_bundled_jdk_supported", "true")
);
public static String getOsRelease() {
if (LINUX) {
return slurp(Paths.get("/etc/os-release"));