Always use bundled JDK for external cluster nodes when BWC testing (#51505) (#51701)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -45,12 +45,8 @@ class RestIntegTestTask extends DefaultTask {
         runner = project.tasks.create("${name}Runner", RestTestRunnerTask.class)
         super.dependsOn(runner)
 
-        project.testClusters {
-                "$name" {
-                    javaHome = BuildParams.runtimeJavaHome
-                }
-        }
-        runner.useCluster project.testClusters."$name"
+        ElasticsearchCluster cluster = project.testClusters.create(name)
+        runner.useCluster cluster
 
         runner.include('**/*IT.class')
         runner.systemProperty('tests.rest.load_packaged', 'false')
@@ -59,7 +55,7 @@ class RestIntegTestTask extends DefaultTask {
             if (System.getProperty("tests.cluster") != null) {
                 throw new IllegalArgumentException("tests.rest.cluster and tests.cluster must both be null or non-null")
             }
-            ElasticsearchCluster cluster = project.testClusters."${name}"
+
             runner.nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",")}")
             runner.nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI}")
             runner.nonInputProperties.systemProperty('tests.clustername', "${-> cluster.getName()}")

buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java

@@ -137,13 +137,25 @@ public class Jdk implements Buildable, Iterable<File> {
         return new Object() {
             @Override
             public String toString() {
-                final String platform = getPlatform();
-                final boolean isOSX = "mac".equals(platform) || "darwin".equals(platform);
-                return getPath() + (isOSX ? "/Contents/Home" : "") + "/bin/java";
+                return getHomeRoot() + "/bin/java";
             }
         };
     }
 
+    public Object getJavaHomePath() {
+        return new Object() {
+            @Override
+            public String toString() {
+                return getHomeRoot();
+            }
+        };
+    }
+
+    private String getHomeRoot() {
+        boolean isOSX = "mac".equals(getPlatform()) || "darwin".equals(getPlatform());
+        return getPath() + (isOSX ? "/Contents/Home" : "");
+    }
+
     // internal, make this jdks configuration unmodifiable
     void finalizeValues() {
         if (version.isPresent() == false) {

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.gradle.testclusters;
 
 import org.elasticsearch.gradle.FileSupplier;
+import org.elasticsearch.gradle.Jdk;
 import org.elasticsearch.gradle.PropertyNormalization;
 import org.elasticsearch.gradle.ReaperService;
 import org.elasticsearch.gradle.http.WaitForHttpResource;
@@ -58,20 +59,22 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
     private final String path;
     private final String clusterName;
     private final NamedDomainObjectContainer<ElasticsearchNode> nodes;
+    private final Jdk bwcJdk;
     private final File workingDirBase;
     private final LinkedHashMap<String, Predicate<TestClusterConfiguration>> waitConditions = new LinkedHashMap<>();
     private final Project project;
     private final ReaperService reaper;
     private int nodeIndex = 0;
 
-    public ElasticsearchCluster(String path, String clusterName, Project project, ReaperService reaper, File workingDirBase) {
-        this.path = path;
+    public ElasticsearchCluster(String clusterName, Project project, ReaperService reaper, File workingDirBase, Jdk bwcJdk) {
+        this.path = project.getPath();
         this.clusterName = clusterName;
         this.project = project;
         this.reaper = reaper;
         this.workingDirBase = workingDirBase;
         this.nodes = project.container(ElasticsearchNode.class);
-        this.nodes.add(new ElasticsearchNode(path, clusterName + "-0", project, reaper, workingDirBase));
+        this.bwcJdk = bwcJdk;
+        this.nodes.add(new ElasticsearchNode(clusterName + "-0", project, reaper, workingDirBase, bwcJdk));
         // configure the cluster name eagerly so nodes know about it
         this.nodes.all((node) -> node.defaultConfig.put("cluster.name", safeName(clusterName)));
 
@@ -92,7 +95,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
         }
 
         for (int i = nodes.size(); i < numberOfNodes; i++) {
-            this.nodes.add(new ElasticsearchNode(path, clusterName + "-" + i, project, reaper, workingDirBase));
+            this.nodes.add(new ElasticsearchNode(clusterName + "-" + i, project, reaper, workingDirBase, bwcJdk));
         }
     }
 
@@ -243,11 +246,6 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
         }
     }
 
-    @Override
-    public void setJavaHome(File javaHome) {
-        nodes.all(each -> each.setJavaHome(javaHome));
-    }
-
     @Override
     public void start() {
         commonNodeConfig();

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -21,6 +21,7 @@ package org.elasticsearch.gradle.testclusters;
 import org.elasticsearch.gradle.DistributionDownloadPlugin;
 import org.elasticsearch.gradle.ElasticsearchDistribution;
 import org.elasticsearch.gradle.FileSupplier;
+import org.elasticsearch.gradle.Jdk;
 import org.elasticsearch.gradle.LazyPropertyList;
 import org.elasticsearch.gradle.LazyPropertyMap;
 import org.elasticsearch.gradle.LoggedExec;
@@ -30,6 +31,7 @@ import org.elasticsearch.gradle.ReaperService;
 import org.elasticsearch.gradle.Version;
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.http.WaitForHttpResource;
+import org.elasticsearch.gradle.info.BuildParams;
 import org.gradle.api.Action;
 import org.gradle.api.Named;
 import org.gradle.api.NamedDomainObjectContainer;
@@ -115,6 +117,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final String name;
     private final Project project;
     private final ReaperService reaper;
+    private final Jdk bwcJdk;
     private final AtomicBoolean configurationFrozen = new AtomicBoolean(false);
     private final Path workingDir;
 
@@ -145,7 +148,6 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private int currentDistro = 0;
     private TestDistribution testDistribution;
     private List<ElasticsearchDistribution> distributions = new ArrayList<>();
-    private File javaHome;
     private volatile Process esProcess;
     private Function<String, String> nameCustomization = Function.identity();
     private boolean isWorkingDirConfigured = false;
@@ -153,11 +155,12 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private String transportPort = "0";
     private Path confPathData;
 
-    ElasticsearchNode(String path, String name, Project project, ReaperService reaper, File workingDirBase) {
-        this.path = path;
+    ElasticsearchNode(String name, Project project, ReaperService reaper, File workingDirBase, Jdk bwcJdk) {
+        this.path = project.getPath();
         this.name = name;
         this.project = project;
         this.reaper = reaper;
+        this.bwcJdk = bwcJdk;
         workingDir = workingDirBase.toPath().resolve(safeName(name)).toAbsolutePath();
         confPathRepo = workingDir.resolve("repo");
         configFile = workingDir.resolve("config/elasticsearch.yml");
@@ -373,21 +376,6 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         configurationFrozen.set(true);
     }
 
-    @Override
-    public void setJavaHome(File javaHome) {
-        requireNonNull(javaHome, "null javaHome passed when configuring test cluster `" + this + "`");
-        checkFrozen();
-        if (javaHome.exists() == false) {
-            throw new TestClustersException("java home for `" + this + "` does not exists: `" + javaHome + "`");
-        }
-        this.javaHome = javaHome;
-    }
-
-    @Internal
-    public File getJavaHome() {
-        return javaHome;
-    }
-
     /**
      * Returns a stream of lines in the generated logs similar to Files.lines
      *
@@ -675,9 +663,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
 
     private Map<String, String> getESEnvironment() {
         Map<String, String> defaultEnv = new HashMap<>();
-        if (getJavaHome() != null) {
-            defaultEnv.put("JAVA_HOME", getJavaHome().getAbsolutePath());
-        }
+        getRequiredJavaHome().ifPresent(javaHome -> defaultEnv.put("JAVA_HOME", javaHome));
         defaultEnv.put("ES_PATH_CONF", configFile.getParent().toString());
         String systemPropertiesString = "";
         if (systemProperties.isEmpty() == false) {
@@ -726,6 +712,22 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         return defaultEnv;
     }
 
+    private java.util.Optional<String> getRequiredJavaHome() {
+        // If we are testing the current version of Elasticsearch, use the configured runtime Java
+        if (getTestDistribution() == TestDistribution.INTEG_TEST || getVersion().equals(VersionProperties.getElasticsearchVersion())) {
+            return java.util.Optional.of(BuildParams.getRuntimeJavaHome()).map(File::getAbsolutePath);
+        } else if (getVersion().before("7.0.0")) {
+            return java.util.Optional.of(bwcJdk.getJavaHomePath().toString());
+        } else { // otherwise use the bundled JDK
+            return java.util.Optional.empty();
+        }
+    }
+
+    @Internal
+    Jdk getBwcJdk() {
+        return getVersion().before("7.0.0") ? bwcJdk : null;
+    }
+
     private void startElasticsearchProcess() {
         final ProcessBuilder processBuilder = new ProcessBuilder();
 

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

@@ -83,8 +83,6 @@ public interface TestClusterConfiguration {
 
     void freeze();
 
-    void setJavaHome(File javaHome);
-
     void start();
 
     void restart();

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java

@@ -1,9 +1,11 @@
 package org.elasticsearch.gradle.testclusters;
 
+import org.elasticsearch.gradle.Jdk;
 import org.gradle.api.Task;
 import org.gradle.api.tasks.Nested;
 
 import java.util.Collection;
+import java.util.concurrent.Callable;
 
 interface TestClustersAware extends Task {
 
@@ -15,7 +17,12 @@ interface TestClustersAware extends Task {
             throw new TestClustersException("Task " + getPath() + " can't use test cluster from" + " another project " + cluster);
         }
 
+        // Add configured distributions as task dependencies so they are built before starting the cluster
         cluster.getNodes().stream().flatMap(node -> node.getDistributions().stream()).forEach(distro -> dependsOn(distro.getExtracted()));
+
+        // Add legacy BWC JDK runtime as a dependency so it's downloaded before starting the cluster if necessary
+        cluster.getNodes().stream().map(node -> (Callable<Jdk>) node::getBwcJdk).forEach(this::dependsOn);
+
         getClusters().add(cluster);
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java

@@ -19,6 +19,9 @@
 package org.elasticsearch.gradle.testclusters;
 
 import org.elasticsearch.gradle.DistributionDownloadPlugin;
+import org.elasticsearch.gradle.Jdk;
+import org.elasticsearch.gradle.JdkDownloadPlugin;
+import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.ReaperPlugin;
 import org.elasticsearch.gradle.ReaperService;
 import org.elasticsearch.gradle.tool.Boilerplate;
@@ -43,17 +46,27 @@ public class TestClustersPlugin implements Plugin<Project> {
 
     private static final String LIST_TASK_NAME = "listTestClusters";
     private static final String REGISTRY_SERVICE_NAME = "testClustersRegistry";
+    private static final String LEGACY_JAVA_VENDOR = "adoptopenjdk";
+    private static final String LEGACY_JAVA_VERSION = "8u242+b08";
     private static final Logger logger = Logging.getLogger(TestClustersPlugin.class);
 
     @Override
     public void apply(Project project) {
-        project.getPlugins().apply(DistributionDownloadPlugin.class);
+        project.getPluginManager().apply(DistributionDownloadPlugin.class);
+        project.getPluginManager().apply(JdkDownloadPlugin.class);
         project.getRootProject().getPluginManager().apply(ReaperPlugin.class);
 
         ReaperService reaper = project.getRootProject().getExtensions().getByType(ReaperService.class);
 
+        // register legacy jdk distribution for testing pre-7.0 BWC clusters
+        Jdk bwcJdk = JdkDownloadPlugin.getContainer(project).create("bwc_jdk", jdk -> {
+            jdk.setVendor(LEGACY_JAVA_VENDOR);
+            jdk.setVersion(LEGACY_JAVA_VERSION);
+            jdk.setPlatform(OS.current().name().toLowerCase());
+        });
+
         // enable the DSL to describe clusters
-        NamedDomainObjectContainer<ElasticsearchCluster> container = createTestClustersContainerExtension(project, reaper);
+        NamedDomainObjectContainer<ElasticsearchCluster> container = createTestClustersContainerExtension(project, reaper, bwcJdk);
 
         // provide a task to be able to list defined clusters.
         createListClustersTask(project, container);
@@ -74,11 +87,15 @@ public class TestClustersPlugin implements Plugin<Project> {
         project.getRootProject().getPluginManager().apply(TestClustersHookPlugin.class);
     }
 
-    private NamedDomainObjectContainer<ElasticsearchCluster> createTestClustersContainerExtension(Project project, ReaperService reaper) {
+    private NamedDomainObjectContainer<ElasticsearchCluster> createTestClustersContainerExtension(
+        Project project,
+        ReaperService reaper,
+        Jdk bwcJdk
+    ) {
         // Create an extensions that allows describing clusters
         NamedDomainObjectContainer<ElasticsearchCluster> container = project.container(
             ElasticsearchCluster.class,
-            name -> new ElasticsearchCluster(project.getPath(), name, project, reaper, new File(project.getBuildDir(), "testclusters"))
+            name -> new ElasticsearchCluster(name, project, reaper, new File(project.getBuildDir(), "testclusters"), bwcJdk)
         );
         project.getExtensions().add(EXTENSION_NAME, container);
         return container;

qa/full-cluster-restart/build.gradle

@@ -42,7 +42,6 @@ for (Version bwcVersion : bwcVersions.indexCompatible) {
       setting 'indices.memory.shard_inactive_time', '60m'
       setting 'http.content_type.required', 'true'
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-      javaHome = BuildParams.runtimeJavaHome
     }
   }
 

qa/mixed-cluster/build.gradle

@@ -59,7 +59,6 @@ for (Version bwcVersion : bwcVersions.wireCompatible) {
       numberOfNodes = 4
 
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-      javaHome = BuildParams.runtimeJavaHome
     }
   }
 

qa/repository-multi-version/build.gradle

@@ -43,7 +43,6 @@ for (Version bwcVersion : bwcVersions.indexCompatible) {
       version = v
       numberOfNodes = 2
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-      javaHome = BuildParams.runtimeJavaHome
     }
   }
 

qa/rolling-upgrade/build.gradle

@@ -70,7 +70,6 @@ for (Version bwcVersion : bwcVersions.wireCompatible) {
       setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
       setting 'http.content_type.required', 'true'
-      javaHome = BuildParams.runtimeJavaHome
     }
   }
 

qa/verify-version-constants/build.gradle

@@ -37,7 +37,6 @@ for (Version bwcVersion : bwcVersions.indexCompatible) {
     "${baseName}" {
       version = bwcVersion.toString()
       setting 'http.content_type.required', 'true'
-      javaHome = BuildParams.runtimeJavaHome
     }
   }
 

x-pack/qa/full-cluster-restart/build.gradle

@@ -66,7 +66,6 @@ for (Version bwcVersion : bwcVersions.indexCompatible) {
       versions = [bwcVersion.toString(), project.version]
       numberOfNodes = 2
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-      javaHome = BuildParams.runtimeJavaHome
       user username: "test_user", password: "x-pack-test-password"
 
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"

x-pack/qa/rolling-upgrade-basic/build.gradle

@@ -48,7 +48,6 @@ for (Version bwcVersion : bwcVersions.wireCompatible) {
       setting 'xpack.ml.enabled', 'false'
       setting 'xpack.watcher.enabled', 'false'
       setting 'xpack.license.self_generated.type', 'basic'
-      javaHome = BuildParams.runtimeJavaHome
     }
   }
 

x-pack/qa/rolling-upgrade-multi-cluster/build.gradle

@@ -53,8 +53,6 @@ for (Version bwcVersion : bwcVersions.wireCompatible) {
     setting 'xpack.ml.enabled', 'false'
     setting 'xpack.watcher.enabled', 'false'
     setting 'xpack.license.self_generated.type', 'trial'
-
-    javaHome = BuildParams.runtimeJavaHome
   }
 
   tasks.withType(RestTestRunnerTask).matching { it.name.startsWith(baseName) }.configureEach {

x-pack/qa/rolling-upgrade/build.gradle

@@ -101,7 +101,6 @@ for (Version bwcVersion : bwcVersions.wireCompatible) {
         jvmArgs '-da:org.elasticsearch.xpack.monitoring.exporter.http.HttpExportBulk'
       }
 
-      javaHome = BuildParams.runtimeJavaHome
       setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG'
     }
   }