Ground work to start up the docker image in the build (#37754)

This change adds a docker compose configuration that's used with
the `elasticsearch.test.fixtures` plugin to start up the image
and check that the TCP ports are up.

We can build on this to add other checks for culster health,
run REST tests, etc.

We can add multiple containers and configurations to the compose
file (e.x. test different env vars) and form clusters.

buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixtureExtension.java

@@ -44,5 +44,4 @@ public class TestFixtureExtension {
         fixtures.add(fixtureProject);
     }
 
-
 }

buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -20,6 +20,7 @@ package org.elasticsearch.gradle.testfixtures;
 
 import com.avast.gradle.dockercompose.ComposeExtension;
 import com.avast.gradle.dockercompose.DockerComposePlugin;
+import com.avast.gradle.dockercompose.tasks.ComposeUp;
 import org.elasticsearch.gradle.precommit.JarHellTask;
 import org.elasticsearch.gradle.precommit.TestingConventionsTasks;
 import org.elasticsearch.gradle.precommit.ThirdPartyAuditTask;
@@ -54,7 +55,6 @@ public class TestFixturesPlugin implements Plugin<Project> {
             // convenience boilerplate with build plugin
             // Can't reference tasks that are implemented in Groovy, use reflection  instead
             disableTaskByType(tasks, getTaskClass("org.elasticsearch.gradle.precommit.LicenseHeadersTask"));
-            disableTaskByType(tasks, getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"));
             disableTaskByType(tasks, ThirdPartyAuditTask.class);
             disableTaskByType(tasks, JarHellTask.class);
 
@@ -62,7 +62,6 @@ public class TestFixturesPlugin implements Plugin<Project> {
             Task preProcessFixture = project.getTasks().create("preProcessFixture");
             buildFixture.dependsOn(preProcessFixture);
             Task postProcessFixture = project.getTasks().create("postProcessFixture");
-            buildFixture.dependsOn(postProcessFixture);
 
             if (dockerComposeSupported(project) == false) {
                 preProcessFixture.setEnabled(false);
@@ -83,7 +82,7 @@ public class TestFixturesPlugin implements Plugin<Project> {
 
             buildFixture.dependsOn(tasks.getByName("composeUp"));
             tasks.getByName("composeUp").mustRunAfter(preProcessFixture);
-            postProcessFixture.dependsOn("composeUp");
+            postProcessFixture.dependsOn(buildFixture);
 
             configureServiceInfoForTask(
                 postProcessFixture,
@@ -91,38 +90,36 @@ public class TestFixturesPlugin implements Plugin<Project> {
                 (name, port) -> postProcessFixture.getExtensions()
                     .getByType(ExtraPropertiesExtension.class).set(name, port)
             );
-        } else {
-            extension.fixtures.all(fixtureProject -> project.evaluationDependsOn(fixtureProject.getPath()));
-            if (dockerComposeSupported(project) == false) {
-                project.getLogger().warn(
-                    "Tests for {} require docker-compose at /usr/local/bin/docker-compose or /usr/bin/docker-compose " +
-                        "but none could be found so these will be skipped", project.getPath()
-                );
-                tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task ->
-                    task.setEnabled(false)
-                );
-                // conventions are not honored when the tasks are disabled
-                tasks.withType(TestingConventionsTasks.class, task ->
-                    task.setEnabled(false)
-                );
-                return;
-            }
-            tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task ->
-                extension.fixtures.all(fixtureProject -> {
-                    fixtureProject.getTasks().matching(it->it.getName().equals("buildFixture")).all(buildFixture ->
-                        task.dependsOn(buildFixture)
-                    );
-                    fixtureProject.getTasks().matching(it->it.getName().equals("composeDown")).all(composeDown ->
-                        task.finalizedBy(composeDown)
-                    );
-                    configureServiceInfoForTask(
-                        task,
-                        fixtureProject,
-                        (name, port) -> setSystemProperty(task, name, port)
-                    );
-                })
-            );
         }
+
+        extension.fixtures.all(fixtureProject -> project.evaluationDependsOn(fixtureProject.getPath()));
+        if (dockerComposeSupported(project) == false) {
+            project.getLogger().warn(
+                "Tests for {} require docker-compose at /usr/local/bin/docker-compose or /usr/bin/docker-compose " +
+                    "but none could be found so these will be skipped", project.getPath()
+            );
+            disableTaskByType(tasks, getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"));
+            // conventions are not honored when the tasks are disabled
+            disableTaskByType(tasks, TestingConventionsTasks.class);
+            disableTaskByType(tasks, ComposeUp.class);
+            return;
+        }
+        tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task ->
+            extension.fixtures.all(fixtureProject -> {
+                fixtureProject.getTasks().matching(it -> it.getName().equals("buildFixture")).all(buildFixture ->
+                    task.dependsOn(buildFixture)
+                );
+                fixtureProject.getTasks().matching(it -> it.getName().equals("composeDown")).all(composeDown ->
+                    task.finalizedBy(composeDown)
+                );
+                configureServiceInfoForTask(
+                    task,
+                    fixtureProject,
+                    (name, port) -> setSystemProperty(task, name, port)
+                );
+            })
+        );
+
     }
 
     private void configureServiceInfoForTask(Task task, Project fixtureProject, BiConsumer<String, Integer> consumer) {
@@ -131,23 +128,24 @@ public class TestFixturesPlugin implements Plugin<Project> {
         task.doFirst(theTask ->
             fixtureProject.getExtensions().getByType(ComposeExtension.class).getServicesInfos()
                 .forEach((service, infos) -> {
-                    theTask.getLogger().info(
-                        "Port maps for {}\nTCP:{}\nUDP:{}\nexposed to {}",
-                        fixtureProject.getPath(),
-                        infos.getTcpPorts(),
-                        infos.getUdpPorts(),
-                        theTask.getPath()
-                    );
                     infos.getTcpPorts()
-                        .forEach((container, host) -> consumer.accept(
+                        .forEach((container, host) -> {
-                            "test.fixtures." + service + ".tcp." + container,
+                            String name = "test.fixtures." + service + ".tcp." + container;
-                            host
+                            theTask.getLogger().info("port mapping property: {}={}", name, host);
-                        ));
+                            consumer.accept(
+                                name,
+                                host
+                            );
+                        });
                     infos.getUdpPorts()
-                        .forEach((container, host) -> consumer.accept(
+                        .forEach((container, host) -> {
-                            "test.fixtures." + service + ".udp." + container,
+                            String name = "test.fixtures." + service + ".udp." + container;
-                            host
+                            theTask.getLogger().info("port mapping property: {}={}", name, host);
-                        ));
+                            consumer.accept(
+                                name,
+                                host
+                            );
+                        });
                 })
         );
     }

distribution/docker/build.gradle

@@ -4,6 +4,7 @@ import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.VersionProperties
 
 apply plugin: 'base'
+apply plugin: 'elasticsearch.test.fixtures'
 
 configurations {
   dockerPlugins
@@ -68,6 +69,22 @@ void addCopyDockerfileTask(final boolean oss) {
   }
 }
 
+
+preProcessFixture {
+  dependsOn taskName("copy", true, "DockerContext")
+  dependsOn taskName("copy", true, "Dockerfile")
+  dependsOn taskName("copy", false, "DockerContext")
+  dependsOn taskName("copy", false, "Dockerfile")
+}
+
+postProcessFixture.doLast {
+  println "docker default distro is on port: ${ext."test.fixtures.elasticsearch-default.tcp.9200"}, " +
+          "oss is on: ${ext."test.fixtures.elasticsearch-oss.tcp.9200"}"
+}
+
+// TODO: Add some actual tests, this will just check that the TPC port in the container is up
+check.dependsOn postProcessFixture
+
 void addBuildDockerImage(final boolean oss) {
   final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: LoggedExec) {
     dependsOn taskName("copy", oss, "DockerContext")

distribution/docker/docker-compose.yml

@@ -0,0 +1,21 @@
+# Only used for testing the docker images
+version: '3'
+services:
+  elasticsearch-default:
+    build:
+      context: ./build/docker
+      dockerfile: Dockerfile
+    environment:  
+       - cluster.name=elasticsearch-default
+       - discovery.type=single-node
+    ports:
+      - "9200"
+  elasticsearch-oss:
+    build:
+      context: ./build/oss-docker
+      dockerfile: Dockerfile
+    environment:  
+       - cluster.name=elasticsearch-oss
+       - discovery.type=single-node
+    ports:
+      - "9200"

x-pack/test/smb-fixture/build.gradle

@@ -1,2 +1,4 @@
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.test.fixtures'
+
+unitTest.enabled = false