Define aarch64 packaging test tasks (#55228)

Backport of #55073. We added tasks to build an ARM distribution and Docker image, but didn't provide any way to run packaging tests against them. Add extra loops on the possible Architecture values, and skip tasks that can't be run on the current Architecture.
2020-04-15 13:39:18 +01:00 · 2020-04-15 13:39:18 +01:00 · 70616cd76a
parent 30638a0b41
commit 70616cd76a
9 changed files with 143 additions and 55 deletions
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
@ -300,27 +300,47 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
        return projectPath;
    }

+    /**
+     * Works out the gradle project name that provides a distribution artifact.
+     *
+     * @param distribution the distribution from which to derive a project name
+     * @return the name of a project. It is not the full project path, only the name.
+     */
    private static String distributionProjectName(ElasticsearchDistribution distribution) {
+        Platform platform = distribution.getPlatform();
+        Architecture architecture = distribution.getArchitecture();
        String projectName = "";
+
+        final String archString = platform == Platform.WINDOWS || architecture == Architecture.X64
+            ? ""
+            : "-" + architecture.toString().toLowerCase();
+
        if (distribution.getFlavor() == Flavor.OSS) {
            projectName += "oss-";
        }
+
        if (distribution.getBundledJdk() == false) {
            projectName += "no-jdk-";
        }

-        if (distribution.getType() == Type.ARCHIVE) {
+        switch (distribution.getType()) {
+            case ARCHIVE:
                if (Version.fromString(distribution.getVersion()).onOrAfter("7.0.0")) {
-                Platform platform = distribution.getPlatform();
-                projectName += platform.toString() + (platform == Platform.WINDOWS ? "-zip" : "-tar");
+                    projectName += platform.toString() + archString + (platform == Platform.WINDOWS ? "-zip" : "-tar");
                } else {
                    projectName = distribution.getFlavor().equals(Flavor.DEFAULT) ? "zip" : "oss-zip";
                }
-        } else if (distribution.getType() == Type.DOCKER) {
-            projectName += "docker-export";
-        } else {
+                break;
+
+            case DOCKER:
+                projectName += "docker" + archString + "-export";
+                break;
+
+            default:
                projectName += distribution.getType();
+                break;
        }
+
        return projectName;
    }

--- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
@ -118,6 +118,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
    final Configuration configuration;
    private final Extracted extracted;

+    private final Property<Architecture> architecture;
    private final Property<String> version;
    private final Property<Type> type;
    private final Property<Platform> platform;
@ -135,6 +136,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
        this.name = name;
        this.dockerSupport = dockerSupport;
        this.configuration = fileConfiguration;
+        this.architecture = objectFactory.property(Architecture.class);
        this.version = objectFactory.property(String.class).convention(VersionProperties.getElasticsearch());
        this.type = objectFactory.property(Type.class);
        this.type.convention(Type.ARCHIVE);
@ -198,6 +200,14 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
        this.failIfUnavailable.set(failIfUnavailable);
    }

+    public void setArchitecture(Architecture architecture) {
+        this.architecture.set(architecture);
+    }
+
+    public Architecture getArchitecture() {
+        return this.architecture.get();
+    }
+
    @Override
    public String toString() {
        return configuration.getSingleFile().toString();
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
@ -19,6 +19,7 @@

 package org.elasticsearch.gradle.test;

+import org.elasticsearch.gradle.Architecture;
 import org.elasticsearch.gradle.BwcVersions;
 import org.elasticsearch.gradle.DistributionDownloadPlugin;
 import org.elasticsearch.gradle.ElasticsearchDistribution;
@ -108,11 +109,13 @@ public class DistroTestPlugin implements Plugin<Project> {

        Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecyleTasks = lifecyleTasks(project, "destructiveDistroTest");
        TaskProvider<Task> destructiveDistroTest = project.getTasks().register("destructiveDistroTest");
+
        for (ElasticsearchDistribution distribution : distributions) {
            TaskProvider<?> destructiveTask = configureDistroTest(project, distribution, dockerSupport);
            destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask));
            lifecyleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask));
        }
+
        Map<String, TaskProvider<?>> batsTests = new HashMap<>();
        configureBatsTest(project, "plugins", distributionsDir, copyDistributionsTask, copyPluginsTask).configure(
            t -> t.setPluginsDir(pluginsDir)
@ -349,6 +352,8 @@ public class DistroTestPlugin implements Plugin<Project> {
        return project.getTasks().register(destructiveDistroTestTaskName(distribution), Test.class, t -> {
            // Disable Docker distribution tests unless a Docker installation is available
            t.onlyIf(t2 -> distribution.getType() != Type.DOCKER || dockerSupport.get().getDockerAvailability().isAvailable);
+            // Only run tests for the current architecture
+            t.onlyIf(t3 -> distribution.getArchitecture() == Architecture.current());
            t.getOutputs().doNotCacheIf("Build cache is disabled for packaging tests", Specs.satisfyAll());
            t.setMaxParallelForks(1);
            t.setWorkingDir(project.getProjectDir());
@ -382,14 +387,25 @@ public class DistroTestPlugin implements Plugin<Project> {
        List<ElasticsearchDistribution> currentDistros = new ArrayList<>();
        List<ElasticsearchDistribution> upgradeDistros = new ArrayList<>();

+        for (Architecture architecture : Architecture.values()) {
            for (Type type : Arrays.asList(Type.DEB, Type.RPM, Type.DOCKER)) {
                for (Flavor flavor : Flavor.values()) {
                    for (boolean bundledJdk : Arrays.asList(true, false)) {
-                    // All our Docker images include a bundled JDK so it doesn't make sense to test without one
-                    boolean skip = type == Type.DOCKER && bundledJdk == false;
+                        // All our Docker images include a bundled JDK so it doesn't make sense to test without one.
+                        // Also we'll never publish an ARM (aarch64) build without a bundled JDK.
+                        boolean skip = bundledJdk == false && (type == Type.DOCKER || architecture == Architecture.AARCH64);

                        if (skip == false) {
-                        addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch(), currentDistros);
+                            addDistro(
+                                distributions,
+                                architecture,
+                                type,
+                                null,
+                                flavor,
+                                bundledJdk,
+                                VersionProperties.getElasticsearch(),
+                                currentDistros
+                            );
                        }
                    }
                }
@ -404,18 +420,27 @@ public class DistroTestPlugin implements Plugin<Project> {
                    // upgrade version is always bundled jdk
                    // NOTE: this is mimicking the old VagrantTestPlugin upgrade behavior. It will eventually be replaced
                    // witha dedicated upgrade test from every bwc version like other bwc tests
-                addDistro(distributions, type, null, Flavor.DEFAULT, true, upgradeVersion.toString(), upgradeDistros);
+                    addDistro(distributions, architecture, type, null, Flavor.DEFAULT, true, upgradeVersion.toString(), upgradeDistros);
                    if (upgradeVersion.onOrAfter("6.3.0")) {
-                    addDistro(distributions, type, null, Flavor.OSS, true, upgradeVersion.toString(), upgradeDistros);
+                        addDistro(distributions, architecture, type, null, Flavor.OSS, true, upgradeVersion.toString(), upgradeDistros);
+                    }
                }
            }
        }

+        for (Architecture architecture : Architecture.values()) {
            for (Platform platform : Arrays.asList(Platform.LINUX, Platform.WINDOWS)) {
                for (Flavor flavor : Flavor.values()) {
                    for (boolean bundledJdk : Arrays.asList(true, false)) {
+                        if (bundledJdk == false && architecture != Architecture.X64) {
+                            // We will never publish distributions for non-x86 (amd64) platforms
+                            // without a bundled JDK
+                            continue;
+                        }
+
                        addDistro(
                            distributions,
+                            architecture,
                            Type.ARCHIVE,
                            platform,
                            flavor,
@ -426,6 +451,7 @@ public class DistroTestPlugin implements Plugin<Project> {
                    }
                }
            }
+        }

        // temporary until distro tests have one test per distro
        Configuration packagingConfig = project.getConfigurations().create(DISTRIBUTIONS_CONFIGURATION);
@ -446,6 +472,7 @@ public class DistroTestPlugin implements Plugin<Project> {

    private static void addDistro(
        NamedDomainObjectContainer<ElasticsearchDistribution> distributions,
+        Architecture architecture,
        Type type,
        Platform platform,
        Flavor flavor,
@ -453,11 +480,12 @@ public class DistroTestPlugin implements Plugin<Project> {
        String version,
        List<ElasticsearchDistribution> container
    ) {
-        String name = distroId(type, platform, flavor, bundledJdk) + "-" + version;
+        String name = distroId(type, platform, flavor, bundledJdk, architecture) + "-" + version;
        if (distributions.findByName(name) != null) {
            return;
        }
        ElasticsearchDistribution distro = distributions.create(name, d -> {
+            d.setArchitecture(architecture);
            d.setFlavor(flavor);
            d.setType(type);
            if (type == Type.ARCHIVE) {
@ -483,12 +511,18 @@ public class DistroTestPlugin implements Plugin<Project> {
        return project.getName().contains("windows");
    }

-    private static String distroId(Type type, Platform platform, Flavor flavor, boolean bundledJdk) {
-        return flavor + "-" + (type == Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk ? "" : "-no-jdk");
+    private static String distroId(Type type, Platform platform, Flavor flavor, boolean bundledJdk, Architecture architecture) {
+        return flavor
+            + "-"
+            + (type == Type.ARCHIVE ? platform + "-" : "")
+            + type
+            + (bundledJdk ? "" : "-no-jdk")
+            + (architecture == Architecture.X64 ? "" : "-" + architecture.toString().toLowerCase());
    }

    private static String destructiveDistroTestTaskName(ElasticsearchDistribution distro) {
        Type type = distro.getType();
-        return "destructiveDistroTest." + distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk());
+        return "destructiveDistroTest."
+            + distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk(), distro.getArchitecture());
    }
 }
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
@ -18,6 +18,7 @@
 */
 package org.elasticsearch.gradle.testclusters;

+import org.elasticsearch.gradle.Architecture;
 import org.elasticsearch.gradle.DistributionDownloadPlugin;
 import org.elasticsearch.gradle.ElasticsearchDistribution;
 import org.elasticsearch.gradle.FileSupplier;
@ -225,6 +226,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
        }
        ElasticsearchDistribution distro = container.getByName(distroName);
        distro.setVersion(version);
+        distro.setArchitecture(Architecture.current());
        setDistributionType(distro, testDistribution);
        distributions.add(distro);
    }
--- a/buildSrc/src/testKit/distribution-download/subproj/build.gradle
+++ b/buildSrc/src/testKit/distribution-download/subproj/build.gradle
@ -1,3 +1,5 @@
+import org.elasticsearch.gradle.Architecture
+
 plugins {
  id 'elasticsearch.distribution-download'
 }
@ -7,6 +9,7 @@ String distroType = System.getProperty('tests.distro.type')
 String distroPlatform = System.getProperty('tests.distro.platform')
 String distroFlavor = System.getProperty('tests.distro.flavor')
 String distroBundledJdk = System.getProperty('tests.distro.bundledJdk')
+String distroArch = System.getProperty('tests.distro.arch');

 elasticsearch_distributions {
  test_distro {
@ -25,6 +28,11 @@ elasticsearch_distributions {
    if (distroBundledJdk != null) {
      bundledJdk = Boolean.parseBoolean(distroBundledJdk)
    }
+    if (distroArch != null) {
+      architecture = Architecture.valueOf(distroArch);
+    } else {
+      architecture = Architecture.current();
+    }
  }
 }

--- a/distribution/docker/build.gradle
+++ b/distribution/docker/build.gradle
@ -127,8 +127,10 @@ task copyKeystore(type: Sync) {
 }

 elasticsearch_distributions {
+  Architecture.values().each { eachArchitecture ->
    Flavor.values().each { distroFlavor ->
-    "docker_$distroFlavor" {
+      "docker_$distroFlavor${ eachArchitecture == Architecture.AARCH64 ? '_aarch64' : '' }" {
+        architecture = eachArchitecture
        flavor = distroFlavor
        type = 'docker'
        version = VersionProperties.getElasticsearch()
@ -136,6 +138,7 @@ elasticsearch_distributions {
      }
    }
  }
+}

 preProcessFixture {
  dependsOn elasticsearch_distributions.docker_default, elasticsearch_distributions.docker_oss
@ -213,7 +216,7 @@ if (tasks.findByName("composePull")) {
 * that they can be easily reloaded, for example into a VM.
 */
 subprojects { Project subProject ->
-  if (subProject.name.contains('docker-export')) {
+  if (subProject.name.endsWith('-export')) {
    apply plugin: 'distribution'

    final String architecture = subProject.name.contains('aarch64-') ? 'aarch64' : 'x64'
@ -234,6 +237,8 @@ subprojects { Project subProject ->

    exportDockerImageTask.dependsOn(parent.tasks.getByName(buildTaskName))

+    exportDockerImageTask.onlyIf { Architecture.current().name().toLowerCase().equals(architecture) }
+
    artifacts.add('default', file(tarFile)) {
      type 'tar'
      name "elasticsearch${"aarch64".equals(architecture) ? '-aarch64' : ''}${oss ? '-oss' : ''}"
--- a/qa/remote-clusters/build.gradle
+++ b/qa/remote-clusters/build.gradle
@ -16,6 +16,8 @@
 * specific language governing permissions and limitations
 * under the License.
 */
+
+import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin

@ -42,6 +44,7 @@ task copyKeystore(type: Sync) {
 elasticsearch_distributions {
  docker {
    type = 'docker'
+    architecture = Architecture.current()
    flavor = System.getProperty('tests.distribution', 'default')
    version = VersionProperties.getElasticsearch()
    failIfUnavailable = false // This ensures we skip this testing if Docker is unavailable
--- a/qa/wildfly/build.gradle
+++ b/qa/wildfly/build.gradle
@ -17,6 +17,7 @@
 * under the License.
 */

+import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.VersionProperties

 apply plugin: 'war'
@ -55,6 +56,7 @@ war {
 elasticsearch_distributions {
  docker {
    type = 'docker'
+    architecture = Architecture.current()
    flavor = System.getProperty('tests.distribution', 'default')
    version = VersionProperties.getElasticsearch()
    failIfUnavailable = false // This ensures we skip this testing if Docker is unavailable
--- a/settings.gradle
+++ b/settings.gradle
@ -34,6 +34,10 @@ List projects = [
  'distribution:archives:oss-no-jdk-linux-tar',
  'distribution:archives:no-jdk-linux-tar',
  'distribution:docker',
+  'distribution:docker:docker-aarch64-build-context',
+  'distribution:docker:docker-aarch64-export',
+  'distribution:docker:oss-docker-aarch64-build-context',
+  'distribution:docker:oss-docker-aarch64-export',
  'distribution:docker:docker-build-context',
  'distribution:docker:docker-export',
  'distribution:docker:oss-docker-build-context',