diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java index 9b07e5dcbb3..a7d85549404 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java @@ -300,27 +300,47 @@ public class DistributionDownloadPlugin implements Plugin { return projectPath; } + /** + * Works out the gradle project name that provides a distribution artifact. + * + * @param distribution the distribution from which to derive a project name + * @return the name of a project. It is not the full project path, only the name. + */ private static String distributionProjectName(ElasticsearchDistribution distribution) { + Platform platform = distribution.getPlatform(); + Architecture architecture = distribution.getArchitecture(); String projectName = ""; + + final String archString = platform == Platform.WINDOWS || architecture == Architecture.X64 + ? "" + : "-" + architecture.toString().toLowerCase(); + if (distribution.getFlavor() == Flavor.OSS) { projectName += "oss-"; } + if (distribution.getBundledJdk() == false) { projectName += "no-jdk-"; } - if (distribution.getType() == Type.ARCHIVE) { - if (Version.fromString(distribution.getVersion()).onOrAfter("7.0.0")) { - Platform platform = distribution.getPlatform(); - projectName += platform.toString() + (platform == Platform.WINDOWS ? "-zip" : "-tar"); - } else { - projectName = distribution.getFlavor().equals(Flavor.DEFAULT) ? "zip" : "oss-zip"; - } - } else if (distribution.getType() == Type.DOCKER) { - projectName += "docker-export"; - } else { - projectName += distribution.getType(); + switch (distribution.getType()) { + case ARCHIVE: + if (Version.fromString(distribution.getVersion()).onOrAfter("7.0.0")) { + projectName += platform.toString() + archString + (platform == Platform.WINDOWS ? "-zip" : "-tar"); + } else { + projectName = distribution.getFlavor().equals(Flavor.DEFAULT) ? "zip" : "oss-zip"; + } + break; + + case DOCKER: + projectName += "docker" + archString + "-export"; + break; + + default: + projectName += distribution.getType(); + break; } + return projectName; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java index c91ebc28c78..3605e8029ae 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java @@ -118,6 +118,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable { final Configuration configuration; private final Extracted extracted; + private final Property architecture; private final Property version; private final Property type; private final Property platform; @@ -135,6 +136,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable { this.name = name; this.dockerSupport = dockerSupport; this.configuration = fileConfiguration; + this.architecture = objectFactory.property(Architecture.class); this.version = objectFactory.property(String.class).convention(VersionProperties.getElasticsearch()); this.type = objectFactory.property(Type.class); this.type.convention(Type.ARCHIVE); @@ -198,6 +200,14 @@ public class ElasticsearchDistribution implements Buildable, Iterable { this.failIfUnavailable.set(failIfUnavailable); } + public void setArchitecture(Architecture architecture) { + this.architecture.set(architecture); + } + + public Architecture getArchitecture() { + return this.architecture.get(); + } + @Override public String toString() { return configuration.getSingleFile().toString(); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java index a6560cdc00d..1598df105ea 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java @@ -19,6 +19,7 @@ package org.elasticsearch.gradle.test; +import org.elasticsearch.gradle.Architecture; import org.elasticsearch.gradle.BwcVersions; import org.elasticsearch.gradle.DistributionDownloadPlugin; import org.elasticsearch.gradle.ElasticsearchDistribution; @@ -108,11 +109,13 @@ public class DistroTestPlugin implements Plugin { Map> lifecyleTasks = lifecyleTasks(project, "destructiveDistroTest"); TaskProvider destructiveDistroTest = project.getTasks().register("destructiveDistroTest"); + for (ElasticsearchDistribution distribution : distributions) { TaskProvider destructiveTask = configureDistroTest(project, distribution, dockerSupport); destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask)); lifecyleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask)); } + Map> batsTests = new HashMap<>(); configureBatsTest(project, "plugins", distributionsDir, copyDistributionsTask, copyPluginsTask).configure( t -> t.setPluginsDir(pluginsDir) @@ -349,6 +352,8 @@ public class DistroTestPlugin implements Plugin { return project.getTasks().register(destructiveDistroTestTaskName(distribution), Test.class, t -> { // Disable Docker distribution tests unless a Docker installation is available t.onlyIf(t2 -> distribution.getType() != Type.DOCKER || dockerSupport.get().getDockerAvailability().isAvailable); + // Only run tests for the current architecture + t.onlyIf(t3 -> distribution.getArchitecture() == Architecture.current()); t.getOutputs().doNotCacheIf("Build cache is disabled for packaging tests", Specs.satisfyAll()); t.setMaxParallelForks(1); t.setWorkingDir(project.getProjectDir()); @@ -382,47 +387,68 @@ public class DistroTestPlugin implements Plugin { List currentDistros = new ArrayList<>(); List upgradeDistros = new ArrayList<>(); - for (Type type : Arrays.asList(Type.DEB, Type.RPM, Type.DOCKER)) { - for (Flavor flavor : Flavor.values()) { - for (boolean bundledJdk : Arrays.asList(true, false)) { - // All our Docker images include a bundled JDK so it doesn't make sense to test without one - boolean skip = type == Type.DOCKER && bundledJdk == false; + for (Architecture architecture : Architecture.values()) { + for (Type type : Arrays.asList(Type.DEB, Type.RPM, Type.DOCKER)) { + for (Flavor flavor : Flavor.values()) { + for (boolean bundledJdk : Arrays.asList(true, false)) { + // All our Docker images include a bundled JDK so it doesn't make sense to test without one. + // Also we'll never publish an ARM (aarch64) build without a bundled JDK. + boolean skip = bundledJdk == false && (type == Type.DOCKER || architecture == Architecture.AARCH64); - if (skip == false) { - addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch(), currentDistros); + if (skip == false) { + addDistro( + distributions, + architecture, + type, + null, + flavor, + bundledJdk, + VersionProperties.getElasticsearch(), + currentDistros + ); + } } } - } - // We don't configure distributions for prior versions for Docker. This is because doing - // so prompts Gradle to try and resolve the Docker dependencies, which doesn't work as - // they can't be downloaded via Ivy (configured in DistributionDownloadPlugin). Since we - // need these for the BATS upgrade tests, and those tests only cover .rpm and .deb, it's - // OK to omit creating such distributions in the first place. We may need to revisit - // this in the future, so allow upgrade testing using Docker containers. - if (type != Type.DOCKER) { - // upgrade version is always bundled jdk - // NOTE: this is mimicking the old VagrantTestPlugin upgrade behavior. It will eventually be replaced - // witha dedicated upgrade test from every bwc version like other bwc tests - addDistro(distributions, type, null, Flavor.DEFAULT, true, upgradeVersion.toString(), upgradeDistros); - if (upgradeVersion.onOrAfter("6.3.0")) { - addDistro(distributions, type, null, Flavor.OSS, true, upgradeVersion.toString(), upgradeDistros); + // We don't configure distributions for prior versions for Docker. This is because doing + // so prompts Gradle to try and resolve the Docker dependencies, which doesn't work as + // they can't be downloaded via Ivy (configured in DistributionDownloadPlugin). Since we + // need these for the BATS upgrade tests, and those tests only cover .rpm and .deb, it's + // OK to omit creating such distributions in the first place. We may need to revisit + // this in the future, so allow upgrade testing using Docker containers. + if (type != Type.DOCKER) { + // upgrade version is always bundled jdk + // NOTE: this is mimicking the old VagrantTestPlugin upgrade behavior. It will eventually be replaced + // witha dedicated upgrade test from every bwc version like other bwc tests + addDistro(distributions, architecture, type, null, Flavor.DEFAULT, true, upgradeVersion.toString(), upgradeDistros); + if (upgradeVersion.onOrAfter("6.3.0")) { + addDistro(distributions, architecture, type, null, Flavor.OSS, true, upgradeVersion.toString(), upgradeDistros); + } } } } - for (Platform platform : Arrays.asList(Platform.LINUX, Platform.WINDOWS)) { - for (Flavor flavor : Flavor.values()) { - for (boolean bundledJdk : Arrays.asList(true, false)) { - addDistro( - distributions, - Type.ARCHIVE, - platform, - flavor, - bundledJdk, - VersionProperties.getElasticsearch(), - currentDistros - ); + for (Architecture architecture : Architecture.values()) { + for (Platform platform : Arrays.asList(Platform.LINUX, Platform.WINDOWS)) { + for (Flavor flavor : Flavor.values()) { + for (boolean bundledJdk : Arrays.asList(true, false)) { + if (bundledJdk == false && architecture != Architecture.X64) { + // We will never publish distributions for non-x86 (amd64) platforms + // without a bundled JDK + continue; + } + + addDistro( + distributions, + architecture, + Type.ARCHIVE, + platform, + flavor, + bundledJdk, + VersionProperties.getElasticsearch(), + currentDistros + ); + } } } } @@ -446,6 +472,7 @@ public class DistroTestPlugin implements Plugin { private static void addDistro( NamedDomainObjectContainer distributions, + Architecture architecture, Type type, Platform platform, Flavor flavor, @@ -453,11 +480,12 @@ public class DistroTestPlugin implements Plugin { String version, List container ) { - String name = distroId(type, platform, flavor, bundledJdk) + "-" + version; + String name = distroId(type, platform, flavor, bundledJdk, architecture) + "-" + version; if (distributions.findByName(name) != null) { return; } ElasticsearchDistribution distro = distributions.create(name, d -> { + d.setArchitecture(architecture); d.setFlavor(flavor); d.setType(type); if (type == Type.ARCHIVE) { @@ -483,12 +511,18 @@ public class DistroTestPlugin implements Plugin { return project.getName().contains("windows"); } - private static String distroId(Type type, Platform platform, Flavor flavor, boolean bundledJdk) { - return flavor + "-" + (type == Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk ? "" : "-no-jdk"); + private static String distroId(Type type, Platform platform, Flavor flavor, boolean bundledJdk, Architecture architecture) { + return flavor + + "-" + + (type == Type.ARCHIVE ? platform + "-" : "") + + type + + (bundledJdk ? "" : "-no-jdk") + + (architecture == Architecture.X64 ? "" : "-" + architecture.toString().toLowerCase()); } private static String destructiveDistroTestTaskName(ElasticsearchDistribution distro) { Type type = distro.getType(); - return "destructiveDistroTest." + distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk()); + return "destructiveDistroTest." + + distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk(), distro.getArchitecture()); } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 2451d38cf0a..4dd0959a598 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.gradle.testclusters; +import org.elasticsearch.gradle.Architecture; import org.elasticsearch.gradle.DistributionDownloadPlugin; import org.elasticsearch.gradle.ElasticsearchDistribution; import org.elasticsearch.gradle.FileSupplier; @@ -225,6 +226,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { } ElasticsearchDistribution distro = container.getByName(distroName); distro.setVersion(version); + distro.setArchitecture(Architecture.current()); setDistributionType(distro, testDistribution); distributions.add(distro); } diff --git a/buildSrc/src/testKit/distribution-download/subproj/build.gradle b/buildSrc/src/testKit/distribution-download/subproj/build.gradle index 3be075d20cc..6036faf90a6 100644 --- a/buildSrc/src/testKit/distribution-download/subproj/build.gradle +++ b/buildSrc/src/testKit/distribution-download/subproj/build.gradle @@ -1,3 +1,5 @@ +import org.elasticsearch.gradle.Architecture + plugins { id 'elasticsearch.distribution-download' } @@ -7,6 +9,7 @@ String distroType = System.getProperty('tests.distro.type') String distroPlatform = System.getProperty('tests.distro.platform') String distroFlavor = System.getProperty('tests.distro.flavor') String distroBundledJdk = System.getProperty('tests.distro.bundledJdk') +String distroArch = System.getProperty('tests.distro.arch'); elasticsearch_distributions { test_distro { @@ -25,6 +28,11 @@ elasticsearch_distributions { if (distroBundledJdk != null) { bundledJdk = Boolean.parseBoolean(distroBundledJdk) } + if (distroArch != null) { + architecture = Architecture.valueOf(distroArch); + } else { + architecture = Architecture.current(); + } } } diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 8bb211d5736..8e33d66326d 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -127,12 +127,15 @@ task copyKeystore(type: Sync) { } elasticsearch_distributions { - Flavor.values().each { distroFlavor -> - "docker_$distroFlavor" { - flavor = distroFlavor - type = 'docker' - version = VersionProperties.getElasticsearch() - failIfUnavailable = false // This ensures we don't attempt to build images if docker is unavailable + Architecture.values().each { eachArchitecture -> + Flavor.values().each { distroFlavor -> + "docker_$distroFlavor${ eachArchitecture == Architecture.AARCH64 ? '_aarch64' : '' }" { + architecture = eachArchitecture + flavor = distroFlavor + type = 'docker' + version = VersionProperties.getElasticsearch() + failIfUnavailable = false // This ensures we don't attempt to build images if docker is unavailable + } } } } @@ -213,7 +216,7 @@ if (tasks.findByName("composePull")) { * that they can be easily reloaded, for example into a VM. */ subprojects { Project subProject -> - if (subProject.name.contains('docker-export')) { + if (subProject.name.endsWith('-export')) { apply plugin: 'distribution' final String architecture = subProject.name.contains('aarch64-') ? 'aarch64' : 'x64' @@ -234,6 +237,8 @@ subprojects { Project subProject -> exportDockerImageTask.dependsOn(parent.tasks.getByName(buildTaskName)) + exportDockerImageTask.onlyIf { Architecture.current().name().toLowerCase().equals(architecture) } + artifacts.add('default', file(tarFile)) { type 'tar' name "elasticsearch${"aarch64".equals(architecture) ? '-aarch64' : ''}${oss ? '-oss' : ''}" diff --git a/qa/remote-clusters/build.gradle b/qa/remote-clusters/build.gradle index cc3d3851e4d..3be39e6a7be 100644 --- a/qa/remote-clusters/build.gradle +++ b/qa/remote-clusters/build.gradle @@ -16,6 +16,8 @@ * specific language governing permissions and limitations * under the License. */ + +import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin @@ -42,6 +44,7 @@ task copyKeystore(type: Sync) { elasticsearch_distributions { docker { type = 'docker' + architecture = Architecture.current() flavor = System.getProperty('tests.distribution', 'default') version = VersionProperties.getElasticsearch() failIfUnavailable = false // This ensures we skip this testing if Docker is unavailable diff --git a/qa/wildfly/build.gradle b/qa/wildfly/build.gradle index 3a7156aa094..4b44a27cc3e 100644 --- a/qa/wildfly/build.gradle +++ b/qa/wildfly/build.gradle @@ -17,6 +17,7 @@ * under the License. */ +import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.VersionProperties apply plugin: 'war' @@ -55,6 +56,7 @@ war { elasticsearch_distributions { docker { type = 'docker' + architecture = Architecture.current() flavor = System.getProperty('tests.distribution', 'default') version = VersionProperties.getElasticsearch() failIfUnavailable = false // This ensures we skip this testing if Docker is unavailable diff --git a/settings.gradle b/settings.gradle index ba172835f3e..83af1170f48 100644 --- a/settings.gradle +++ b/settings.gradle @@ -34,6 +34,10 @@ List projects = [ 'distribution:archives:oss-no-jdk-linux-tar', 'distribution:archives:no-jdk-linux-tar', 'distribution:docker', + 'distribution:docker:docker-aarch64-build-context', + 'distribution:docker:docker-aarch64-export', + 'distribution:docker:oss-docker-aarch64-build-context', + 'distribution:docker:oss-docker-aarch64-export', 'distribution:docker:docker-build-context', 'distribution:docker:docker-export', 'distribution:docker:oss-docker-build-context',