Avoid packaging / unpacking cycle when using local current distributions (#61592) (#62176)

- Extract distribution archives defaults into plugin
- Added basic test coverage
- Avoid packaging/unpackaging cycle when relying on locally build distributions
- Provide DSL for setting up distribution archives
- Cleanup archives build script

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy

@@ -0,0 +1,208 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.internal
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
+import org.apache.tools.zip.ZipEntry
+import org.apache.tools.zip.ZipFile
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.gradle.testkit.runner.BuildResult
+import org.gradle.testkit.runner.TaskOutcome
+
+class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncTest {
+
+    def setup() {
+        buildFile << """
+        import org.elasticsearch.gradle.tar.SymbolicLinkPreservingTar
+        
+        plugins {
+            id 'elasticsearch.internal-distribution-archive-setup'
+        }
+        """
+        file('someFile.txt') << "some content"
+    }
+
+    def "applies defaults to tar tasks"() {
+        given:
+        file('someFile.txt') << "some content"
+        buildFile << """
+        tasks.register('${buildTaskName}', SymbolicLinkPreservingTar) {
+            from 'someFile.txt'
+        }
+        """
+
+        when:
+        def result = gradleRunner(buildTaskName).build()
+
+        then:
+        file(expectedOutputArchivePath).exists()
+        assertTarPermissionDefaults(file(expectedOutputArchivePath))
+        assertEmptyDirTasksTriggered(result)
+
+        where:
+        buildTaskName       | expectedOutputArchivePath
+        "buildDarwinTar"    | "darwin-tar/build/distributions/elasticsearch.tar.gz"
+        "buildOssDarwinTar" | "oss-darwin-tar/build/distributions/elasticsearch-oss.tar.gz"
+    }
+
+    def "applies defaults to zip tasks"() {
+        given:
+        buildFile << """
+        tasks.register('${buildTaskName}', Zip) {
+            from 'someFile.txt'
+        }
+        """
+
+        when:
+        def result = gradleRunner(buildTaskName).build()
+
+        then:
+        file(expectedOutputArchivePath).exists()
+        assertZipPermissionDefaults(file(expectedOutputArchivePath))
+        assertEmptyDirTasksTriggered(result)
+
+        where:
+        buildTaskName       | expectedOutputArchivePath
+        "buildDarwinZip"    | "darwin-zip/build/distributions/elasticsearch.zip"
+        "buildOssDarwinZip" | "oss-darwin-zip/build/distributions/elasticsearch-oss.zip"
+    }
+
+    def "registered distribution provides archives and directory variant"() {
+        given:
+        file('someFile.txt') << "some content"
+
+        settingsFile << """
+            include ':consumer'
+            include ':producer-tar'
+        """
+
+        buildFile << """
+        import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+        import org.gradle.api.internal.artifacts.ArtifactAttributes;
+
+        distribution_archives {
+            producerTar {
+                content {
+                    project.copySpec {
+                        from 'someFile.txt'
+                    }
+                }
+            }
+        }
+        
+        project('consumer') { p ->
+            configurations {
+                consumeArchive {}
+                consumeDir {}
+            }
+            
+            dependencies {
+                consumeDir project(path: ':producer-tar', configuration:'extracted')
+                consumeArchive project(path: ':producer-tar', configuration:'default' )
+            }
+            
+            tasks.register("copyDir", Copy) {
+                from(configurations.consumeDir)
+                into('build/dir')
+            }
+            
+            tasks.register("copyArchive", Copy) {
+                from(configurations.consumeArchive)
+                into('build/archives')
+            }
+        }
+        """
+        when:
+        def result = gradleRunner("copyArchive").build()
+
+        then:"tar task executed and target folder contains plain tar"
+        result.task(':buildProducerTar').outcome == TaskOutcome.SUCCESS
+        result.task(':consumer:copyArchive').outcome == TaskOutcome.SUCCESS
+        file("producer-tar/build/distributions/elasticsearch.tar.gz").exists()
+        file("consumer/build/archives/elasticsearch.tar.gz").exists()
+
+        when:
+        result = gradleRunner("copyDir").build()
+        then:"plain copy task executed and target folder contains plain content"
+        result.task(':buildProducer').outcome == TaskOutcome.SUCCESS
+        result.task(':consumer:copyDir').outcome == TaskOutcome.SUCCESS
+        file("producer-tar/build/install/someFile.txt").exists()
+        file("consumer/build/dir/someFile.txt").exists()
+    }
+
+    private static boolean assertTarPermissionDefaults(File tarArchive) {
+        TarArchiveInputStream tarInput = new TarArchiveInputStream(new GzipCompressorInputStream(new FileInputStream(tarArchive)))
+        try {
+            TarArchiveEntry currentEntry = tarInput.getNextTarEntry()
+            while (currentEntry != null) {
+                if (currentEntry.isDirectory()) {
+                    assertDefaultDirPermissions(currentEntry.getMode())
+                } else {
+                    assertDefaultFilePermissions(currentEntry.getMode())
+                }
+                currentEntry = tarInput.getNextTarEntry()
+            }
+            return true
+        } finally {
+            tarInput.close()
+        }
+    }
+
+    private static boolean assertZipPermissionDefaults(File archive) {
+        ZipFile zip = new ZipFile(archive)
+        try {
+            Enumeration<ZipEntry> entries = zip.getEntries()
+            while (entries.hasMoreElements()) {
+                ZipEntry zipEntry = entries.nextElement()
+                if (zipEntry.isDirectory()) {
+                    assertDefaultDirPermissions(zipEntry.getUnixMode())
+                } else {
+                    assertDefaultFilePermissions(zipEntry.getUnixMode())
+                }
+            }
+        } finally {
+            zip.close()
+        }
+        true
+    }
+
+    private static boolean assertDefaultDirPermissions(int mode) {
+        assert ((mode >> 6) & 07) == 7
+        assert ((mode >> 3) & 07) == 5
+        assert ((mode >> 0) & 07) == 5
+        true
+    }
+
+    private static boolean assertDefaultFilePermissions(int mode) {
+        assert ((mode >> 6) & 07) == 6
+        assert ((mode >> 3) & 07) == 4
+        assert ((mode >> 0) & 07) == 4
+        true
+    }
+
+    private static boolean assertEmptyDirTasksTriggered(BuildResult result) {
+        result.task(":createJvmOptionsDir").outcome == TaskOutcome.SUCCESS
+        result.task(":createLogsDir").outcome == TaskOutcome.SUCCESS
+        result.task(":createPluginsDir").outcome == TaskOutcome.SUCCESS
+        true
+    }
+}

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -72,7 +72,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build()
 
         then:
-        result.task(":distribution:archives:linux-tar:buildTar").outcome == TaskOutcome.SUCCESS
+        result.task(":distribution:archives:linux-tar:buildExploded").outcome == TaskOutcome.SUCCESS
         result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
         assertExtractedDistroIsCreated(distroVersion, "build/distro", 'current-marker.txt')
     }
@@ -183,14 +183,29 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", "linux-tar")
         new File(bwcSubProjectFolder, 'current-marker.txt') << "current"
         new File(bwcSubProjectFolder, 'build.gradle') << """
+            import org.gradle.api.internal.artifacts.ArtifactAttributes;
+
             apply plugin:'distribution'
-            tasks.register("buildTar", Tar) {
+
+            def buildTar = tasks.register("buildTar", Tar) {
                 from('current-marker.txt')
                 archiveExtension = "tar.gz"
                 compression = Compression.GZIP
             }
+            def buildExploded = tasks.register("buildExploded", Copy) {
+                from('current-marker.txt')
+                into("build/local")
+            }
+            configurations {
+                extracted {
+                    attributes {
+                          attribute(ArtifactAttributes.ARTIFACT_FORMAT, "directory")
+                    }
+                }
+            }
             artifacts {
                 it.add("default", buildTar)
+                it.add("extracted", buildExploded)
             }
         """
         buildFile << """

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDependency.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+public interface DistributionDependency {
+    static DistributionDependency of(String dependencyNotation) {
+        return new StringBasedDistributionDependency(dependencyNotation);
+    }
+
+    Object getDefaultNotation();
+
+    Object getExtractedNotation();
+
+    class StringBasedDistributionDependency implements DistributionDependency {
+        private final String notation;
+
+        public StringBasedDistributionDependency(String notation) {
+            this.notation = notation;
+        }
+
+        @Override
+        public Object getDefaultNotation() {
+            return notation;
+        }
+
+        @Override
+        public Object getExtractedNotation() {
+            return notation;
+        }
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -44,7 +44,7 @@ import static org.elasticsearch.gradle.util.Util.capitalize;
 
 /**
  * A plugin to manage getting and extracting distributions of Elasticsearch.
- *
+ * <p>
  * The plugin provides hooks to register custom distribution resolutions.
  * This plugin resolves distributions from the Elastic downloads service if
  * no registered resolution strategy can resolve to a distribution.
@@ -122,24 +122,24 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
             distribution.finalizeValues();
             DependencyHandler dependencies = project.getDependencies();
             // for the distribution as a file, just depend on the artifact directly
-            Object resolvedDependency = resolveDependencyNotation(project, distribution);
-            dependencies.add(distribution.configuration.getName(), resolvedDependency);
+            DistributionDependency distributionDependency = resolveDependencyNotation(project, distribution);
+            dependencies.add(distribution.configuration.getName(), distributionDependency.getDefaultNotation());
             // no extraction allowed for rpm, deb or docker
             if (distribution.getType().shouldExtract()) {
                 // The extracted configuration depends on the artifact directly but has
                 // an artifact transform registered to resolve it as an unpacked folder.
-                dependencies.add(distribution.getExtracted().getName(), resolvedDependency);
+                dependencies.add(distribution.getExtracted().getName(), distributionDependency.getExtractedNotation());
             }
         }
     }
 
-    private Object resolveDependencyNotation(Project p, ElasticsearchDistribution distribution) {
+    private DistributionDependency resolveDependencyNotation(Project p, ElasticsearchDistribution distribution) {
         return distributionsResolutionStrategiesContainer.stream()
             .sorted(Comparator.comparingInt(DistributionResolution::getPriority))
             .map(r -> r.getResolver().resolve(p, distribution))
             .filter(d -> d != null)
             .findFirst()
-            .orElseGet(() -> dependencyNotation(distribution));
+            .orElseGet(() -> DistributionDependency.of(dependencyNotation(distribution)));
     }
 
     private static void addIvyRepo(Project project, String name, String url, String group) {
@@ -177,7 +177,7 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
      * Maven coordinates point to either the integ-test-zip coordinates on maven central, or a set of artificial
      * coordinates that resolve to the Elastic download service through an ivy repository.
      */
-    private Object dependencyNotation(ElasticsearchDistribution distribution) {
+    private String dependencyNotation(ElasticsearchDistribution distribution) {
         if (distribution.getType() == Type.INTEG_TEST_ZIP) {
             return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip";
         }

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionResolution.java

@@ -51,6 +51,6 @@ public class DistributionResolution {
     }
 
     public interface Resolver {
-        Object resolve(Project project, ElasticsearchDistribution distribution);
+        DistributionDependency resolve(Project project, ElasticsearchDistribution distribution);
     }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -193,6 +193,10 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
 
     @Override
     public String toString() {
+        return getName() + "_" + getType() + "_" + getVersion();
+    }
+
+    public String getFilepath() {
         return configuration.getSingleFile().toString();
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/internal/DistributionArchive.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.gradle.api.Named;
+import org.gradle.api.file.CopySpec;
+import org.gradle.api.tasks.Copy;
+import org.gradle.api.tasks.TaskProvider;
+import org.gradle.api.tasks.bundling.AbstractArchiveTask;
+
+import java.util.function.Supplier;
+
+public class DistributionArchive implements Named {
+
+    private TaskProvider<? extends AbstractArchiveTask> archiveTask;
+    private TaskProvider<Copy> explodedDistTask;
+    private final String name;
+
+    public DistributionArchive(TaskProvider<? extends AbstractArchiveTask> archiveTask, TaskProvider<Copy> explodedDistTask, String name) {
+        this.archiveTask = archiveTask;
+        this.explodedDistTask = explodedDistTask;
+        this.name = name;
+    }
+
+    public void setArchiveClassifier(String classifier) {
+        this.archiveTask.configure(abstractArchiveTask -> abstractArchiveTask.getArchiveClassifier().set(classifier));
+    }
+
+    public void content(Supplier<CopySpec> p) {
+        this.archiveTask.configure(t -> t.with(p.get()));
+        this.explodedDistTask.configure(t -> t.with(p.get()));
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    public TaskProvider<? extends AbstractArchiveTask> getArchiveTask() {
+        return archiveTask;
+    }
+
+    public TaskProvider<Copy> getExplodedArchiveTask() {
+        return explodedDistTask;
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java

@@ -40,6 +40,8 @@ import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.gradle.util.Util.capitalize;
+
 public class InternalDistributionArchiveCheckPlugin implements Plugin<Project> {
 
     private ArchiveOperations archiveOperations;
@@ -246,8 +248,4 @@ public class InternalDistributionArchiveCheckPlugin implements Plugin<Project> {
         return "build" + Arrays.stream(projectName.split("-")).map(f -> capitalize(f)).collect(Collectors.joining());
     }
 
-    private static String capitalize(String str) {
-        if (str == null) return str;
-        return str.substring(0, 1).toUpperCase() + str.substring(1);
-    }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java

@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.elasticsearch.gradle.EmptyDirTask;
+import org.elasticsearch.gradle.tar.SymbolicLinkPreservingTar;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+import org.gradle.api.plugins.BasePlugin;
+import org.gradle.api.tasks.AbstractCopyTask;
+import org.gradle.api.tasks.Copy;
+import org.gradle.api.tasks.TaskContainer;
+import org.gradle.api.tasks.bundling.AbstractArchiveTask;
+import org.gradle.api.tasks.bundling.Compression;
+import org.gradle.api.tasks.bundling.Zip;
+
+import java.io.File;
+
+import static org.elasticsearch.gradle.util.Util.capitalize;
+import static org.gradle.api.internal.artifacts.ArtifactAttributes.ARTIFACT_FORMAT;
+
+/**
+ * Provides a DSL and common configurations to define different types of
+ * Elasticsearch distribution archives. See ':distribution:archives'.
+ * <p>
+ * This configures the default artifacts for the distribution specific
+ * subprojects. We have subprojects for two reasons:
+ * 1. Gradle project substitutions can only bind to the default
+ * configuration of a project
+ * 2. The integ-test-zip and zip distributions have the exact same
+ * filename, so they must be placed in different directories.
+ * 3. We provide a packed and an unpacked variant of the distribution
+ * - the unpacked variant is used by consumers like test cluster definitions
+ * 4. Having per-distribution sub-projects means we can build them in parallel.
+ */
+public class InternalDistributionArchiveSetupPlugin implements Plugin<Project> {
+
+    public static final String DEFAULT_CONFIGURATION_NAME = "default";
+    public static final String EXTRACTED_CONFIGURATION_NAME = "extracted";
+    private NamedDomainObjectContainer<DistributionArchive> container;
+
+    @Override
+    public void apply(Project project) {
+        project.getPlugins().apply(BasePlugin.class);
+        registerAndConfigureDistributionArchivesExtension(project);
+        registerEmptyDirectoryTasks(project);
+        configureGeneralTaskDefaults(project);
+        configureTarDefaults(project);
+    }
+
+    private void registerAndConfigureDistributionArchivesExtension(Project project) {
+        container = project.container(DistributionArchive.class, name -> {
+            var subProjectDir = archiveToSubprojectName(name);
+            var copyDistributionTaskName = "build" + capitalize(name.substring(0, name.length() - 3));
+            TaskContainer tasks = project.getTasks();
+            var explodedDist = tasks.register(copyDistributionTaskName, Copy.class, copy -> copy.into(subProjectDir + "/build/install/"));
+            var archiveTaskName = "build" + capitalize(name);
+            return name.endsWith("Tar")
+                ? new DistributionArchive(tasks.register(archiveTaskName, SymbolicLinkPreservingTar.class), explodedDist, name)
+                : new DistributionArchive(tasks.register(archiveTaskName, Zip.class), explodedDist, name);
+        });
+        // Each defined distribution archive is linked to a subproject.
+        // A distribution archive definition not matching a sub project will result in build failure.
+        container.whenObjectAdded(distributionArchive -> {
+            var subProjectName = archiveToSubprojectName(distributionArchive.getName());
+            project.project(subProjectName, sub -> {
+                sub.getPlugins().apply(BasePlugin.class);
+                sub.getArtifacts().add(DEFAULT_CONFIGURATION_NAME, distributionArchive.getArchiveTask());
+                var extractedConfiguration = sub.getConfigurations().create("extracted");
+                extractedConfiguration.setCanBeResolved(false);
+                extractedConfiguration.getAttributes().attribute(ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
+                sub.getArtifacts().add(EXTRACTED_CONFIGURATION_NAME, distributionArchive.getExplodedArchiveTask());
+
+            });
+        });
+        project.getExtensions().add("distribution_archives", container);
+    }
+
+    private void configureGeneralTaskDefaults(Project project) {
+        // common config across all copy / archive tasks
+        project.getTasks().withType(AbstractCopyTask.class).configureEach(t -> {
+            t.dependsOn(project.getTasks().withType(EmptyDirTask.class));
+            t.setIncludeEmptyDirs(true);
+            t.setDirMode(0755);
+            t.setFileMode(0644);
+        });
+
+        // common config across all archives
+        project.getTasks().withType(AbstractArchiveTask.class).configureEach(t -> {
+            String subdir = archiveTaskToSubprojectName(t.getName());
+            t.getDestinationDirectory().set(project.file(subdir + "/build/distributions"));
+            t.getArchiveBaseName().set(subdir.contains("oss") ? "elasticsearch-oss" : "elasticsearch");
+        });
+    }
+
+    private void configureTarDefaults(Project project) {
+        // common config across all tars
+        project.getTasks().withType(SymbolicLinkPreservingTar.class).configureEach(t -> {
+            t.getArchiveExtension().set("tar.gz");
+            t.setCompression(Compression.GZIP);
+        });
+    }
+
+    private void registerEmptyDirectoryTasks(Project project) {
+        // CopySpec does not make it easy to create an empty directory so we
+        // create the directory that we want, and then point CopySpec to its
+        // parent to copy to the root of the distribution
+        File logsDir = new File(project.getBuildDir(), "logs-hack/logs");
+        project.getExtensions().getExtraProperties().set("logsDir", new File(project.getBuildDir(), "logs-hack/logs"));
+        project.getTasks().register("createLogsDir", EmptyDirTask.class, t -> {
+            t.setDir(logsDir);
+            t.setDirMode(0755);
+        });
+
+        File pluginsDir = new File(project.getBuildDir(), "plugins-hack/plugins");
+        project.getExtensions().add("pluginsDir", pluginsDir);
+        project.getTasks().register("createPluginsDir", EmptyDirTask.class, t -> {
+            t.setDir(pluginsDir);
+            t.setDirMode(0755);
+        });
+
+        File jvmOptionsDir = new File(project.getBuildDir(), "jvm-options-hack/jvm.options.d");
+        project.getExtensions().add("jvmOptionsDir", jvmOptionsDir);
+        project.getTasks().register("createJvmOptionsDir", EmptyDirTask.class, t -> {
+            t.setDir(jvmOptionsDir);
+            t.setDirMode(0750);
+        });
+    }
+
+    private static String archiveTaskToSubprojectName(String taskName) {
+        return archiveToSubprojectName(taskName).substring("build".length() + 1);
+    }
+
+    private static String archiveToSubprojectName(String taskName) {
+        return taskName.replaceAll("[A-Z]", "-$0").toLowerCase();
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

@@ -21,6 +21,7 @@ package org.elasticsearch.gradle.internal;
 
 import org.elasticsearch.gradle.Architecture;
 import org.elasticsearch.gradle.BwcVersions;
+import org.elasticsearch.gradle.DistributionDependency;
 import org.elasticsearch.gradle.DistributionDownloadPlugin;
 import org.elasticsearch.gradle.DistributionResolution;
 import org.elasticsearch.gradle.ElasticsearchDistribution;
@@ -32,6 +33,9 @@ import org.gradle.api.GradleException;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.artifacts.Dependency;
+
+import java.util.function.Function;
 
 import static org.elasticsearch.gradle.util.GradleUtils.projectDependency;
 
@@ -61,18 +65,20 @@ public class InternalDistributionDownloadPlugin implements Plugin<Project> {
 
     /**
      * Registers internal distribution resolutions.
-     *
+     * <p>
      * Elasticsearch distributions are resolved as project dependencies either representing
      * the current version pointing to a project either under `:distribution:archives` or :distribution:packages`.
-     *
+     * <p>
      * BWC versions are resolved as project to projects under `:distribution:bwc`.
-     * */
+     */
     private void registerInternalDistributionResolutions(NamedDomainObjectContainer<DistributionResolution> resolutions) {
 
         resolutions.register("localBuild", distributionResolution -> distributionResolution.setResolver((project, distribution) -> {
             if (VersionProperties.getElasticsearch().equals(distribution.getVersion())) {
                 // non-external project, so depend on local build
-                return projectDependency(project, distributionProjectPath(distribution), "default");
+                return new ProjectBasedDistributionDependency(
+                    config -> projectDependency(project, distributionProjectPath(distribution), config)
+                );
             }
             return null;
         }));
@@ -88,7 +94,9 @@ public class InternalDistributionDownloadPlugin implements Plugin<Project> {
                             + "without a bundled JDK is not supported."
                     );
                 }
-                return projectDependency(project, unreleasedInfo.gradleProjectPath, distributionProjectName(distribution));
+                return new ProjectBasedDistributionDependency(
+                    (config) -> projectDependency(project, unreleasedInfo.gradleProjectPath, distributionProjectName(distribution))
+                );
             }
             return null;
         }));
@@ -162,4 +170,23 @@ public class InternalDistributionDownloadPlugin implements Plugin<Project> {
         }
         return projectName;
     }
+
+    private class ProjectBasedDistributionDependency implements DistributionDependency {
+
+        private Function<String, Dependency> function;
+
+        ProjectBasedDistributionDependency(Function<String, Dependency> function) {
+            this.function = function;
+        }
+
+        @Override
+        public Object getDefaultNotation() {
+            return function.apply("default");
+        }
+
+        @Override
+        public Object getExtractedNotation() {
+            return function.apply("extracted");
+        }
+    }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -108,7 +108,7 @@ public class DistroTestPlugin implements Plugin<Project> {
             depsTasks.put(taskname, depsTask);
             TaskProvider<Test> destructiveTask = configureTestTask(project, taskname, distribution, t -> {
                 t.onlyIf(t2 -> distribution.isDocker() == false || dockerSupport.get().getDockerAvailability().isAvailable);
-                addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::toString);
+                addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath);
                 addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString());
                 t.exclude("**/PackageUpgradeTests.class");
             }, depsTask);
@@ -147,8 +147,8 @@ public class DistroTestPlugin implements Plugin<Project> {
                     upgradeDepsTask.configure(t -> t.dependsOn(distribution, bwcDistro));
                     depsTasks.put(upgradeTaskname, upgradeDepsTask);
                     TaskProvider<Test> upgradeTest = configureTestTask(project, upgradeTaskname, distribution, t -> {
-                        addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::toString);
-                        addDistributionSysprop(t, BWC_DISTRIBUTION_SYSPROP, bwcDistro::toString);
+                        addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath);
+                        addDistributionSysprop(t, BWC_DISTRIBUTION_SYSPROP, bwcDistro::getFilepath);
                         t.include("**/PackageUpgradeTests.class");
                     }, upgradeDepsTask);
                     versionTasks.get(version.toString()).configure(t -> t.dependsOn(upgradeTest));

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-archive-setup.properties

@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.internal.InternalDistributionArchiveSetupPlugin

distribution/archives/build.gradle

@@ -17,41 +17,12 @@
  * under the License.
  */
 
-
 import org.apache.tools.ant.taskdefs.condition.Os
-import org.elasticsearch.gradle.BuildPlugin
-import org.elasticsearch.gradle.EmptyDirTask
-import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.MavenFilteringHack
-import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.info.BuildParams
-import org.elasticsearch.gradle.plugin.PluginBuildPlugin
-import org.elasticsearch.gradle.tar.SymbolicLinkPreservingTar
-import groovy.io.FileType
 import java.nio.file.Files
 import java.nio.file.Path
 
-// need this so Zip/Tar tasks get basic defaults...
-apply plugin: 'base'
-
-// CopySpec does not make it easy to create an empty directory so we
-// create the directory that we want, and then point CopySpec to its
-// parent to copy to the root of the distribution
-ext.logsDir = new File(buildDir, 'logs-hack/logs')
-tasks.register('createLogsDir', EmptyDirTask) {
-  dir = "${logsDir}"
-  dirMode = 0755
-}
-ext.pluginsDir = new File(buildDir, 'plugins-hack/plugins')
-tasks.register('createPluginsDir', EmptyDirTask) {
-  dir = "${pluginsDir}"
-  dirMode = 0755
-}
-ext.jvmOptionsDir = new File(buildDir, 'jvm-options-hack/jvm.options.d')
-tasks.register('createJvmOptionsDir', EmptyDirTask) {
-  dir = "${jvmOptionsDir}"
-  dirMode = 0750
-}
+apply plugin: 'elasticsearch.internal-distribution-archive-setup'
 
 CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, boolean oss, boolean jdk) {
   return copySpec {
@@ -104,144 +75,123 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
   }
 }
 
-// common config across all zip/tar
-tasks.withType(AbstractArchiveTask).configureEach {
-  dependsOn createLogsDir, createPluginsDir, createJvmOptionsDir
-  String subdir = it.name.substring('build'.size()).replaceAll(/[A-Z]/) { '-' + it.toLowerCase() }.substring(1)
-  destinationDirectory = file("${subdir}/build/distributions")
-  archiveBaseName = "elasticsearch${subdir.contains('oss') ? '-oss' : ''}"
+distribution_archives {
+  integTestZip {
+    content {
+      archiveFiles(transportModulesFiles, 'zip', null, 'x64', true, false)
+    }
+  }
+
+  windowsZip {
+    archiveClassifier = 'windows-x86_64'
+    content {
+      archiveFiles(modulesFiles(false, 'windows-x86_64'), 'zip', 'windows', 'x64', false, true)
+    }
+  }
+
+  ossWindowsZip {
+    archiveClassifier = 'windows-x86_64'
+    content {
+      archiveFiles(modulesFiles(true, 'windows-x86_64'), 'zip', 'windows', 'x64', true, true)
+    }
+  }
+
+  noJdkWindowsZip {
+    archiveClassifier = 'no-jdk-windows-x86_64'
+    content {
+      archiveFiles(modulesFiles(false, 'windows-x86_64'), 'zip', 'windows', 'x64', false, false)
+    }
+  }
+
+  ossNoJdkWindowsZip {
+    archiveClassifier = 'no-jdk-windows-x86_64'
+    content {
+      archiveFiles(modulesFiles(true, 'windows-x86_64'), 'zip', 'windows', 'x64', true, false)
+    }
+  }
+
+  darwinTar {
+    archiveClassifier = 'darwin-x86_64'
+    content {
+      archiveFiles(modulesFiles(false, 'darwin-x86_64'), 'tar', 'darwin', 'x64', false, true)
+    }
+  }
+
+  ossDarwinTar {
+    archiveClassifier = 'darwin-x86_64'
+    content {
+      archiveFiles(modulesFiles(true, 'darwin-x86_64'), 'tar', 'darwin', 'x64', true, true)
+    }
+  }
+
+  noJdkDarwinTar {
+    archiveClassifier = 'no-jdk-darwin-x86_64'
+    content {
+      archiveFiles(modulesFiles(false, 'darwin-x86_64'), 'tar', 'darwin', 'x64', false, false)
+    }
+  }
+
+  ossNoJdkDarwinTar {
+    archiveClassifier = 'no-jdk-darwin-x86_64'
+    content {
+      archiveFiles(modulesFiles(true, 'darwin-x86_64'), 'tar', 'darwin', 'x64', true, false)
+    }
+  }
+
+  linuxAarch64Tar {
+    archiveClassifier = 'linux-aarch64'
+    content {
+      archiveFiles(modulesFiles(false, 'linux-aarch64'), 'tar', 'linux', 'aarch64', false, true)
+    }
+  }
+
+  linuxTar {
+    archiveClassifier = 'linux-x86_64'
+    content {
+      archiveFiles(modulesFiles(false, 'linux-x86_64'), 'tar', 'linux', 'x64', false, true)
+    }
+  }
+
+  ossLinuxAarch64Tar {
+    archiveClassifier = 'linux-aarch64'
+    content {
+      archiveFiles(modulesFiles(true, 'linux-aarch64'), 'tar', 'linux', 'aarch64', true, true)
+    }
+  }
+
+  ossLinuxTar {
+    archiveClassifier = 'linux-x86_64'
+    content {
+      archiveFiles(modulesFiles(true, 'linux-x86_64'), 'tar', 'linux', 'x64', true, true)
+    }
+  }
+
+  noJdkLinuxTar {
+    archiveClassifier = 'no-jdk-linux-x86_64'
+    content {
+      archiveFiles(modulesFiles(false, 'linux-x86_64'), 'tar', 'linux', 'x64', false, false)
+    }
+  }
+
+  ossNoJdkLinuxTar {
+    archiveClassifier = 'no-jdk-linux-x86_64'
+    content {
+      archiveFiles(modulesFiles(true, 'linux-x86_64'), 'tar', 'linux', 'x64', true, false)
+    }
+  }
 }
 
-Closure commonZipConfig = {
-  dirMode 0755
-  fileMode 0644
-}
-
-tasks.register('buildIntegTestZip', Zip) {
-  configure(commonZipConfig)
-  with archiveFiles(transportModulesFiles, 'zip', null, 'x64', true, false)
-}
-
-tasks.register('buildWindowsZip', Zip) {
-  configure(commonZipConfig)
-  archiveClassifier = 'windows-x86_64'
-  with archiveFiles(modulesFiles(false, 'windows-x86_64'), 'zip', 'windows', 'x64', false, true)
-}
-
-tasks.register('buildOssWindowsZip', Zip) {
-  configure(commonZipConfig)
-  archiveClassifier = 'windows-x86_64'
-  with archiveFiles(modulesFiles(true, 'windows-x86_64'), 'zip', 'windows', 'x64', true, true)
-}
-
-tasks.register('buildNoJdkWindowsZip', Zip) {
-  configure(commonZipConfig)
-  archiveClassifier = 'no-jdk-windows-x86_64'
-  with archiveFiles(modulesFiles(false, 'windows-x86_64'), 'zip', 'windows', 'x64', false, false)
-}
-
-tasks.register('buildOssNoJdkWindowsZip', Zip) {
-  configure(commonZipConfig)
-  archiveClassifier = 'no-jdk-windows-x86_64'
-  with archiveFiles(modulesFiles(true, 'windows-x86_64'), 'zip', 'windows', 'x64', true, false)
-}
-
-Closure commonTarConfig = {
-  archiveExtension = 'tar.gz'
-  compression = Compression.GZIP
-  dirMode 0755
-  fileMode 0644
-}
-
-tasks.register('buildDarwinTar', SymbolicLinkPreservingTar) {
-  configure(commonTarConfig)
-  archiveClassifier = 'darwin-x86_64'
-  with archiveFiles(modulesFiles(false, 'darwin-x86_64'), 'tar', 'darwin', 'x64', false, true)
-}
-
-tasks.register('buildOssDarwinTar', SymbolicLinkPreservingTar) {
-  configure(commonTarConfig)
-  archiveClassifier = 'darwin-x86_64'
-  with archiveFiles(modulesFiles(true, 'darwin-x86_64'), 'tar', 'darwin', 'x64', true, true)
-}
-
-tasks.register('buildNoJdkDarwinTar', SymbolicLinkPreservingTar) {
-  configure(commonTarConfig)
-  archiveClassifier = 'no-jdk-darwin-x86_64'
-  with archiveFiles(modulesFiles(false, 'darwin-x86_64'), 'tar', 'darwin', 'x64', false, false)
-}
-
-tasks.register('buildOssNoJdkDarwinTar', SymbolicLinkPreservingTar) {
-  configure(commonTarConfig)
-  archiveClassifier = 'no-jdk-darwin-x86_64'
-  with archiveFiles(modulesFiles(true, 'darwin-x86_64'), 'tar', 'darwin', 'x64', true, false)
-}
-
-tasks.register('buildLinuxAarch64Tar', SymbolicLinkPreservingTar) {
-  configure(commonTarConfig)
-  archiveClassifier = 'linux-aarch64'
-  with archiveFiles(modulesFiles(false, 'linux-aarch64'), 'tar', 'linux', 'aarch64', false, true)
-}
-
-tasks.register('buildLinuxTar', SymbolicLinkPreservingTar) {
-  configure(commonTarConfig)
-  archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(false, 'linux-x86_64'), 'tar', 'linux', 'x64', false, true)
-}
-
-tasks.register('buildOssLinuxAarch64Tar', SymbolicLinkPreservingTar) {
-  configure(commonTarConfig)
-  archiveClassifier = 'linux-aarch64'
-  with archiveFiles(modulesFiles(true, 'linux-aarch64'), 'tar', 'linux', 'aarch64', true, true)
-}
-
-tasks.register('buildOssLinuxTar', SymbolicLinkPreservingTar) {
-  configure(commonTarConfig)
-  archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(true, 'linux-x86_64'), 'tar', 'linux', 'x64', true, true)
-}
-
-tasks.register('buildNoJdkLinuxTar', SymbolicLinkPreservingTar) {
-  configure(commonTarConfig)
-  archiveClassifier = 'no-jdk-linux-x86_64'
-  with archiveFiles(modulesFiles(false, 'linux-x86_64'), 'tar', 'linux', 'x64', false, false)
-}
-
-tasks.register('buildOssNoJdkLinuxTar', SymbolicLinkPreservingTar) {
-  configure(commonTarConfig)
-  archiveClassifier = 'no-jdk-linux-x86_64'
-  with archiveFiles(modulesFiles(true, 'linux-x86_64'), 'tar', 'linux', 'x64', true, false)
-}
-
-// This configures the default artifact for the distribution specific
-// subprojects. We have subprojects for two reasons:
-// 1. Gradle project substitutions can only bind to the default
-//    configuration of a project
-// 2. The integ-test-zip and zip distributions have the exact same
-//    filename, so they must be placed in different directories.
 subprojects {
   apply plugin: 'distribution'
   apply plugin: 'elasticsearch.internal-distribution-archive-check'
 
-  distributionArchiveCheck {
-    expectedMlLicenses.add("Boost Software License - Version 1.0 - August 17th, 2003")
-  }
-
-  String buildTask = "build${it.name.replaceAll(/-[a-z]/) { it.substring(1).toUpperCase() }.capitalize()}"
-  ext.buildDist = parent.tasks.named(buildTask)
-  artifacts {
-    'default' buildDist
-  }
-}
-
-subprojects {
   group = "org.elasticsearch.distribution.${name.startsWith("oss-") ? "oss" : "default"}"
 }
 
 /*****************************************************************************
  *                            Rest test config                               *
  *****************************************************************************/
-configure(subprojects.findAll { it.name == 'integ-test-zip' }) {
+project('integ-test-zip') {
   apply plugin: 'elasticsearch.standalone-rest-test'
   apply plugin: 'elasticsearch.rest-test'
 
@@ -266,6 +216,9 @@ configure(subprojects.findAll { it.name == 'integ-test-zip' }) {
   // make the pom file name use elasticsearch instead of the project name
   archivesBaseName = "elasticsearch${it.name.contains('oss') ? '-oss' : ''}"
 
+  String buildTask = "build${it.name.replaceAll(/-[a-z]/) { it.substring(1).toUpperCase() }.capitalize()}"
+  ext.buildDist = parent.tasks.named(buildTask)
+
   publishing {
     publications {
       nebula {

distribution/build.gradle

@@ -666,6 +666,6 @@ subprojects {
   Project subproject = project("${project.path}:${subName}")
   Configuration configuration = configurations.create(subproject.name)
   dependencies {
-    "${configuration.name}" project(subproject.path)
+    "${configuration.name}" project(path: subproject.path, configuration:'default')
   }
 }

distribution/docker/build.gradle

@@ -6,7 +6,6 @@ import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.docker.DockerBuildTask
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin
-
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.internal-distribution-download'
@@ -15,7 +14,6 @@ apply plugin: 'elasticsearch.rest-resources'
 testFixtures.useFixture()
 
 configurations {
-  dockerPlugins
   aarch64DockerSource
   dockerSource
   aarch64OssDockerSource
@@ -23,10 +21,10 @@ configurations {
 }
 
 dependencies {
-  aarch64DockerSource project(path: ":distribution:archives:linux-aarch64-tar")
-  dockerSource project(path: ":distribution:archives:linux-tar")
-  aarch64OssDockerSource project(path: ":distribution:archives:oss-linux-aarch64-tar")
-  ossDockerSource project(path: ":distribution:archives:oss-linux-tar")
+  aarch64DockerSource project(path: ":distribution:archives:linux-aarch64-tar", configuration:"default")
+  dockerSource project(path: ":distribution:archives:linux-tar", configuration:"default")
+  aarch64OssDockerSource project(path: ":distribution:archives:oss-linux-aarch64-tar", configuration:"default")
+  ossDockerSource project(path: ":distribution:archives:oss-linux-tar", configuration:"default")
 }
 
 ext.expansions = { Architecture architecture, boolean oss, DockerBase base, boolean local ->
@@ -146,8 +144,6 @@ void addCopyDockerContextTask(Architecture architecture, boolean oss, DockerBase
         from configurations.dockerSource
       }
     }
-
-    from configurations.dockerPlugins
   }
 }