Split internal distribution handling into separate internal plugin (7.x backport) (#60270)

* Split internal distribution handling into separate internal plugin (#60295)
* Provide proper failure if unexpected non jdk bundled bwc version is requested

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/InternalDistributionDownloadPluginFuncTest.groovy

@@ -0,0 +1,204 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.gradle.testkit.runner.GradleRunner
+import org.gradle.testkit.runner.TaskOutcome
+import org.junit.Rule
+import org.junit.rules.TemporaryFolder
+
+import java.lang.management.ManagementFactory
+
+class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
+
+    def "plugin application fails on non internal build"() {
+        given:
+        buildFile.text = """
+            plugins {
+             id 'elasticsearch.internal-distribution-download'
+            }
+        """
+
+        when:
+        def result = gradleRunner("createExtractedTestDistro").buildAndFail()
+
+        then:
+        assertOutputContains(result.output, "Plugin 'elasticsearch.internal-distribution-download' is not supported. " +
+            "Use 'elasticsearch.distribution-download' plugin instead")
+    }
+
+    def "resolves current version from local build"() {
+        given:
+        internalBuild()
+        localDistroSetup()
+        def distroVersion = VersionProperties.getElasticsearch()
+        buildFile << """
+            apply plugin: 'elasticsearch.internal-distribution-download'
+
+            elasticsearch_distributions {
+              test_distro {
+                  version = "$distroVersion"
+                  type = "archive"
+                  platform = "linux"
+                  architecture = Architecture.current();
+              }
+            }
+            tasks.register("createExtractedTestDistro") {
+                dependsOn elasticsearch_distributions.test_distro.extracted
+            }
+        """
+
+        when:
+        def result = gradleRunner("createExtractedTestDistro").build()
+
+        then:
+        result.task(":distribution:archives:linux-tar:buildTar").outcome == TaskOutcome.SUCCESS
+        result.task(":extractElasticsearchLinux$distroVersion").outcome == TaskOutcome.SUCCESS
+        assertExtractedDistroIsCreated(distroVersion, 'current-marker.txt')
+    }
+
+    def "resolves bwc versions from source"() {
+        given:
+        internalBuild()
+        bwcMinorProjectSetup()
+        def distroVersion = "8.1.0"
+        buildFile << """
+            apply plugin: 'elasticsearch.internal-distribution-download'
+
+            elasticsearch_distributions {
+              test_distro {
+                  version = "8.1.0"
+                  type = "archive"
+                  platform = "linux"
+                  architecture = Architecture.current();
+              }
+            }
+            tasks.register("createExtractedTestDistro") {
+                dependsOn elasticsearch_distributions.test_distro.extracted
+            }
+        """
+        when:
+        def result = gradleRunner("createExtractedTestDistro").build()
+        then:
+        result.task(":distribution:bwc:minor:buildBwcTask").outcome == TaskOutcome.SUCCESS
+        result.task(":extractElasticsearchLinux8.1.0").outcome == TaskOutcome.SUCCESS
+        assertExtractedDistroIsCreated(distroVersion,'bwc-marker.txt')
+    }
+
+    def "fails on resolving bwc versions with no bundled jdk"() {
+        given:
+        internalBuild()
+        bwcMinorProjectSetup()
+        def distroVersion = "8.1.0"
+        buildFile << """
+            apply plugin: 'elasticsearch.internal-distribution-download'
+
+            elasticsearch_distributions {
+              test_distro {
+                  version = "8.1.0"
+                  type = "archive"
+                  platform = "linux"
+                  architecture = Architecture.current();
+                  bundledJdk = false
+              }
+            }
+            tasks.register("createExtractedTestDistro") {
+                dependsOn elasticsearch_distributions.test_distro.extracted
+            }
+        """
+        when:
+        def result = gradleRunner("createExtractedTestDistro").buildAndFail()
+        then:
+        assertOutputContains(result.output, "Configuring a snapshot bwc distribution ('test_distro') " +
+                "without a bundled JDK is not supported.")
+    }
+
+    private File internalBuild() {
+        buildFile << """plugins {
+          id 'elasticsearch.global-build-info'
+        }
+        import org.elasticsearch.gradle.Architecture
+        import org.elasticsearch.gradle.info.BuildParams
+
+        BuildParams.init { it.setIsInternal(true) }
+
+        import org.elasticsearch.gradle.BwcVersions
+        import org.elasticsearch.gradle.Version
+
+        Version currentVersion = Version.fromString("9.0.0")
+        BwcVersions versions = new BwcVersions(new TreeSet<>(
+        Arrays.asList(Version.fromString("8.0.0"), Version.fromString("8.0.1"), Version.fromString("8.1.0"), currentVersion)),
+            currentVersion)
+
+        BuildParams.init { it.setBwcVersions(versions) }
+        """
+    }
+
+
+    private void bwcMinorProjectSetup() {
+        settingsFile << """
+        include ':distribution:bwc:minor'
+        """
+        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", "minor")
+        new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=minor"
+        new File(bwcSubProjectFolder, 'build.gradle') << """
+            apply plugin:'base'
+            configurations.create("linux-tar")
+            tasks.register("buildBwcTask", Tar) {
+                from('bwc-marker.txt')
+                archiveExtension = "tar.gz"
+                compression = Compression.GZIP
+            }
+            artifacts {
+                it.add("linux-tar", buildBwcTask)
+            }
+        """
+    }
+
+    private void localDistroSetup() {
+        settingsFile << """
+        include ":distribution:archives:linux-tar"
+        """
+        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", "linux-tar")
+        new File(bwcSubProjectFolder, 'current-marker.txt') << "current"
+        new File(bwcSubProjectFolder, 'build.gradle') << """
+            apply plugin:'distribution'
+            tasks.register("buildTar", Tar) {
+                from('current-marker.txt')
+                archiveExtension = "tar.gz"
+                compression = Compression.GZIP
+            }
+            artifacts {
+                it.add("default", buildTar)
+            }
+        """
+        buildFile << """
+        """
+
+    }
+
+    boolean assertExtractedDistroIsCreated(String version, String markerFileName) {
+        File extractedFolder = new File(testProjectDir.root, "build/elasticsearch-distros/extracted_elasticsearch_${version}_archive_linux_default")
+        assert extractedFolder.exists()
+        assert new File(extractedFolder, markerFileName).exists()
+        true
+    }
+}

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy

@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.fixtures
+
+import org.gradle.testkit.runner.GradleRunner
+import org.junit.Rule
+import org.junit.rules.TemporaryFolder
+import spock.lang.Specification
+
+import java.lang.management.ManagementFactory
+
+abstract class AbstractGradleFuncTest extends Specification{
+
+    @Rule
+    TemporaryFolder testProjectDir = new TemporaryFolder()
+
+    File settingsFile
+    File buildFile
+
+    def setup() {
+        settingsFile = testProjectDir.newFile('settings.gradle')
+        settingsFile << "rootProject.name = 'hello-world'"
+        buildFile = testProjectDir.newFile('build.gradle')
+    }
+
+    GradleRunner gradleRunner(String... arguments) {
+        GradleRunner.create()
+            .withDebug(ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0)
+            .withProjectDir(testProjectDir.root)
+            .withArguments(arguments)
+            .withPluginClasspath()
+            .forwardOutput()
+    }
+
+    def assertOutputContains(String givenOutput, String expected) {
+        assert normalizedString(givenOutput).contains(normalizedString(expected))
+        true
+    }
+
+    String normalizedString(String input) {
+        return input.readLines().join("\n")
+    }
+
+}

buildSrc/src/integTest/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java

@@ -41,22 +41,6 @@ public class DistributionDownloadPluginIT extends GradleIntegrationTestCase {
 
     // TODO: check reuse of root task across projects MOVE TO UNIT TEST
     // TODO: future: check integ-test-zip to maven, snapshots to snapshot service for external project
-
-    public void testCurrent() throws Exception {
-        String projectName = ":distribution:archives:linux-tar";
-        assertExtractedDistro(
-            VersionProperties.getElasticsearch(),
-            "archive",
-            "linux",
-            null,
-            null,
-            "tests.local_distro.config",
-            "default",
-            "tests.local_distro.project",
-            projectName
-        );
-    }
-
     public void testCurrentExternal() throws Exception {
         checkService(
             VersionProperties.getElasticsearch(),
@@ -70,22 +54,6 @@ public class DistributionDownloadPluginIT extends GradleIntegrationTestCase {
         );
     }
 
-    public void testBwc() throws Exception {
-        assertExtractedDistro(
-            "8.1.0",
-            "archive",
-            "linux",
-            null,
-            null,
-            "tests.local_distro.config",
-            "linux-tar",
-            "tests.local_distro.project",
-            ":distribution:bwc:minor",
-            "tests.current_version",
-            "8.0.0"
-        );
-    }
-
     public void testBwcExternal() throws Exception {
         checkService(
             "8.1.0-SNAPSHOT",

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -24,17 +24,13 @@ import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
 import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
 import org.elasticsearch.gradle.docker.DockerSupportPlugin;
 import org.elasticsearch.gradle.docker.DockerSupportService;
-import org.elasticsearch.gradle.info.BuildParams;
-import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
-import org.gradle.api.GradleException;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.UnknownTaskException;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ConfigurationContainer;
-import org.gradle.api.artifacts.Dependency;
 import org.gradle.api.artifacts.dsl.DependencyHandler;
 import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
 import org.gradle.api.credentials.HttpHeaderCredentials;
@@ -47,55 +43,62 @@ import org.gradle.authentication.http.HttpHeaderAuthentication;
 
 import java.io.File;
 import java.util.Arrays;
-import java.util.HashMap;
+import java.util.Comparator;
-import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.gradle.util.GradleUtils.projectDependency;
 import static org.elasticsearch.gradle.util.Util.capitalize;
 
 /**
  * A plugin to manage getting and extracting distributions of Elasticsearch.
  *
- * The source of the distribution could be from a local snapshot, a locally built
+ * The plugin provides hooks to register custom distribution resolutions.
- * bwc snapshot, or the Elastic downloads service.
+ * This plugin resolves distributions from the Elastic downloads service if
+ * no registered resolution strategy can resolve to a distribution.
  */
 public class DistributionDownloadPlugin implements Plugin<Project> {
 
+    static final String RESOLUTION_CONTAINER_NAME = "elasticsearch_distributions_resolutions";
     private static final String CONTAINER_NAME = "elasticsearch_distributions";
     private static final String FAKE_IVY_GROUP = "elasticsearch-distribution";
     private static final String FAKE_SNAPSHOT_IVY_GROUP = "elasticsearch-distribution-snapshot";
     private static final String DOWNLOAD_REPO_NAME = "elasticsearch-downloads";
     private static final String SNAPSHOT_REPO_NAME = "elasticsearch-snapshots";
 
-    private BwcVersions bwcVersions;
     private NamedDomainObjectContainer<ElasticsearchDistribution> distributionsContainer;
+    private NamedDomainObjectContainer<DistributionResolution> distributionsResolutionStrategiesContainer;
 
     @Override
     public void apply(Project project) {
-        // this is needed for isInternal
-        project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
         project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class);
-
         Provider<DockerSupportService> dockerSupport = GradleUtils.getBuildService(
             project.getGradle().getSharedServices(),
             DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
         );
 
+        setupResolutionsContainer(project);
+        setupDistributionContainer(project, dockerSupport);
+        setupDownloadServiceRepo(project);
+        project.afterEvaluate(this::setupDistributions);
+    }
+
+    private void setupDistributionContainer(Project project, Provider<DockerSupportService> dockerSupport) {
         distributionsContainer = project.container(ElasticsearchDistribution.class, name -> {
             Configuration fileConfiguration = project.getConfigurations().create("es_distro_file_" + name);
             Configuration extractedConfiguration = project.getConfigurations().create("es_distro_extracted_" + name);
             return new ElasticsearchDistribution(name, project.getObjects(), dockerSupport, fileConfiguration, extractedConfiguration);
         });
         project.getExtensions().add(CONTAINER_NAME, distributionsContainer);
+    }
 
-        setupDownloadServiceRepo(project);
+    private void setupResolutionsContainer(Project project) {
-
+        distributionsResolutionStrategiesContainer = project.container(DistributionResolution.class);
-        if (BuildParams.isInternal()) {
+        // We want this ordered in the same resolution strategies are added
-            this.bwcVersions = BuildParams.getBwcVersions();
+        distributionsResolutionStrategiesContainer.whenObjectAdded(
-        }
+            resolveDependencyNotation -> resolveDependencyNotation.setPriority(distributionsResolutionStrategiesContainer.size())
-
+        );
-        project.afterEvaluate(this::setupDistributions);
+        project.getExtensions().add(RESOLUTION_CONTAINER_NAME, distributionsResolutionStrategiesContainer);
     }
 
     @SuppressWarnings("unchecked")
@@ -103,6 +106,11 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
         return (NamedDomainObjectContainer<ElasticsearchDistribution>) project.getExtensions().getByName(CONTAINER_NAME);
     }
 
+    @SuppressWarnings("unchecked")
+    public static NamedDomainObjectContainer<DistributionResolution> getRegistrationsContainer(Project project) {
+        return (NamedDomainObjectContainer<DistributionResolution>) project.getExtensions().getByName(RESOLUTION_CONTAINER_NAME);
+    }
+
     // pkg private for tests
     void setupDistributions(Project project) {
         for (ElasticsearchDistribution distribution : distributionsContainer) {
@@ -110,7 +118,7 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
 
             DependencyHandler dependencies = project.getDependencies();
             // for the distribution as a file, just depend on the artifact directly
-            dependencies.add(distribution.configuration.getName(), dependencyNotation(project, distribution));
+            dependencies.add(distribution.configuration.getName(), resolveDependencyNotation(project, distribution));
 
             // no extraction allowed for rpm, deb or docker
             if (distribution.getType().shouldExtract()) {
@@ -126,6 +134,15 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
         }
     }
 
+    private Object resolveDependencyNotation(Project p, ElasticsearchDistribution distribution) {
+        return distributionsResolutionStrategiesContainer.stream()
+            .sorted(Comparator.comparingInt(DistributionResolution::getPriority))
+            .map(r -> r.getResolver().resolve(p, distribution))
+            .filter(d -> d != null)
+            .findFirst()
+            .orElseGet(() -> dependencyNotation(distribution));
+    }
+
     private void setupRootDownload(Project rootProject, ElasticsearchDistribution distribution) {
         String extractTaskName = extractTaskName(distribution);
         // NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task
@@ -137,13 +154,12 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
             // fall through: register the task
         }
         setupDownloadServiceRepo(rootProject);
-
         final ConfigurationContainer configurations = rootProject.getConfigurations();
         String downloadConfigName = configName("elasticsearch", distribution);
         String extractedConfigName = "extracted_" + downloadConfigName;
         final Configuration downloadConfig = configurations.create(downloadConfigName);
         configurations.create(extractedConfigName);
-        rootProject.getDependencies().add(downloadConfigName, dependencyNotation(rootProject, distribution));
+        rootProject.getDependencies().add(downloadConfigName, resolveDependencyNotation(rootProject, distribution));
 
         // add task for extraction, delaying resolving config until runtime
         if (distribution.getType() == Type.ARCHIVE || distribution.getType() == Type.INTEG_TEST_ZIP) {
@@ -204,38 +220,18 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
             return;
         }
         addIvyRepo(project, DOWNLOAD_REPO_NAME, "https://artifacts.elastic.co", FAKE_IVY_GROUP);
-        if (BuildParams.isInternal() == false) {
+        addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://snapshots.elastic.co", FAKE_SNAPSHOT_IVY_GROUP);
-            // external, so add snapshot repo as well
-            addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://snapshots.elastic.co", FAKE_SNAPSHOT_IVY_GROUP);
-        }
     }
 
     /**
      * Returns a dependency object representing the given distribution.
-     *
+     * <p>
      * The returned object is suitable to be passed to {@link DependencyHandler}.
-     * The concrete type of the object will either be a project {@link Dependency} or
+     * The concrete type of the object will be a set of maven coordinates as a {@link String}.
-     * a set of maven coordinates as a {@link String}. Project dependencies point to
+     * Maven coordinates point to either the integ-test-zip coordinates on maven central, or a set of artificial
-     * a project in the Elasticsearch repo either under `:distribution:bwc`,
-     * `:distribution:archives` or :distribution:packages`. Maven coordinates point to
-     * either the integ-test-zip coordinates on maven central, or a set of artificial
      * coordinates that resolve to the Elastic download service through an ivy repository.
      */
-    private Object dependencyNotation(Project project, ElasticsearchDistribution distribution) {
+    private Object dependencyNotation(ElasticsearchDistribution distribution) {
-
-        if (BuildParams.isInternal()) {
-            // non-external project, so depend on local build
-
-            if (VersionProperties.getElasticsearch().equals(distribution.getVersion())) {
-                return projectDependency(project, distributionProjectPath(distribution), "default");
-            }
-            BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.unreleasedInfo(Version.fromString(distribution.getVersion()));
-            if (unreleasedInfo != null) {
-                assert distribution.getBundledJdk();
-                return projectDependency(project, unreleasedInfo.gradleProjectPath, distributionProjectName(distribution));
-            }
-        }
-
         if (distribution.getType() == Type.INTEG_TEST_ZIP) {
             return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip";
         }
@@ -268,80 +264,6 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
         return group + ":elasticsearch" + flavor + ":" + distribution.getVersion() + classifier + "@" + extension;
     }
 
-    private static Dependency projectDependency(Project project, String projectPath, String projectConfig) {
-        if (project.findProject(projectPath) == null) {
-            throw new GradleException("no project [" + projectPath + "], project names: " + project.getRootProject().getAllprojects());
-        }
-        Map<String, Object> depConfig = new HashMap<>();
-        depConfig.put("path", projectPath);
-        depConfig.put("configuration", projectConfig);
-        return project.getDependencies().project(depConfig);
-    }
-
-    private static String distributionProjectPath(ElasticsearchDistribution distribution) {
-        String projectPath = ":distribution";
-        switch (distribution.getType()) {
-            case INTEG_TEST_ZIP:
-                projectPath += ":archives:integ-test-zip";
-                break;
-
-            case DOCKER:
-                projectPath += ":docker:";
-                projectPath += distributionProjectName(distribution);
-                break;
-
-            default:
-                projectPath += distribution.getType() == Type.ARCHIVE ? ":archives:" : ":packages:";
-                projectPath += distributionProjectName(distribution);
-                break;
-        }
-        return projectPath;
-    }
-
-    /**
-     * Works out the gradle project name that provides a distribution artifact.
-     *
-     * @param distribution the distribution from which to derive a project name
-     * @return the name of a project. It is not the full project path, only the name.
-     */
-    private static String distributionProjectName(ElasticsearchDistribution distribution) {
-        Platform platform = distribution.getPlatform();
-        Architecture architecture = distribution.getArchitecture();
-        String projectName = "";
-
-        final String archString = platform == Platform.WINDOWS || architecture == Architecture.X64
-            ? ""
-            : "-" + architecture.toString().toLowerCase();
-
-        if (distribution.getFlavor() == Flavor.OSS) {
-            projectName += "oss-";
-        }
-
-        if (distribution.getBundledJdk() == false) {
-            projectName += "no-jdk-";
-        }
-
-        switch (distribution.getType()) {
-            case ARCHIVE:
-                if (Version.fromString(distribution.getVersion()).onOrAfter("7.0.0")) {
-                    projectName += platform.toString() + archString + (platform == Platform.WINDOWS ? "-zip" : "-tar");
-                } else {
-                    projectName = distribution.getFlavor().equals(Flavor.DEFAULT) ? "zip" : "oss-zip";
-                }
-                break;
-
-            case DOCKER:
-                projectName += "docker" + archString + "-export";
-                break;
-
-            default:
-                projectName += distribution.getType();
-                break;
-        }
-
-        return projectName;
-    }
-
     private static String configName(String prefix, ElasticsearchDistribution distribution) {
         return String.format(
             "%s_%s_%s_%s%s%s",

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionResolution.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.gradle.api.Project;
+
+public class DistributionResolution {
+    private Resolver resolver;
+    private String name;
+    private int priority;
+
+    public DistributionResolution(String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Resolver getResolver() {
+        return resolver;
+    }
+
+    public void setResolver(Resolver resolver) {
+        this.resolver = resolver;
+    }
+
+    public void setPriority(int priority) {
+        this.priority = priority;
+    }
+
+    public int getPriority() {
+        return priority;
+    }
+
+    public interface Resolver {
+        Object resolve(Project project, ElasticsearchDistribution distribution);
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

@@ -0,0 +1,160 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.elasticsearch.gradle.Architecture;
+import org.elasticsearch.gradle.BwcVersions;
+import org.elasticsearch.gradle.DistributionDownloadPlugin;
+import org.elasticsearch.gradle.DistributionResolution;
+import org.elasticsearch.gradle.ElasticsearchDistribution;
+import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.info.BuildParams;
+import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
+import org.gradle.api.GradleException;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+
+import static org.elasticsearch.gradle.util.GradleUtils.projectDependency;
+
+/**
+ * An internal elasticsearch build plugin that registers additional
+ * distribution resolution strategies to the 'elasticsearch.download-distribution' plugin
+ * to resolve distributions from a local snapshot or a locally built bwc snapshot.
+ */
+public class InternalDistributionDownloadPlugin implements Plugin<Project> {
+
+    private BwcVersions bwcVersions = null;
+
+    @Override
+    public void apply(Project project) {
+        // this is needed for isInternal
+        project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
+        if (!BuildParams.isInternal()) {
+            throw new GradleException(
+                "Plugin 'elasticsearch.internal-distribution-download' is not supported. "
+                    + "Use 'elasticsearch.distribution-download' plugin instead."
+            );
+        }
+        project.getPluginManager().apply(DistributionDownloadPlugin.class);
+        this.bwcVersions = BuildParams.getBwcVersions();
+        registerInternalDistributionResolutions(DistributionDownloadPlugin.getRegistrationsContainer(project));
+    }
+
+    /**
+     * Registers internal distribution resolutions.
+     *
+     * Elasticsearch distributions are resolved as project dependencies either representing
+     * the current version pointing to a project either under `:distribution:archives` or :distribution:packages`.
+     *
+     * BWC versions are resolved as project to projects under `:distribution:bwc`.
+     * */
+    private void registerInternalDistributionResolutions(NamedDomainObjectContainer<DistributionResolution> resolutions) {
+
+        resolutions.register("localBuild", distributionResolution -> distributionResolution.setResolver((project, distribution) -> {
+            if (VersionProperties.getElasticsearch().equals(distribution.getVersion())) {
+                // non-external project, so depend on local build
+                return projectDependency(project, distributionProjectPath(distribution), "default");
+            }
+            return null;
+        }));
+
+        resolutions.register("bwc", distributionResolution -> distributionResolution.setResolver((project, distribution) -> {
+            BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.unreleasedInfo(Version.fromString(distribution.getVersion()));
+            if (unreleasedInfo != null) {
+                if (!distribution.getBundledJdk()) {
+                    throw new GradleException(
+                        "Configuring a snapshot bwc distribution ('"
+                            + distribution.getName()
+                            + "') "
+                            + "without a bundled JDK is not supported."
+                    );
+                }
+                return projectDependency(project, unreleasedInfo.gradleProjectPath, distributionProjectName(distribution));
+            }
+            return null;
+        }));
+    }
+
+    private static String distributionProjectPath(ElasticsearchDistribution distribution) {
+        String projectPath = ":distribution";
+        switch (distribution.getType()) {
+            case INTEG_TEST_ZIP:
+                projectPath += ":archives:integ-test-zip";
+                break;
+
+            case DOCKER:
+                projectPath += ":docker:";
+                projectPath += distributionProjectName(distribution);
+                break;
+
+            default:
+                projectPath += distribution.getType() == ElasticsearchDistribution.Type.ARCHIVE ? ":archives:" : ":packages:";
+                projectPath += distributionProjectName(distribution);
+                break;
+        }
+        return projectPath;
+    }
+
+    /**
+     * Works out the gradle project name that provides a distribution artifact.
+     *
+     * @param distribution the distribution from which to derive a project name
+     * @return the name of a project. It is not the full project path, only the name.
+     */
+    private static String distributionProjectName(ElasticsearchDistribution distribution) {
+        ElasticsearchDistribution.Platform platform = distribution.getPlatform();
+        Architecture architecture = distribution.getArchitecture();
+        String projectName = "";
+
+        final String archString = platform == ElasticsearchDistribution.Platform.WINDOWS || architecture == Architecture.X64
+            ? ""
+            : "-" + architecture.toString().toLowerCase();
+
+        if (distribution.getFlavor() == ElasticsearchDistribution.Flavor.OSS) {
+            projectName += "oss-";
+        }
+
+        if (distribution.getBundledJdk() == false) {
+            projectName += "no-jdk-";
+        }
+        switch (distribution.getType()) {
+            case ARCHIVE:
+                if (Version.fromString(distribution.getVersion()).onOrAfter("7.0.0")) {
+                    projectName += platform.toString() + archString + (platform == ElasticsearchDistribution.Platform.WINDOWS
+                        ? "-zip"
+                        : "-tar");
+                } else {
+                    projectName = distribution.getFlavor().equals(ElasticsearchDistribution.Flavor.DEFAULT) ? "zip" : "oss-zip";
+                }
+                break;
+
+            case DOCKER:
+                projectName += "docker" + archString + "-export";
+                break;
+
+            default:
+                projectName += distribution.getType();
+                break;
+        }
+        return projectName;
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -33,6 +33,7 @@ import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.docker.DockerSupportPlugin;
 import org.elasticsearch.gradle.docker.DockerSupportService;
 import org.elasticsearch.gradle.info.BuildParams;
+import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.elasticsearch.gradle.vagrant.BatsProgressLogger;
 import org.elasticsearch.gradle.vagrant.VagrantBasePlugin;
@@ -89,7 +90,7 @@ public class DistroTestPlugin implements Plugin<Project> {
     @Override
     public void apply(Project project) {
         project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class);
-        project.getPluginManager().apply(DistributionDownloadPlugin.class);
+        project.getPlugins().apply(InternalDistributionDownloadPlugin.class);
         project.getPluginManager().apply("elasticsearch.build");
 
         Provider<DockerSupportService> dockerSupport = GradleUtils.getBuildService(

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java

@@ -25,6 +25,9 @@ import org.elasticsearch.gradle.JdkDownloadPlugin;
 import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.ReaperPlugin;
 import org.elasticsearch.gradle.ReaperService;
+import org.elasticsearch.gradle.info.BuildParams;
+import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
+import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
@@ -55,8 +58,13 @@ public class TestClustersPlugin implements Plugin<Project> {
 
     @Override
     public void apply(Project project) {
-        project.getPluginManager().apply(DistributionDownloadPlugin.class);
         project.getPluginManager().apply(JdkDownloadPlugin.class);
+        project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
+        if (BuildParams.isInternal()) {
+            project.getPlugins().apply(InternalDistributionDownloadPlugin.class);
+        } else {
+            project.getPlugins().apply(DistributionDownloadPlugin.class);
+        }
         project.getRootProject().getPluginManager().apply(ReaperPlugin.class);
 
         ReaperService reaper = project.getRootProject().getExtensions().getByType(ReaperService.class);

buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java

@@ -27,6 +27,7 @@ import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.UnknownTaskException;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.Dependency;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.provider.Provider;
@@ -43,6 +44,7 @@ import org.gradle.plugins.ide.idea.model.IdeaModel;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -218,4 +220,14 @@ public abstract class GradleUtils {
         child.setCompileClasspath(project.getObjects().fileCollection().from(child.getCompileClasspath(), parent.getOutput()));
         child.setRuntimeClasspath(project.getObjects().fileCollection().from(child.getRuntimeClasspath(), parent.getOutput()));
     }
+
+    public static Dependency projectDependency(Project project, String projectPath, String projectConfig) {
+        if (project.findProject(projectPath) == null) {
+            throw new GradleException("no project [" + projectPath + "], project names: " + project.getRootProject().getAllprojects());
+        }
+        Map<String, Object> depConfig = new HashMap<>();
+        depConfig.put("path", projectPath);
+        depConfig.put("configuration", projectConfig);
+        return project.getDependencies().project(depConfig);
+    }
 }

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distribution-download.properties

@@ -17,4 +17,4 @@
 # under the License.
 #
 
-implementation-class=org.elasticsearch.gradle.DistributionDownloadPlugin
+implementation-class=org.elasticsearch.gradle.DistributionDownloadPlugin

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-download.properties

@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin

distribution/docker/build.gradle

@@ -8,7 +8,7 @@ import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin
 
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
-apply plugin: 'elasticsearch.distribution-download'
+apply plugin: 'elasticsearch.internal-distribution-download'
 apply plugin: 'elasticsearch.rest-resources'
 
 testFixtures.useFixture()

gradle/local-distribution.gradle

@@ -25,7 +25,7 @@
  * */
 import org.elasticsearch.gradle.Architecture
 
-apply plugin:'elasticsearch.distribution-download'
+apply plugin:'elasticsearch.internal-distribution-download'
 
 elasticsearch_distributions {
   local {

qa/remote-clusters/build.gradle

@@ -23,7 +23,7 @@ import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin
 
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
-apply plugin: 'elasticsearch.distribution-download'
+apply plugin: 'elasticsearch.internal-distribution-download'
 
 testFixtures.useFixture()
 

qa/wildfly/build.gradle

@@ -23,7 +23,7 @@ import org.elasticsearch.gradle.VersionProperties
 apply plugin: 'war'
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.test.fixtures'
-apply plugin: 'elasticsearch.distribution-download'
+apply plugin: 'elasticsearch.internal-distribution-download'
 
 testFixtures.useFixture()