Use project dependency instead of substitutions for distributions (#37730)

Currently integration tests which use either bwc snapshot versions or
the current version of elasticsearch depend on project substitutions to
link to the build of those artifacts. Likewise, vagrant tests use
dependency substitutions to get to bwc snapshots of rpm and debs.
This commit changes those to depend on the relevant project/configuration
and removes the dependency substitutions for distributions we do not
publish.

build.gradle

@@ -221,14 +221,6 @@ allprojects {
     "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
-    "downloads.zip:elasticsearch:${version}": ':distribution:archives:zip',
-    "downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
-    "downloads.tar:elasticsearch:${version}": ':distribution:archives:tar',
-    "downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
-    "downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
-    "downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
-    "downloads.deb:elasticsearch:${version}": ':distribution:packages:deb',
-    "downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
     "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
     "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
     // for transport client
@@ -240,19 +232,6 @@ allprojects {
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
     "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
   ]
-  // substitute unreleased versions with projects that check out and build locally
-  bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
-    Version unreleased = unreleasedVersion.version
-    String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
-    ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject
-    if (unreleased.onOrAfter('6.3.0')) {
-      ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
-    }
-  }
 
   /*
    * Gradle only resolve project substitutions during dependency resolution but

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
@@ -171,6 +172,12 @@ class ClusterFormationTasks {
 
     /** Adds a dependency on the given distribution */
     static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
+        if (distro.equals("integ-test-zip")) {
+            // short circuit integ test so it doesn't complicate the rest of the distribution setup below
+            project.dependencies.add(configuration.name,
+                    "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${elasticsearchVersion}@zip")
+            return
+        }
         // TEMP HACK
         // The oss docs CI build overrides the distro on the command line. This hack handles backcompat until CI is updated.
         if (distro.equals('oss-zip')) {
@@ -180,22 +187,31 @@ class ClusterFormationTasks {
             distro = 'default'
         }
         // END TEMP HACK
-        if (['integ-test-zip', 'oss', 'default'].contains(distro) == false) {
+        if (['oss', 'default'].contains(distro) == false) {
             throw new GradleException("Unknown distribution: ${distro} in project ${project.path}")
         }
         Version version = Version.fromString(elasticsearchVersion)
-        if (version.before('6.3.0') && distro.startsWith('oss-')) {
+        String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo
-            distro = distro.substring('oss-'.length())
-        }
-        String group = "downloads.zip"
-        if (distro.equals("integ-test-zip")) {
-            group = "org.elasticsearch.distribution.integ-test-zip"
-        }
         String artifactName = 'elasticsearch'
         if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) {
             artifactName += '-oss'
         }
-        project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@zip")
+        String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip'
+        Object dependency
+        boolean internalBuild = project.hasProperty('bwcVersions')
+        VersionCollection.UnreleasedVersionInfo unreleasedInfo = null
+        if (project.hasProperty('bwcVersions')) {
+            // NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools?
+            unreleasedInfo = project.bwcVersions.unreleasedInfo(version)
+        }
+        if (unreleasedInfo != null) {
+            dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject)
+        } else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) {
+            dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}")
+        } else {
+            dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip"
+        }
+        project.dependencies.add(configuration.name, dependency)
     }
 
     /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -4,6 +4,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.FileContentsTask
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionCollection
 import org.gradle.api.*
 import org.gradle.api.artifacts.dsl.RepositoryHandler
 import org.gradle.api.execution.TaskExecutionAdapter
@@ -184,20 +185,36 @@ class VagrantTestPlugin implements Plugin<Project> {
             upgradeFromVersion = Version.fromString(upgradeFromVersionRaw)
         }
 
+        List<Object> dependencies = new ArrayList<>()
         DISTRIBUTIONS.each {
             // Adds a dependency for the current version
-            project.dependencies.add(PACKAGING_CONFIGURATION,
+            dependencies.add(project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
-                    project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
         }
 
-        UPGRADE_FROM_ARCHIVES.each {
+        // The version of elasticsearch that we upgrade *from*
-            // The version of elasticsearch that we upgrade *from*
+        VersionCollection.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(upgradeFromVersion)
-            project.dependencies.add(PACKAGING_CONFIGURATION,
+        if (unreleasedInfo != null) {
-                    "downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
+            // handle snapshots pointing to bwc build
-            if (upgradeFromVersion.onOrAfter('6.3.0')) {
+            UPGRADE_FROM_ARCHIVES.each {
-                project.dependencies.add(PACKAGING_CONFIGURATION,
+                dependencies.add(project.dependencies.project(
-                        "downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+                        path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: it))
+                if (upgradeFromVersion.onOrAfter('6.3.0')) {
+                    dependencies.add(project.dependencies.project(
+                            path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: "oss-${it}"))
+                }
             }
+        } else {
+            UPGRADE_FROM_ARCHIVES.each {
+                // The version of elasticsearch that we upgrade *from*
+                dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
+                if (upgradeFromVersion.onOrAfter('6.3.0')) {
+                    dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+                }
+            }
+        }
+
+        for (Object dependency : dependencies) {
+            project.dependencies.add(PACKAGING_CONFIGURATION, dependency)
         }
 
         project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion

buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java

@@ -20,6 +20,8 @@ package org.elasticsearch.gradle;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -86,6 +88,7 @@ public class VersionCollection {
 
     private final Version currentVersion;
     private final Map<Integer, List<Version>> groupByMajor;
+    private final Map<Version, UnreleasedVersionInfo> unreleased;
 
     public class UnreleasedVersionInfo {
         public final Version version;
@@ -129,6 +132,16 @@ public class VersionCollection {
         assertCurrentVersionMatchesParsed(currentVersionProperty);
 
         assertNoOlderThanTwoMajors();
+
+        Map<Version, UnreleasedVersionInfo> unreleased = new HashMap<>();
+        for (Version unreleasedVersion : getUnreleased()) {
+            if (unreleasedVersion.equals(currentVersion)) {
+                continue;
+            }
+            unreleased.put(unreleasedVersion,
+                new UnreleasedVersionInfo(unreleasedVersion, getBranchFor(unreleasedVersion), getGradleProjectNameFor(unreleasedVersion)));
+        }
+        this.unreleased = Collections.unmodifiableMap(unreleased);
     }
 
     private void assertNoOlderThanTwoMajors() {
@@ -150,6 +163,13 @@ public class VersionCollection {
         }
     }
 
+    /**
+      * Returns info about the unreleased version, or {@code null} if the version is released.
+      */
+    public UnreleasedVersionInfo unreleasedInfo(Version version) {
+        return unreleased.get(version);
+    }
+
     public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
         getUnreleased().stream()
             .filter(version -> version.equals(currentVersion) == false)

distribution/bwc/build.gradle

@@ -120,18 +120,19 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
         }
     }
 
-    List<File> artifactFiles = []
+    Map<String, File> artifactFiles = [:]
     List<String> projectDirs = []
-    for (String project : ['zip', 'deb', 'rpm']) {
+    List<String> projects = ['zip', 'deb', 'rpm']
+    for (String projectName : projects) {
         String baseDir = "distribution"
         if (bwcVersion.onOrAfter('6.3.0')) {
-            baseDir += project == 'zip' ? '/archives' : '/packages'
+            baseDir += projectName == 'zip' ? '/archives' : '/packages'
             // add oss variant first
-            projectDirs.add("${baseDir}/oss-${project}")
+            projectDirs.add("${baseDir}/oss-${projectName}")
-            artifactFiles.add(file("${checkoutDir}/${baseDir}/oss-${project}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${project}"))
+            artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${projectName}"))
         }
-        projectDirs.add("${baseDir}/${project}")
+        projectDirs.add("${baseDir}/${projectName}")
-        artifactFiles.add(file("${checkoutDir}/${baseDir}/${project}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${project}"))
+        artifactFiles.put(projectName, file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${projectName}"))
     }
 
     task buildBwcVersion(type: Exec) {
@@ -187,7 +188,7 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
         standardOutput = new IndentingOutputStream(System.out, bwcVersion)
         errorOutput = new IndentingOutputStream(System.err, bwcVersion)
         doLast {
-            List missing = artifactFiles.grep { file ->
+            List missing = artifactFiles.values().grep { file ->
                 false == file.exists()
             }
             if (false == missing.empty) {
@@ -197,12 +198,16 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
         }
     }
 
-    artifacts {
+    for (e in artifactFiles) {
-      for (File artifactFile : artifactFiles) {
+        String projectName = e.key
-        String artifactName = artifactFile.name.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
+        File artifactFile = e.value
+        String artifactFileName = artifactFile.name
+        String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
         String suffix = artifactFile.toString()[-3..-1]
-        'default' file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion
+        configurations.create(projectName)
-      }
+        artifacts {
+            it.add(projectName, [file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion])
+        }
     }
     // make sure no dependencies were added to assemble; we want it to be a no-op
     assemble.dependsOn = []