Use ivy pattern for elasticsearch artifacts download (#37557)

The rpm, deb and tar distributions were removed some time ago from maven central. The zip distribution still exists there, but it does not need to. Instead, this commit sets up an ivy repository with pattern pointing to the elasticsearch artifacts download service. Note that the integ-test-zip remains in maven central, since it is not present in the download service.
2019-01-20 23:33:11 -08:00 · 2019-01-20 23:33:11 -08:00 · c9c088ff02
parent 88f4b0a326
commit c9c088ff02
7 changed files with 66 additions and 27 deletions
--- a/build.gradle
+++ b/build.gradle
@ -221,14 +221,14 @@ allprojects {
    "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
    "org.elasticsearch.test:framework:${version}": ':test:framework',
    "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
-    "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip',
-    "org.elasticsearch.distribution.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
-    "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:archives:tar',
-    "org.elasticsearch.distribution.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
-    "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
-    "org.elasticsearch.distribution.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
-    "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:packages:deb',
-    "org.elasticsearch.distribution.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
+    "downloads.zip:elasticsearch:${version}": ':distribution:archives:zip',
+    "downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
+    "downloads.tar:elasticsearch:${version}": ':distribution:archives:tar',
+    "downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
+    "downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
+    "downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
+    "downloads.deb:elasticsearch:${version}": ':distribution:packages:deb',
+    "downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
    "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
    "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
    // for transport client
@ -244,13 +244,13 @@ allprojects {
  bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
    Version unreleased = unreleasedVersion.version
    String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
-    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject
    if (unreleased.onOrAfter('6.3.0')) {
-      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
    }
  }

--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@ -178,6 +178,21 @@ if (project != rootProject) {
  jarHell.enabled = false
  thirdPartyAudit.enabled = false

+  configurations {
+    distribution
+  }
+
+  dependencies {
+    distribution project(':distribution:archives:zip')
+    distribution project(':distribution:archives:oss-zip')
+  }
+
+  String localDownloads = "${rootProject.buildDir}/local-downloads"
+  task setupLocalDownloads(type:Copy) {
+    from configurations.distribution
+    into localDownloads
+  }
+
  unitTest {
    // The test task is configured to runtimeJava version, but build-tools doesn't support all of them, so test
    // with compiler instead on the ones that are too old.
@ -192,6 +207,7 @@ if (project != rootProject) {
    dependsOn project.rootProject.allprojects.collect {
      it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'}
    }
+    dependsOn setupLocalDownloads
    exclude "**/*Tests.class"
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
@ -204,6 +220,7 @@ if (project != rootProject) {
            ).asPath,
    )
    systemProperty 'test.local-test-repo-path', "${rootProject.buildDir}/local-test-repo"
+    systemProperty 'test.local-test-downloads-path', localDownloads
    systemProperty 'test.version_under_test', version
    systemProperty 'test.lucene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1]
  }
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@ -565,6 +565,12 @@ class BuildPlugin implements Plugin<Project> {
            repos.mavenLocal()
        }
        repos.jcenter()
+        repos.ivy {
+            url "https://artifacts.elastic.co/downloads"
+            patternLayout {
+                artifact "elasticsearch/[module]-[revision](-[classifier]).[ext]"
+            }
+        }
        repos.maven {
            name "elastic"
            url "https://artifacts.elastic.co/maven"
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@ -18,14 +18,12 @@
 */
 package org.elasticsearch.gradle.test

-import java.util.stream.Collectors
 import org.apache.tools.ant.DefaultLogger
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
-
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
 import org.gradle.api.AntBuilder
@ -45,7 +43,7 @@ import org.gradle.api.tasks.Exec
 import java.nio.charset.StandardCharsets
 import java.nio.file.Paths
 import java.util.concurrent.TimeUnit
-
+import java.util.stream.Collectors
 /**
 * A helper for creating tasks to build a cluster that is used by a task, and tear down the cluster when the task is finished.
 */
@ -174,24 +172,31 @@ class ClusterFormationTasks {

    /** Adds a dependency on the given distribution */
    static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
-        if (Version.fromString(elasticsearchVersion).before('6.3.0') &&
-                distro.startsWith('oss-')
-        ) {
+        Version version = Version.fromString(elasticsearchVersion)
+        if (version.before('6.3.0') && distro.startsWith('oss-')) {
            distro = distro.substring('oss-'.length())
        }
+        String os = "linux"
+        if (Os.FAMILY_WINDOWS) {
+            os = "windows"
+        } else if (Os.FAMILY_MAC) {
+            os = "darwin"
+        }
        String packaging = distro
        if (distro.contains('tar')) {
            packaging = 'tar.gz'\
        } else if (distro.contains('zip')) {
            packaging = 'zip'
        }
-        String subgroup = distro
+        String group = "downloads.${packaging}"
+        if (distro.equals("integ-test-zip")) {
+            group = "org.elasticsearch.distribution.integ-test-zip"
+        }
        String artifactName = 'elasticsearch'
        if (distro.contains('oss')) {
            artifactName += '-oss'
-            subgroup = distro.substring('oss-'.length())
        }
-        project.dependencies.add(configuration.name, "org.elasticsearch.distribution.${subgroup}:${artifactName}:${elasticsearchVersion}@${packaging}")
+        project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@${packaging}")
    }

    /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
@ -193,10 +193,10 @@ class VagrantTestPlugin implements Plugin<Project> {
        UPGRADE_FROM_ARCHIVES.each {
            // The version of elasticsearch that we upgrade *from*
            project.dependencies.add(PACKAGING_CONFIGURATION,
-                    "org.elasticsearch.distribution.${it}:elasticsearch:${upgradeFromVersion}@${it}")
+                    "downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
            if (upgradeFromVersion.onOrAfter('6.3.0')) {
                project.dependencies.add(PACKAGING_CONFIGURATION,
-                        "org.elasticsearch.distribution.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+                        "downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
            }
        }

--- a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java
@ -127,6 +127,9 @@ public class BuildExamplePluginsIT extends GradleIntegrationTestCase {
                "  maven {\n" +
                "    url \"" + getLocalTestRepoPath()  + "\"\n" +
                "  }\n" +
+                "  flatDir {\n" +
+                "    dir '" + getLocalTestDownloadsPath() + "'\n" +
+                "  }\n" +
                luceneSnapshotRepo +
                "}\n"
        );
--- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
@ -144,8 +144,16 @@ public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {
    }

    protected String getLocalTestRepoPath() {
-        String property = System.getProperty("test.local-test-repo-path");
-        Objects.requireNonNull(property, "test.local-test-repo-path not passed to tests");
+        return getLocalTestPath("test.local-test-repo-path");
+    }
+
+    protected String getLocalTestDownloadsPath() {
+        return getLocalTestPath("test.local-test-downloads-path");
+    }
+
+    private String getLocalTestPath(String propertyName) {
+        String property = System.getProperty(propertyName);
+        Objects.requireNonNull(property, propertyName + " not passed to tests");
        File file = new File(property);
        assertTrue("Expected " + property + " to exist, but it did not!", file.exists());
        if (File.separator.equals("\\")) {