From c9c088ff0212ba144b64326c4d297183b1c0eb8f Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Sun, 20 Jan 2019 23:33:11 -0800 Subject: [PATCH] Use ivy pattern for elasticsearch artifacts download (#37557) The rpm, deb and tar distributions were removed some time ago from maven central. The zip distribution still exists there, but it does not need to. Instead, this commit sets up an ivy repository with pattern pointing to the elasticsearch artifacts download service. Note that the integ-test-zip remains in maven central, since it is not present in the download service. --- build.gradle | 28 +++++++++---------- buildSrc/build.gradle | 17 +++++++++++ .../elasticsearch/gradle/BuildPlugin.groovy | 6 ++++ .../gradle/test/ClusterFormationTasks.groovy | 23 +++++++++------ .../gradle/vagrant/VagrantTestPlugin.groovy | 4 +-- .../gradle/BuildExamplePluginsIT.java | 3 ++ .../test/GradleIntegrationTestCase.java | 12 ++++++-- 7 files changed, 66 insertions(+), 27 deletions(-) diff --git a/build.gradle b/build.gradle index 439bd32727d..dade656f78b 100644 --- a/build.gradle +++ b/build.gradle @@ -221,14 +221,14 @@ allprojects { "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi', "org.elasticsearch.test:framework:${version}": ':test:framework', "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip', - "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip', - "org.elasticsearch.distribution.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip', - "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:archives:tar', - "org.elasticsearch.distribution.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar', - "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:packages:rpm', - "org.elasticsearch.distribution.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm', - "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:packages:deb', - "org.elasticsearch.distribution.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb', + "downloads.zip:elasticsearch:${version}": ':distribution:archives:zip', + "downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip', + "downloads.tar:elasticsearch:${version}": ':distribution:archives:tar', + "downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar', + "downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm', + "downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm', + "downloads.deb:elasticsearch:${version}": ':distribution:packages:deb', + "downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb', "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage', "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware', // for transport client @@ -244,13 +244,13 @@ allprojects { bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion -> Version unreleased = unreleasedVersion.version String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}" - ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${unreleased}"] = snapshotProject - ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${unreleased}"] = snapshotProject - ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${unreleased}"] = snapshotProject + ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject + ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject + ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject if (unreleased.onOrAfter('6.3.0')) { - ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch-oss:${unreleased}"] = snapshotProject - ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject - ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch-oss:${unreleased}"] = snapshotProject + ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject + ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject + ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject } } diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index b9ff1fd485e..b98cdd8788b 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -178,6 +178,21 @@ if (project != rootProject) { jarHell.enabled = false thirdPartyAudit.enabled = false + configurations { + distribution + } + + dependencies { + distribution project(':distribution:archives:zip') + distribution project(':distribution:archives:oss-zip') + } + + String localDownloads = "${rootProject.buildDir}/local-downloads" + task setupLocalDownloads(type:Copy) { + from configurations.distribution + into localDownloads + } + unitTest { // The test task is configured to runtimeJava version, but build-tools doesn't support all of them, so test // with compiler instead on the ones that are too old. @@ -192,6 +207,7 @@ if (project != rootProject) { dependsOn project.rootProject.allprojects.collect { it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'} } + dependsOn setupLocalDownloads exclude "**/*Tests.class" testClassesDirs = sourceSets.test.output.classesDirs classpath = sourceSets.test.runtimeClasspath @@ -204,6 +220,7 @@ if (project != rootProject) { ).asPath, ) systemProperty 'test.local-test-repo-path', "${rootProject.buildDir}/local-test-repo" + systemProperty 'test.local-test-downloads-path', localDownloads systemProperty 'test.version_under_test', version systemProperty 'test.lucene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1] } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 8e71883c2ef..af207bcae7c 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -565,6 +565,12 @@ class BuildPlugin implements Plugin { repos.mavenLocal() } repos.jcenter() + repos.ivy { + url "https://artifacts.elastic.co/downloads" + patternLayout { + artifact "elasticsearch/[module]-[revision](-[classifier]).[ext]" + } + } repos.maven { name "elastic" url "https://artifacts.elastic.co/maven" diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index d87e33166e7..2aec6057091 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -18,14 +18,12 @@ */ package org.elasticsearch.gradle.test -import java.util.stream.Collectors import org.apache.tools.ant.DefaultLogger import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.BuildPlugin import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties - import org.elasticsearch.gradle.plugin.PluginBuildPlugin import org.elasticsearch.gradle.plugin.PluginPropertiesExtension import org.gradle.api.AntBuilder @@ -45,7 +43,7 @@ import org.gradle.api.tasks.Exec import java.nio.charset.StandardCharsets import java.nio.file.Paths import java.util.concurrent.TimeUnit - +import java.util.stream.Collectors /** * A helper for creating tasks to build a cluster that is used by a task, and tear down the cluster when the task is finished. */ @@ -174,24 +172,31 @@ class ClusterFormationTasks { /** Adds a dependency on the given distribution */ static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) { - if (Version.fromString(elasticsearchVersion).before('6.3.0') && - distro.startsWith('oss-') - ) { + Version version = Version.fromString(elasticsearchVersion) + if (version.before('6.3.0') && distro.startsWith('oss-')) { distro = distro.substring('oss-'.length()) } + String os = "linux" + if (Os.FAMILY_WINDOWS) { + os = "windows" + } else if (Os.FAMILY_MAC) { + os = "darwin" + } String packaging = distro if (distro.contains('tar')) { packaging = 'tar.gz'\ } else if (distro.contains('zip')) { packaging = 'zip' } - String subgroup = distro + String group = "downloads.${packaging}" + if (distro.equals("integ-test-zip")) { + group = "org.elasticsearch.distribution.integ-test-zip" + } String artifactName = 'elasticsearch' if (distro.contains('oss')) { artifactName += '-oss' - subgroup = distro.substring('oss-'.length()) } - project.dependencies.add(configuration.name, "org.elasticsearch.distribution.${subgroup}:${artifactName}:${elasticsearchVersion}@${packaging}") + project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@${packaging}") } /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */ diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index aabd0cf1192..dfb825afea2 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -193,10 +193,10 @@ class VagrantTestPlugin implements Plugin { UPGRADE_FROM_ARCHIVES.each { // The version of elasticsearch that we upgrade *from* project.dependencies.add(PACKAGING_CONFIGURATION, - "org.elasticsearch.distribution.${it}:elasticsearch:${upgradeFromVersion}@${it}") + "downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}") if (upgradeFromVersion.onOrAfter('6.3.0')) { project.dependencies.add(PACKAGING_CONFIGURATION, - "org.elasticsearch.distribution.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}") + "downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}") } } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java index aca99067011..c0632873eca 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java @@ -127,6 +127,9 @@ public class BuildExamplePluginsIT extends GradleIntegrationTestCase { " maven {\n" + " url \"" + getLocalTestRepoPath() + "\"\n" + " }\n" + + " flatDir {\n" + + " dir '" + getLocalTestDownloadsPath() + "'\n" + + " }\n" + luceneSnapshotRepo + "}\n" ); diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java index 3e1d0b176b0..f7a0382cec7 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java @@ -144,8 +144,16 @@ public abstract class GradleIntegrationTestCase extends GradleUnitTestCase { } protected String getLocalTestRepoPath() { - String property = System.getProperty("test.local-test-repo-path"); - Objects.requireNonNull(property, "test.local-test-repo-path not passed to tests"); + return getLocalTestPath("test.local-test-repo-path"); + } + + protected String getLocalTestDownloadsPath() { + return getLocalTestPath("test.local-test-downloads-path"); + } + + private String getLocalTestPath(String propertyName) { + String property = System.getProperty(propertyName); + Objects.requireNonNull(property, propertyName + " not passed to tests"); File file = new File(property); assertTrue("Expected " + property + " to exist, but it did not!", file.exists()); if (File.separator.equals("\\")) {