Use ivy pattern for elasticsearch artifacts download (#37557)

The rpm, deb and tar distributions were removed some time ago from maven
central. The zip distribution still exists there, but it does not need
to. Instead, this commit sets up an ivy repository with pattern pointing
to the elasticsearch artifacts download service. Note that the
integ-test-zip remains in maven central, since it is not present in the
download service.
This commit is contained in:
Ryan Ernst 2019-01-20 23:33:11 -08:00 committed by GitHub
parent 88f4b0a326
commit c9c088ff02
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 66 additions and 27 deletions

View File

@ -221,14 +221,14 @@ allprojects {
"org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi', "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
"org.elasticsearch.test:framework:${version}": ':test:framework', "org.elasticsearch.test:framework:${version}": ':test:framework',
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip', "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
"org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip', "downloads.zip:elasticsearch:${version}": ':distribution:archives:zip',
"org.elasticsearch.distribution.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip', "downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
"org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:archives:tar', "downloads.tar:elasticsearch:${version}": ':distribution:archives:tar',
"org.elasticsearch.distribution.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar', "downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
"org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:packages:rpm', "downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
"org.elasticsearch.distribution.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm', "downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
"org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:packages:deb', "downloads.deb:elasticsearch:${version}": ':distribution:packages:deb',
"org.elasticsearch.distribution.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb', "downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
"org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage', "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
"org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware', "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
// for transport client // for transport client
@ -244,13 +244,13 @@ allprojects {
bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion -> bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
Version unreleased = unreleasedVersion.version Version unreleased = unreleasedVersion.version
String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}" String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${unreleased}"] = snapshotProject ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject
ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${unreleased}"] = snapshotProject ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject
ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${unreleased}"] = snapshotProject ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject
if (unreleased.onOrAfter('6.3.0')) { if (unreleased.onOrAfter('6.3.0')) {
ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch-oss:${unreleased}"] = snapshotProject ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch-oss:${unreleased}"] = snapshotProject ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
} }
} }

View File

@ -178,6 +178,21 @@ if (project != rootProject) {
jarHell.enabled = false jarHell.enabled = false
thirdPartyAudit.enabled = false thirdPartyAudit.enabled = false
configurations {
distribution
}
dependencies {
distribution project(':distribution:archives:zip')
distribution project(':distribution:archives:oss-zip')
}
String localDownloads = "${rootProject.buildDir}/local-downloads"
task setupLocalDownloads(type:Copy) {
from configurations.distribution
into localDownloads
}
unitTest { unitTest {
// The test task is configured to runtimeJava version, but build-tools doesn't support all of them, so test // The test task is configured to runtimeJava version, but build-tools doesn't support all of them, so test
// with compiler instead on the ones that are too old. // with compiler instead on the ones that are too old.
@ -192,6 +207,7 @@ if (project != rootProject) {
dependsOn project.rootProject.allprojects.collect { dependsOn project.rootProject.allprojects.collect {
it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'} it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'}
} }
dependsOn setupLocalDownloads
exclude "**/*Tests.class" exclude "**/*Tests.class"
testClassesDirs = sourceSets.test.output.classesDirs testClassesDirs = sourceSets.test.output.classesDirs
classpath = sourceSets.test.runtimeClasspath classpath = sourceSets.test.runtimeClasspath
@ -204,6 +220,7 @@ if (project != rootProject) {
).asPath, ).asPath,
) )
systemProperty 'test.local-test-repo-path', "${rootProject.buildDir}/local-test-repo" systemProperty 'test.local-test-repo-path', "${rootProject.buildDir}/local-test-repo"
systemProperty 'test.local-test-downloads-path', localDownloads
systemProperty 'test.version_under_test', version systemProperty 'test.version_under_test', version
systemProperty 'test.lucene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1] systemProperty 'test.lucene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1]
} }

View File

@ -565,6 +565,12 @@ class BuildPlugin implements Plugin<Project> {
repos.mavenLocal() repos.mavenLocal()
} }
repos.jcenter() repos.jcenter()
repos.ivy {
url "https://artifacts.elastic.co/downloads"
patternLayout {
artifact "elasticsearch/[module]-[revision](-[classifier]).[ext]"
}
}
repos.maven { repos.maven {
name "elastic" name "elastic"
url "https://artifacts.elastic.co/maven" url "https://artifacts.elastic.co/maven"

View File

@ -18,14 +18,12 @@
*/ */
package org.elasticsearch.gradle.test package org.elasticsearch.gradle.test
import java.util.stream.Collectors
import org.apache.tools.ant.DefaultLogger import org.apache.tools.ant.DefaultLogger
import org.apache.tools.ant.taskdefs.condition.Os import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.BuildPlugin import org.elasticsearch.gradle.BuildPlugin
import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.plugin.PluginBuildPlugin import org.elasticsearch.gradle.plugin.PluginBuildPlugin
import org.elasticsearch.gradle.plugin.PluginPropertiesExtension import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
import org.gradle.api.AntBuilder import org.gradle.api.AntBuilder
@ -45,7 +43,7 @@ import org.gradle.api.tasks.Exec
import java.nio.charset.StandardCharsets import java.nio.charset.StandardCharsets
import java.nio.file.Paths import java.nio.file.Paths
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import java.util.stream.Collectors
/** /**
* A helper for creating tasks to build a cluster that is used by a task, and tear down the cluster when the task is finished. * A helper for creating tasks to build a cluster that is used by a task, and tear down the cluster when the task is finished.
*/ */
@ -174,24 +172,31 @@ class ClusterFormationTasks {
/** Adds a dependency on the given distribution */ /** Adds a dependency on the given distribution */
static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) { static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
if (Version.fromString(elasticsearchVersion).before('6.3.0') && Version version = Version.fromString(elasticsearchVersion)
distro.startsWith('oss-') if (version.before('6.3.0') && distro.startsWith('oss-')) {
) {
distro = distro.substring('oss-'.length()) distro = distro.substring('oss-'.length())
} }
String os = "linux"
if (Os.FAMILY_WINDOWS) {
os = "windows"
} else if (Os.FAMILY_MAC) {
os = "darwin"
}
String packaging = distro String packaging = distro
if (distro.contains('tar')) { if (distro.contains('tar')) {
packaging = 'tar.gz'\ packaging = 'tar.gz'\
} else if (distro.contains('zip')) { } else if (distro.contains('zip')) {
packaging = 'zip' packaging = 'zip'
} }
String subgroup = distro String group = "downloads.${packaging}"
if (distro.equals("integ-test-zip")) {
group = "org.elasticsearch.distribution.integ-test-zip"
}
String artifactName = 'elasticsearch' String artifactName = 'elasticsearch'
if (distro.contains('oss')) { if (distro.contains('oss')) {
artifactName += '-oss' artifactName += '-oss'
subgroup = distro.substring('oss-'.length())
} }
project.dependencies.add(configuration.name, "org.elasticsearch.distribution.${subgroup}:${artifactName}:${elasticsearchVersion}@${packaging}") project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@${packaging}")
} }
/** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */ /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */

View File

@ -193,10 +193,10 @@ class VagrantTestPlugin implements Plugin<Project> {
UPGRADE_FROM_ARCHIVES.each { UPGRADE_FROM_ARCHIVES.each {
// The version of elasticsearch that we upgrade *from* // The version of elasticsearch that we upgrade *from*
project.dependencies.add(PACKAGING_CONFIGURATION, project.dependencies.add(PACKAGING_CONFIGURATION,
"org.elasticsearch.distribution.${it}:elasticsearch:${upgradeFromVersion}@${it}") "downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
if (upgradeFromVersion.onOrAfter('6.3.0')) { if (upgradeFromVersion.onOrAfter('6.3.0')) {
project.dependencies.add(PACKAGING_CONFIGURATION, project.dependencies.add(PACKAGING_CONFIGURATION,
"org.elasticsearch.distribution.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}") "downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
} }
} }

View File

@ -127,6 +127,9 @@ public class BuildExamplePluginsIT extends GradleIntegrationTestCase {
" maven {\n" + " maven {\n" +
" url \"" + getLocalTestRepoPath() + "\"\n" + " url \"" + getLocalTestRepoPath() + "\"\n" +
" }\n" + " }\n" +
" flatDir {\n" +
" dir '" + getLocalTestDownloadsPath() + "'\n" +
" }\n" +
luceneSnapshotRepo + luceneSnapshotRepo +
"}\n" "}\n"
); );

View File

@ -144,8 +144,16 @@ public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {
} }
protected String getLocalTestRepoPath() { protected String getLocalTestRepoPath() {
String property = System.getProperty("test.local-test-repo-path"); return getLocalTestPath("test.local-test-repo-path");
Objects.requireNonNull(property, "test.local-test-repo-path not passed to tests"); }
protected String getLocalTestDownloadsPath() {
return getLocalTestPath("test.local-test-downloads-path");
}
private String getLocalTestPath(String propertyName) {
String property = System.getProperty(propertyName);
Objects.requireNonNull(property, propertyName + " not passed to tests");
File file = new File(property); File file = new File(property);
assertTrue("Expected " + property + " to exist, but it did not!", file.exists()); assertTrue("Expected " + property + " to exist, but it did not!", file.exists());
if (File.separator.equals("\\")) { if (File.separator.equals("\\")) {