From af4ad4fe8db0c003b2d6e4edda734b0bc48ff25a Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 30 Jun 2021 11:32:55 -0700 Subject: [PATCH] Update DistributionDownloader to fetch snapshots and staging bundles. (#904) (#907) This change updates DistributionDownloader by registering a second repository containing release-candidate bundles. This repository will only be checked if the release repository does not contain the requested version. This change also updates the snapshot repository to point to a new snapshot repository. Signed-off-by: Marc Handalian --- .../DistributionDownloadFixture.groovy | 5 ++- .../gradle/DistributionDownloadPlugin.java | 31 ++++++++++++++----- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/buildSrc/src/integTest/groovy/org/opensearch/gradle/fixtures/DistributionDownloadFixture.groovy b/buildSrc/src/integTest/groovy/org/opensearch/gradle/fixtures/DistributionDownloadFixture.groovy index baf03c08be0..f39eecb2015 100644 --- a/buildSrc/src/integTest/groovy/org/opensearch/gradle/fixtures/DistributionDownloadFixture.groovy +++ b/buildSrc/src/integTest/groovy/org/opensearch/gradle/fixtures/DistributionDownloadFixture.groovy @@ -67,7 +67,10 @@ class DistributionDownloadFixture { String fileType = ((platform == OpenSearchDistribution.Platform.LINUX || platform == OpenSearchDistribution.Platform.DARWIN)) ? "tar.gz" : "zip" if (Version.fromString(version).onOrAfter(Version.fromString("1.0.0"))) { - return "/releases/core/opensearch/${version}/opensearch-${version}-${platform}-x64.$fileType" + if (version.contains("SNAPSHOT")) { + return "/snapshots/core/opensearch/${version}/opensearch-min-${version}-${platform}-x64.$fileType" + } + return "/releases/core/opensearch/${version}/opensearch-min-${version}-${platform}-x64.$fileType" } else { return "/downloads/elasticsearch/elasticsearch-oss-${version}-${platform}-x86_64.$fileType" } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java index a53860bdc84..868bf8af0c0 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java @@ -49,7 +49,10 @@ import org.gradle.api.artifacts.type.ArtifactTypeDefinition; import org.gradle.api.internal.artifacts.ArtifactAttributes; import org.gradle.api.provider.Provider; +import java.util.Arrays; import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; /** * A plugin to manage getting and extracting distributions of OpenSearch. @@ -74,6 +77,10 @@ public class DistributionDownloadPlugin implements Plugin { private static final String SNAPSHOT_REPO_NAME_ES = "elasticsearch-snapshots"; private static final String FAKE_SNAPSHOT_IVY_GROUP_ES = "elasticsearch-distribution-snapshot"; + private static final String RELEASE_PATTERN_LAYOUT = "/core/opensearch/[revision]/[module]-min-[revision](-[classifier]).[ext]"; + private static final String SNAPSHOT_PATTERN_LAYOUT = + "/snapshots/core/opensearch/[revision]/[module]-min-[revision](-[classifier]).[ext]"; + private NamedDomainObjectContainer distributionsContainer; private NamedDomainObjectContainer distributionsResolutionStrategiesContainer; @@ -157,16 +164,17 @@ public class DistributionDownloadPlugin implements Plugin { .orElseGet(() -> DistributionDependency.of(dependencyNotation(distribution))); } - private static void addIvyRepo(Project project, String name, String url, String group) { - IvyArtifactRepository ivyRepo = project.getRepositories().ivy(repo -> { + private static void addIvyRepo(Project project, String name, String url, String group, String... patternLayout) { + final List repos = Arrays.stream(patternLayout).map(pattern -> project.getRepositories().ivy(repo -> { repo.setName(name); repo.setUrl(url); repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact); - repo.patternLayout(layout -> layout.artifact("/releases/core/opensearch/[revision]/[module]-[revision](-[classifier]).[ext]")); - }); + repo.patternLayout(layout -> layout.artifact(pattern)); + })).collect(Collectors.toList()); + project.getRepositories().exclusiveContent(exclusiveContentRepository -> { exclusiveContentRepository.filter(config -> config.includeGroup(group)); - exclusiveContentRepository.forRepositories(ivyRepo); + exclusiveContentRepository.forRepositories(repos.toArray(new IvyArtifactRepository[repos.size()])); }); } @@ -187,8 +195,17 @@ public class DistributionDownloadPlugin implements Plugin { if (project.getRepositories().findByName(DOWNLOAD_REPO_NAME) != null) { return; } - addIvyRepo(project, DOWNLOAD_REPO_NAME, "https://artifacts.opensearch.org", FAKE_IVY_GROUP); - addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://snapshots-no-kpi.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP); + addIvyRepo( + project, + DOWNLOAD_REPO_NAME, + "https://artifacts.opensearch.org", + FAKE_IVY_GROUP, + "/releases" + RELEASE_PATTERN_LAYOUT, + "/release-candidates" + RELEASE_PATTERN_LAYOUT + ); + + addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://artifacts.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, SNAPSHOT_PATTERN_LAYOUT); + addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES); addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES); }