Update DistributionDownloader to fetch snapshots and staging bundles. (#904)

This change updates DistributionDownloader by registering a second
repository containing release-candidate bundles.  This repository will
only be checked if the release repository does not contain the requested
version.

This change also updates the snapshot repository to point to a new
snapshot repository.

Signed-off-by: Marc Handalian <handalm@amazon.com>

buildSrc/src/integTest/groovy/org/opensearch/gradle/fixtures/DistributionDownloadFixture.groovy

@@ -67,7 +67,10 @@ class DistributionDownloadFixture {
         String fileType = ((platform == OpenSearchDistribution.Platform.LINUX ||
                 platform == OpenSearchDistribution.Platform.DARWIN)) ? "tar.gz" : "zip"
         if (Version.fromString(version).onOrAfter(Version.fromString("1.0.0"))) {
-            return "/releases/core/opensearch/${version}/opensearch-${version}-${platform}-x64.$fileType"
+            if (version.contains("SNAPSHOT")) {
+                return "/snapshots/core/opensearch/${version}/opensearch-min-${version}-${platform}-x64.$fileType"
+            }
+            return "/releases/core/opensearch/${version}/opensearch-min-${version}-${platform}-x64.$fileType"
         } else {
             return "/downloads/elasticsearch/elasticsearch-oss-${version}-${platform}-x86_64.$fileType"
         }

buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java

@@ -49,7 +49,10 @@ import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
 import org.gradle.api.internal.artifacts.ArtifactAttributes;
 import org.gradle.api.provider.Provider;
 
+import java.util.Arrays;
 import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * A plugin to manage getting and extracting distributions of OpenSearch.
@@ -74,6 +77,10 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
     private static final String SNAPSHOT_REPO_NAME_ES = "elasticsearch-snapshots";
     private static final String FAKE_SNAPSHOT_IVY_GROUP_ES = "elasticsearch-distribution-snapshot";
 
+    private static final String RELEASE_PATTERN_LAYOUT = "/core/opensearch/[revision]/[module]-min-[revision](-[classifier]).[ext]";
+    private static final String SNAPSHOT_PATTERN_LAYOUT =
+        "/snapshots/core/opensearch/[revision]/[module]-min-[revision](-[classifier]).[ext]";
+
     private NamedDomainObjectContainer<OpenSearchDistribution> distributionsContainer;
     private NamedDomainObjectContainer<DistributionResolution> distributionsResolutionStrategiesContainer;
 
@@ -157,16 +164,17 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
             .orElseGet(() -> DistributionDependency.of(dependencyNotation(distribution)));
     }
 
-    private static void addIvyRepo(Project project, String name, String url, String group) {
+    private static void addIvyRepo(Project project, String name, String url, String group, String... patternLayout) {
-        IvyArtifactRepository ivyRepo = project.getRepositories().ivy(repo -> {
+        final List<IvyArtifactRepository> repos = Arrays.stream(patternLayout).map(pattern -> project.getRepositories().ivy(repo -> {
             repo.setName(name);
             repo.setUrl(url);
             repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
-            repo.patternLayout(layout -> layout.artifact("/releases/core/opensearch/[revision]/[module]-[revision](-[classifier]).[ext]"));
+            repo.patternLayout(layout -> layout.artifact(pattern));
-        });
+        })).collect(Collectors.toList());
+
         project.getRepositories().exclusiveContent(exclusiveContentRepository -> {
             exclusiveContentRepository.filter(config -> config.includeGroup(group));
-            exclusiveContentRepository.forRepositories(ivyRepo);
+            exclusiveContentRepository.forRepositories(repos.toArray(new IvyArtifactRepository[repos.size()]));
         });
     }
 
@@ -187,8 +195,17 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
         if (project.getRepositories().findByName(DOWNLOAD_REPO_NAME) != null) {
             return;
         }
-        addIvyRepo(project, DOWNLOAD_REPO_NAME, "https://artifacts.opensearch.org", FAKE_IVY_GROUP);
+        addIvyRepo(
-        addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://snapshots-no-kpi.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP);
+            project,
+            DOWNLOAD_REPO_NAME,
+            "https://artifacts.opensearch.org",
+            FAKE_IVY_GROUP,
+            "/releases" + RELEASE_PATTERN_LAYOUT,
+            "/release-candidates" + RELEASE_PATTERN_LAYOUT
+        );
+
+        addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://artifacts.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, SNAPSHOT_PATTERN_LAYOUT);
+
         addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES);
         addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES);
     }