S3 3rd Party Test Goal (#44799) (#45004)

* Create S3 Third Party Test Task that Covers the S3 CLI Tool
* Adjust snapshot cli test tool tests to work with real S3
  * Build adjustment
  * Clean up repo path before testing
* Dedup the logic for asserting path contents by using the correct utility method here that somehow became unused

plugins/repository-s3/build.gradle

@@ -291,6 +291,10 @@ testClusters.integTest {
   }
 }
 
+task s3ThirdPartyTests {
+  dependsOn check
+}
+
 if (useFixture) {
   task integTestECS(type: RestIntegTestTask.class) {
     description = "Runs tests using the ECS repository."
@@ -311,6 +315,13 @@ if (useFixture) {
     plugin file(tasks.bundlePlugin.archiveFile)
     environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "http://${s3Fixture.addressAndPort}/ecs_credentials_endpoint" }, IGNORE_VALUE
   }
+
+  gradle.taskGraph.whenReady {
+    if (it.hasTask(s3ThirdPartyTests)) {
+      throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'amazon_s3_access_key', " +
+        "'amazon_s3_secret_key', 'amazon_s3_bucket', and 'amazon_s3_base_path' are set.");
+    }
+  }
 }
 
 thirdPartyAudit.ignoreMissingClasses (

test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java

@@ -167,24 +167,7 @@ public abstract class AbstractThirdPartyRepositoryTestCase extends ESSingleNodeT
     }
 
     protected void assertBlobsByPrefix(BlobPath path, String prefix, Map<String, BlobMetaData> blobs) throws Exception {
-        final PlainActionFuture<Map<String, BlobMetaData>> future = PlainActionFuture.newFuture();
-        final BlobStoreRepository repository = getRepository();
-        repository.threadPool().generic().execute(new ActionRunnable<Map<String, BlobMetaData>>(future) {
-            @Override
-            protected void doRun() throws Exception {
-                final BlobStore blobStore = repository.blobStore();
-                future.onResponse(blobStore.blobContainer(path).listBlobsByPrefix(prefix));
-            }
-        });
-        Map<String, BlobMetaData> foundBlobs = future.actionGet();
-        if (blobs.isEmpty()) {
-            assertThat(foundBlobs.keySet(), empty());
-        } else {
-            assertThat(foundBlobs.keySet(), containsInAnyOrder(blobs.keySet().toArray(Strings.EMPTY_ARRAY)));
-            for (Map.Entry<String, BlobMetaData> entry : foundBlobs.entrySet()) {
-                assertEquals(entry.getValue().length(), blobs.get(entry.getKey()).length());
-            }
-        }
+        BlobStoreTestUtil.assertBlobsByPrefix(getRepository(), path, prefix, blobs);
     }
 
     public void testCleanup() throws Exception {

x-pack/snapshot-tool/build.gradle

@@ -8,7 +8,6 @@ import java.nio.file.Paths
 import org.elasticsearch.gradle.ElasticsearchDistribution
 
 apply plugin: 'elasticsearch.build'
-apply plugin: 'elasticsearch.test.fixtures'
 
 dependencies {
   compile project(":server")
@@ -47,10 +46,24 @@ test {
 // Disabled to quiet the testing convention check since we only run third party tests
 test.enabled = false
 
-String s3PermanentAccessKey = 's3_integration_test_permanent_access_key'
-String s3PermanentSecretKey = 's3_integration_test_permanent_secret_key'
-String s3PermanentBucket = 'permanent-bucket-test'
-String s3PermanentBasePath = 'integration_test'
+boolean useFixture = false
+
+String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
+String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
+String s3PermanentBucket = System.getenv("amazon_s3_bucket")
+String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
+
+if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) {
+  s3PermanentAccessKey = 's3_integration_test_permanent_access_key'
+  s3PermanentSecretKey = 's3_integration_test_permanent_secret_key'
+  s3PermanentBucket = 'permanent-bucket-test'
+  s3PermanentBasePath = 'integration_test'
+
+  useFixture = true
+
+} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
+  throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
+}
 
 task thirdPartyTest(type: Test) {
   include '**/S3CleanupTests.class'
@@ -61,32 +74,47 @@ task thirdPartyTest(type: Test) {
   systemProperty 'test.s3.base', s3PermanentBasePath
 }
 
-task writeDockerFile {
-  File minioDockerfile = new File("${project.buildDir}/minio-docker/Dockerfile")
-  outputs.file(minioDockerfile)
-  doLast {
-    minioDockerfile.parentFile.mkdirs()
-    minioDockerfile.text = 
-            "FROM minio/minio:RELEASE.2019-01-23T23-18-58Z\n" +
-            "RUN mkdir -p /minio/data/${s3PermanentBucket}\n" +
-            "ENV MINIO_ACCESS_KEY ${s3PermanentAccessKey}\n" +
-            "ENV MINIO_SECRET_KEY ${s3PermanentSecretKey}"
+task s3ThirdPartyTests {
+  dependsOn check
+}
+
+if (useFixture) {
+  apply plugin: 'elasticsearch.test.fixtures'
+
+  task writeDockerFile {
+    File minioDockerfile = new File("${project.buildDir}/minio-docker/Dockerfile")
+    outputs.file(minioDockerfile)
+    doLast {
+      minioDockerfile.parentFile.mkdirs()
+      minioDockerfile.text =
+        "FROM minio/minio:RELEASE.2019-01-23T23-18-58Z\n" +
+          "RUN mkdir -p /minio/data/${s3PermanentBucket}\n" +
+          "ENV MINIO_ACCESS_KEY ${s3PermanentAccessKey}\n" +
+          "ENV MINIO_SECRET_KEY ${s3PermanentSecretKey}"
+    }
   }
-}
 
-preProcessFixture {
-  dependsOn(writeDockerFile)
-}
+  preProcessFixture {
+    dependsOn(writeDockerFile)
+  }
 
-def minioAddress = {
-  int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
-  assert minioPort > 0
-  'http://127.0.0.1:' + minioPort
-}
+  def minioAddress = {
+    int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
+    assert minioPort > 0
+    'http://127.0.0.1:' + minioPort
+  }
 
-thirdPartyTest {
-  dependsOn tasks.postProcessFixture
-  nonInputProperties.systemProperty 'test.s3.endpoint', "${ -> minioAddress.call() }"
+  thirdPartyTest {
+    dependsOn tasks.postProcessFixture
+    nonInputProperties.systemProperty 'test.s3.endpoint', "${ -> minioAddress.call() }"
+  }
+
+  gradle.taskGraph.whenReady {
+    if (it.hasTask(s3ThirdPartyTests)) {
+      throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'amazon_s3_access_key', " +
+        "'amazon_s3_secret_key', 'amazon_s3_bucket', and 'amazon_s3_base_path' are set.");
+    }
+  }
 }
 
 task unpackArchive(dependsOn: tasks.assemble, type: Copy) {

x-pack/snapshot-tool/src/test/java/org/elasticsearch/snapshots/S3CleanupTests.java

@@ -5,9 +5,12 @@
  */
 package org.elasticsearch.snapshots;
 
+import com.amazonaws.services.s3.internal.Constants;
 import joptsimple.OptionSet;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.cli.Terminal;
@@ -45,6 +48,17 @@ public class S3CleanupTests extends ESSingleNodeTestCase {
         super.setUp();
         createRepository("test-repo");
         repository = (BlobStoreRepository) getInstanceFromNode(RepositoriesService.class).repository("test-repo");
+        final PlainActionFuture<Void> future = PlainActionFuture.newFuture();
+        repository.threadPool().generic().execute(new ActionRunnable<Void>(future) {
+            @Override
+            protected void doRun() throws Exception {
+                repository.blobStore().blobContainer(repository.basePath()).delete();
+                future.onResponse(null);
+            }
+        });
+        future.actionGet();
+        assertBusy(() -> BlobStoreTestUtil.assertBlobsByPrefix(repository, repository.basePath(), "", Collections.emptyMap()), 10L,
+            TimeUnit.MINUTES);
     }
 
     @Override
@@ -81,7 +95,7 @@ public class S3CleanupTests extends ESSingleNodeTestCase {
     }
 
     private String getEndpoint() {
-        return System.getProperty("test.s3.endpoint");
+        return System.getProperty("test.s3.endpoint", Constants.S3_HOSTNAME);
     }
 
     private String getRegion() {