diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index dd5ac1b8378..688b07110e9 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -291,6 +291,10 @@ testClusters.integTest { } } +task s3ThirdPartyTests { + dependsOn check +} + if (useFixture) { task integTestECS(type: RestIntegTestTask.class) { description = "Runs tests using the ECS repository." @@ -311,6 +315,13 @@ if (useFixture) { plugin file(tasks.bundlePlugin.archiveFile) environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "http://${s3Fixture.addressAndPort}/ecs_credentials_endpoint" }, IGNORE_VALUE } + + gradle.taskGraph.whenReady { + if (it.hasTask(s3ThirdPartyTests)) { + throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'amazon_s3_access_key', " + + "'amazon_s3_secret_key', 'amazon_s3_bucket', and 'amazon_s3_base_path' are set."); + } + } } thirdPartyAudit.ignoreMissingClasses ( diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java index 7a45a9c9cb5..b16a05f5b99 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java @@ -167,24 +167,7 @@ public abstract class AbstractThirdPartyRepositoryTestCase extends ESSingleNodeT } protected void assertBlobsByPrefix(BlobPath path, String prefix, Map blobs) throws Exception { - final PlainActionFuture> future = PlainActionFuture.newFuture(); - final BlobStoreRepository repository = getRepository(); - repository.threadPool().generic().execute(new ActionRunnable>(future) { - @Override - protected void doRun() throws Exception { - final BlobStore blobStore = repository.blobStore(); - future.onResponse(blobStore.blobContainer(path).listBlobsByPrefix(prefix)); - } - }); - Map foundBlobs = future.actionGet(); - if (blobs.isEmpty()) { - assertThat(foundBlobs.keySet(), empty()); - } else { - assertThat(foundBlobs.keySet(), containsInAnyOrder(blobs.keySet().toArray(Strings.EMPTY_ARRAY))); - for (Map.Entry entry : foundBlobs.entrySet()) { - assertEquals(entry.getValue().length(), blobs.get(entry.getKey()).length()); - } - } + BlobStoreTestUtil.assertBlobsByPrefix(getRepository(), path, prefix, blobs); } public void testCleanup() throws Exception { diff --git a/x-pack/snapshot-tool/build.gradle b/x-pack/snapshot-tool/build.gradle index e2550cf78f4..2986c33f1d2 100644 --- a/x-pack/snapshot-tool/build.gradle +++ b/x-pack/snapshot-tool/build.gradle @@ -8,7 +8,6 @@ import java.nio.file.Paths import org.elasticsearch.gradle.ElasticsearchDistribution apply plugin: 'elasticsearch.build' -apply plugin: 'elasticsearch.test.fixtures' dependencies { compile project(":server") @@ -47,10 +46,24 @@ test { // Disabled to quiet the testing convention check since we only run third party tests test.enabled = false -String s3PermanentAccessKey = 's3_integration_test_permanent_access_key' -String s3PermanentSecretKey = 's3_integration_test_permanent_secret_key' -String s3PermanentBucket = 'permanent-bucket-test' -String s3PermanentBasePath = 'integration_test' +boolean useFixture = false + +String s3PermanentAccessKey = System.getenv("amazon_s3_access_key") +String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key") +String s3PermanentBucket = System.getenv("amazon_s3_bucket") +String s3PermanentBasePath = System.getenv("amazon_s3_base_path") + +if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) { + s3PermanentAccessKey = 's3_integration_test_permanent_access_key' + s3PermanentSecretKey = 's3_integration_test_permanent_secret_key' + s3PermanentBucket = 'permanent-bucket-test' + s3PermanentBasePath = 'integration_test' + + useFixture = true + +} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) { + throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present") +} task thirdPartyTest(type: Test) { include '**/S3CleanupTests.class' @@ -61,32 +74,47 @@ task thirdPartyTest(type: Test) { systemProperty 'test.s3.base', s3PermanentBasePath } -task writeDockerFile { - File minioDockerfile = new File("${project.buildDir}/minio-docker/Dockerfile") - outputs.file(minioDockerfile) - doLast { - minioDockerfile.parentFile.mkdirs() - minioDockerfile.text = - "FROM minio/minio:RELEASE.2019-01-23T23-18-58Z\n" + - "RUN mkdir -p /minio/data/${s3PermanentBucket}\n" + - "ENV MINIO_ACCESS_KEY ${s3PermanentAccessKey}\n" + - "ENV MINIO_SECRET_KEY ${s3PermanentSecretKey}" +task s3ThirdPartyTests { + dependsOn check +} + +if (useFixture) { + apply plugin: 'elasticsearch.test.fixtures' + + task writeDockerFile { + File minioDockerfile = new File("${project.buildDir}/minio-docker/Dockerfile") + outputs.file(minioDockerfile) + doLast { + minioDockerfile.parentFile.mkdirs() + minioDockerfile.text = + "FROM minio/minio:RELEASE.2019-01-23T23-18-58Z\n" + + "RUN mkdir -p /minio/data/${s3PermanentBucket}\n" + + "ENV MINIO_ACCESS_KEY ${s3PermanentAccessKey}\n" + + "ENV MINIO_SECRET_KEY ${s3PermanentSecretKey}" + } } -} -preProcessFixture { - dependsOn(writeDockerFile) -} + preProcessFixture { + dependsOn(writeDockerFile) + } -def minioAddress = { - int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000" - assert minioPort > 0 - 'http://127.0.0.1:' + minioPort -} + def minioAddress = { + int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000" + assert minioPort > 0 + 'http://127.0.0.1:' + minioPort + } -thirdPartyTest { - dependsOn tasks.postProcessFixture - nonInputProperties.systemProperty 'test.s3.endpoint', "${ -> minioAddress.call() }" + thirdPartyTest { + dependsOn tasks.postProcessFixture + nonInputProperties.systemProperty 'test.s3.endpoint', "${ -> minioAddress.call() }" + } + + gradle.taskGraph.whenReady { + if (it.hasTask(s3ThirdPartyTests)) { + throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'amazon_s3_access_key', " + + "'amazon_s3_secret_key', 'amazon_s3_bucket', and 'amazon_s3_base_path' are set."); + } + } } task unpackArchive(dependsOn: tasks.assemble, type: Copy) { diff --git a/x-pack/snapshot-tool/src/test/java/org/elasticsearch/snapshots/S3CleanupTests.java b/x-pack/snapshot-tool/src/test/java/org/elasticsearch/snapshots/S3CleanupTests.java index f4e350aedcb..3358eedf4c8 100644 --- a/x-pack/snapshot-tool/src/test/java/org/elasticsearch/snapshots/S3CleanupTests.java +++ b/x-pack/snapshot-tool/src/test/java/org/elasticsearch/snapshots/S3CleanupTests.java @@ -5,9 +5,12 @@ */ package org.elasticsearch.snapshots; +import com.amazonaws.services.s3.internal.Constants; import joptsimple.OptionSet; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.Terminal; @@ -45,6 +48,17 @@ public class S3CleanupTests extends ESSingleNodeTestCase { super.setUp(); createRepository("test-repo"); repository = (BlobStoreRepository) getInstanceFromNode(RepositoriesService.class).repository("test-repo"); + final PlainActionFuture future = PlainActionFuture.newFuture(); + repository.threadPool().generic().execute(new ActionRunnable(future) { + @Override + protected void doRun() throws Exception { + repository.blobStore().blobContainer(repository.basePath()).delete(); + future.onResponse(null); + } + }); + future.actionGet(); + assertBusy(() -> BlobStoreTestUtil.assertBlobsByPrefix(repository, repository.basePath(), "", Collections.emptyMap()), 10L, + TimeUnit.MINUTES); } @Override @@ -81,7 +95,7 @@ public class S3CleanupTests extends ESSingleNodeTestCase { } private String getEndpoint() { - return System.getProperty("test.s3.endpoint"); + return System.getProperty("test.s3.endpoint", Constants.S3_HOSTNAME); } private String getRegion() {