Add Infrastructure to Run 3rd Party Repository Tests (#42586) (#42604)

* Add Infrastructure to Run 3rd Party Repository Tests

* Add infrastructure to run third party repository tests using our standard JUnit infrastructure
* This is a prerequisite of #42189

plugins/repository-azure/build.gradle

@@ -71,3 +71,24 @@ testClusters {
         keystore 'azure.client.integration_test.key', 'azure_key'
     }
 }
+
+String azureAccount = System.getenv("azure_storage_account")
+String azureKey = System.getenv("azure_storage_key")
+String azureContainer = System.getenv("azure_storage_container")
+String azureBasePath = System.getenv("azure_storage_base_path")
+
+test {
+  exclude '**/AzureStorageCleanupThirdPartyTests.class'
+}
+
+task thirdPartyTest(type: Test) {
+  include '**/AzureStorageCleanupThirdPartyTests.class'
+  systemProperty 'test.azure.account', azureAccount ? azureAccount : ""
+  systemProperty 'test.azure.key', azureKey ? azureKey : ""
+  systemProperty 'test.azure.container', azureContainer ? azureContainer : ""
+  systemProperty 'test.azure.base', azureBasePath ? azureBasePath : ""
+}
+
+if (azureAccount || azureKey || azureContainer || azureBasePath) {
+  check.dependsOn(thirdPartyTest)
+}

plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.azure;
+
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
+
+import java.util.Collection;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(AzureRepositoryPlugin.class);
+    }
+
+    @Override
+    protected SecureSettings credentials() {
+        assertThat(System.getProperty("test.azure.account"), not(blankOrNullString()));
+        assertThat(System.getProperty("test.azure.key"), not(blankOrNullString()));
+        assertThat(System.getProperty("test.azure.container"), not(blankOrNullString()));
+        assertThat(System.getProperty("test.azure.base"), not(blankOrNullString()));
+
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("azure.client.default.account", System.getProperty("test.azure.account"));
+        secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key"));
+        return secureSettings;
+    }
+
+    @Override
+    protected void createRepository(String repoName) {
+        AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository(repoName)
+            .setType("azure")
+            .setSettings(Settings.builder()
+                .put("container", System.getProperty("test.azure.container"))
+                .put("base_path", System.getProperty("test.azure.base"))
+            ).get();
+        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
+    }
+}

plugins/repository-gcs/build.gradle

@@ -1,3 +1,5 @@
+import java.nio.file.Files
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -122,3 +124,22 @@ check {
   // also execute the QA tests when testing the plugin
   dependsOn 'qa:google-cloud-storage:check'
 }
+
+String gcsServiceAccount = System.getenv("google_storage_service_account")
+String gcsBucket = System.getenv("google_storage_bucket")
+String gcsBasePath = System.getenv("google_storage_base_path")
+
+test {
+  exclude '**/GoogleCloudStorageThirdPartyTests.class'
+}
+
+task thirdPartyTest(type: Test) {
+  include '**/GoogleCloudStorageThirdPartyTests.class'
+  systemProperty 'test.google.account', gcsServiceAccount ? Base64.encoder.encodeToString(Files.readAllBytes(file(gcsServiceAccount).toPath())) : ""
+  systemProperty 'test.google.bucket', gcsBucket ? gcsBucket : ""
+  systemProperty 'test.google.base', gcsBasePath ? gcsBasePath : "/"
+}
+
+if (gcsServiceAccount || gcsBucket || gcsBasePath) {
+  check.dependsOn(thirdPartyTest)
+}

plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.gcs;
+
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
+
+import java.util.Base64;
+import java.util.Collection;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+public class GoogleCloudStorageThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(GoogleCloudStoragePlugin.class);
+    }
+
+    @Override
+    protected SecureSettings credentials() {
+        assertThat(System.getProperty("test.google.account"), not(blankOrNullString()));
+        assertThat(System.getProperty("test.google.bucket"), not(blankOrNullString()));
+
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setFile("gcs.client.default.credentials_file",
+            Base64.getDecoder().decode(System.getProperty("test.google.account")));
+        return secureSettings;
+    }
+
+    @Override
+    protected void createRepository(final String repoName) {
+        AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
+            .setType("gcs")
+            .setSettings(Settings.builder()
+                .put("bucket", System.getProperty("test.google.bucket"))
+                .put("base_path", System.getProperty("test.google.base", "/"))
+            ).get();
+        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
+    }
+}

plugins/repository-s3/build.gradle

@@ -75,6 +75,7 @@ test {
   // these are tested explicitly in separate test tasks
   exclude '**/*CredentialsTests.class'
   exclude '**/S3BlobStoreRepositoryTests.class'
+  exclude '**/S3RepositoryThirdPartyTests.class'
 }
 
 boolean useFixture = false
@@ -134,6 +135,14 @@ if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
   throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
 }
 
+task thirdPartyTest(type: Test) {
+  include '**/S3RepositoryThirdPartyTests.class'
+  systemProperty 'test.s3.account', s3PermanentAccessKey
+  systemProperty 'test.s3.key', s3PermanentSecretKey
+  systemProperty 'test.s3.bucket', s3PermanentBucket
+  systemProperty 'test.s3.base', s3PermanentBasePath
+}
+
 if (useFixture) {
   apply plugin: 'elasticsearch.test.fixtures'
   task writeDockerFile {
@@ -151,6 +160,32 @@ if (useFixture) {
     dependsOn(writeDockerFile)
   }
 
+  def minioAddress = {
+    int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
+    assert minioPort > 0
+    return 'http://127.0.0.1:' + minioPort
+  }
+
+  File minioAddressFile = new File(project.buildDir, 'generated-resources/s3Fixture.address')
+
+  // We can't lazy evaluate a system property for the Minio address passed to JUnit so we write it to a resource file
+  // and pass its name instead.
+  task writeMinioAddress {
+    dependsOn tasks.bundlePlugin, tasks.postProcessFixture
+    outputs.file(minioAddressFile)
+    doLast {
+      file(minioAddressFile).text = "${ -> minioAddress.call() }"
+    }
+  }
+
+  thirdPartyTest {
+    dependsOn writeMinioAddress
+    inputs.file(minioAddressFile)
+    systemProperty 'test.s3.endpoint', minioAddressFile.name
+  }
+
+  BuildPlugin.requireDocker(tasks.thirdPartyTest)
+
   task integTestMinio(type: RestIntegTestTask) {
     description = "Runs REST tests using the Minio repository."
     dependsOn tasks.bundlePlugin, tasks.postProcessFixture
@@ -169,11 +204,7 @@ if (useFixture) {
   testClusters.integTestMinio {
     keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
     keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
-    setting 's3.client.integration_test_permanent.endpoint', {
-      int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
-      assert minioPort > 0
-      return 'http://127.0.0.1:' + minioPort
-    }
+    setting 's3.client.integration_test_permanent.endpoint', minioAddress
     plugin file(tasks.bundlePlugin.archiveFile)
   }
 
@@ -191,6 +222,8 @@ if (useFixture) {
   }
 }
 
+check.dependsOn(thirdPartyTest)
+
 File parentFixtures = new File(project.buildDir, "fixtures")
 File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties')
 

plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.repositories.s3;
+
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
+import org.elasticsearch.test.StreamsUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(S3RepositoryPlugin.class);
+    }
+
+    @Override
+    protected SecureSettings credentials() {
+        assertThat(System.getProperty("test.s3.account"), not(blankOrNullString()));
+        assertThat(System.getProperty("test.s3.key"), not(blankOrNullString()));
+        assertThat(System.getProperty("test.s3.bucket"), not(blankOrNullString()));
+
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.default.access_key", System.getProperty("test.s3.account"));
+        secureSettings.setString("s3.client.default.secret_key", System.getProperty("test.s3.key"));
+        return secureSettings;
+    }
+
+    @Override
+    protected void createRepository(String repoName) {
+        Settings.Builder settings = Settings.builder()
+            .put("bucket", System.getProperty("test.s3.bucket"))
+            .put("base_path", System.getProperty("test.s3.base", "/"));
+        final String endpointPath = System.getProperty("test.s3.endpoint");
+        if (endpointPath != null) {
+            try {
+                settings = settings.put("endpoint", StreamsUtils.copyToStringFromClasspath("/" + endpointPath));
+            } catch (IOException e) {
+                throw new AssertionError(e);
+            }
+        }
+        AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
+            .setType("s3")
+            .setSettings(settings).get();
+        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
+    }
+}

test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.repositories;
+
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.snapshots.SnapshotState;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public abstract class AbstractThirdPartyRepositoryTestCase extends ESSingleNodeTestCase {
+
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder()
+            .put(super.nodeSettings())
+            .setSecureSettings(credentials())
+            .build();
+    }
+
+    protected abstract SecureSettings credentials();
+
+    protected abstract void createRepository(String repoName);
+
+
+    public void testCreateSnapshot() {
+        createRepository("test-repo");
+
+        createIndex("test-idx-1");
+        createIndex("test-idx-2");
+        createIndex("test-idx-3");
+        ensureGreen();
+
+        logger.info("--> indexing some data");
+        for (int i = 0; i < 100; i++) {
+            client().prepareIndex("test-idx-1", "doc", Integer.toString(i)).setSource("foo", "bar" + i).get();
+            client().prepareIndex("test-idx-2", "doc", Integer.toString(i)).setSource("foo", "bar" + i).get();
+            client().prepareIndex("test-idx-3", "doc", Integer.toString(i)).setSource("foo", "bar" + i).get();
+        }
+        client().admin().indices().prepareRefresh().get();
+
+        final String snapshotName = "test-snap-" + System.currentTimeMillis();
+
+        logger.info("--> snapshot");
+        CreateSnapshotResponse createSnapshotResponse = client().admin()
+            .cluster()
+            .prepareCreateSnapshot("test-repo", snapshotName)
+            .setWaitForCompletion(true)
+            .setIndices("test-idx-*", "-test-idx-3")
+            .get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
+            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+
+        assertThat(client().admin()
+                .cluster()
+                .prepareGetSnapshots("test-repo")
+                .setSnapshots(snapshotName)
+                .get()
+                .getSnapshots()
+                .get(0)
+                .state(),
+            equalTo(SnapshotState.SUCCESS));
+
+        assertTrue(client().admin()
+                .cluster()
+                .prepareDeleteSnapshot("test-repo", snapshotName)
+                .get()
+                .isAcknowledged());
+
+    }
+}