diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 77ebcfec532..6d7412f8250 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ -import org.elasticsearch.gradle.test.AntFixture esplugin { description 'The Azure Repository plugin adds support for Azure storage repositories.' @@ -43,28 +42,12 @@ thirdPartyAudit.excludes = [ 'org.slf4j.LoggerFactory', ] -forbiddenApisTest { - // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage - bundledSignatures -= 'jdk-non-portable' - bundledSignatures += 'jdk-internal' -} - -/** A task to start the fixture which emulates an Azure Storage service **/ -task azureStorageFixture(type: AntFixture) { - dependsOn compileTestJava - env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" - executable = new File(project.runtimeJavaHome, 'bin/java') - args 'org.elasticsearch.repositories.azure.AzureStorageFixture', baseDir, 'container_test' +check { + // also execute the QA tests when testing the plugin + dependsOn 'qa:microsoft-azure-storage:check' } integTestCluster { - dependsOn azureStorageFixture - - keystoreSetting 'azure.client.integration_test.account', "azure_integration_test_account" - /* The key is "azure_integration_test_key" encoded using base64 */ - keystoreSetting 'azure.client.integration_test.key', "YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=" - // Use a closure on the string to delay evaluation until tests are executed. The endpoint_suffix is used - // in a hacky way to change the protocol and endpoint. We must fix that. - setting 'azure.client.integration_test.endpoint_suffix', - "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${ -> azureStorageFixture.addressAndPort }" -} \ No newline at end of file + keystoreSetting 'azure.client.integration_test.account', 'azure_account' + keystoreSetting 'azure.client.integration_test.key', 'azure_key' +} diff --git a/plugins/repository-azure/qa/build.gradle b/plugins/repository-azure/qa/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle new file mode 100644 index 00000000000..c7deb8a9c76 --- /dev/null +++ b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.test.AntFixture + +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testCompile project(path: ':plugins:repository-azure', configuration: 'runtime') +} + +integTestCluster { + plugin ':plugins:repository-azure' +} + +forbiddenApisTest { + // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage + bundledSignatures -= 'jdk-non-portable' + bundledSignatures += 'jdk-internal' +} + +boolean useFixture = false + +String azureAccount = System.getenv("azure_storage_account") +String azureKey = System.getenv("azure_storage_key") +String azureContainer = System.getenv("azure_storage_container") +String azureBasePath = System.getenv("azure_storage_base_path") + +if (!azureAccount && !azureKey && !azureContainer && !azureBasePath) { + azureAccount = 'azure_integration_test_account' + azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64 + azureContainer = 'container_test' + azureBasePath = 'integration_test' + useFixture = true +} + +/** A task to start the fixture which emulates an Azure Storage service **/ +task azureStorageFixture(type: AntFixture) { + dependsOn compileTestJava + env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" + executable = new File(project.runtimeJavaHome, 'bin/java') + args 'org.elasticsearch.repositories.azure.AzureStorageFixture', baseDir, azureContainer +} + +Map expansions = [ + 'container': azureContainer, + 'base_path': azureBasePath +] +processTestResources { + inputs.properties(expansions) + MavenFilteringHack.filter(it, expansions) +} + +integTestCluster { + keystoreSetting 'azure.client.integration_test.account', azureAccount + keystoreSetting 'azure.client.integration_test.key', azureKey + + if (useFixture) { + dependsOn azureStorageFixture + // Use a closure on the string to delay evaluation until tests are executed. The endpoint_suffix is used + // in a hacky way to change the protocol and endpoint. We must fix that. + setting 'azure.client.integration_test.endpoint_suffix', + "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${ -> azureStorageFixture.addressAndPort }" + } else { + println "Using an external service to test the repository-azure plugin" + } +} diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageFixture.java b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageFixture.java similarity index 100% rename from plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageFixture.java rename to plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageFixture.java diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java new file mode 100644 index 00000000000..720f152265a --- /dev/null +++ b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.azure; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +public class AzureStorageRepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public AzureStorageRepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + + @Override + protected Settings restClientSettings() { + // Give more time to repository-azure to complete the snapshot operations + return Settings.builder().put(super.restClientSettings()) + .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "60s") + .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "60s") + .build(); + } +} diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageTestServer.java b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageTestServer.java similarity index 100% rename from plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageTestServer.java rename to plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageTestServer.java diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml new file mode 100644 index 00000000000..9ab1fafac3e --- /dev/null +++ b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml @@ -0,0 +1,174 @@ +# Integration tests for repository-azure +--- +"Snapshot/Restore with repository-azure": + + # Register repository + - do: + snapshot.create_repository: + repository: repository + body: + type: azure + settings: + container: ${container} + client: "integration_test" + base_path: ${base_path} + + - match: { acknowledged: true } + + # Get repository + - do: + snapshot.get_repository: + repository: repository + + - match: { repository.settings.container: ${container} } + - match: { repository.settings.client : "integration_test" } + - match: { repository.settings.base_path : ${base_path} } + + # Index documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 1 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 2 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 3 + - snapshot: one + + - do: + count: + index: docs + + - match: {count: 3} + + # Create a first snapshot + - do: + snapshot.create: + repository: repository + snapshot: snapshot-one + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-one } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.include_global_state: true } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.status: + repository: repository + snapshot: snapshot-one + + - is_true: snapshots + - match: { snapshots.0.snapshot: snapshot-one } + - match: { snapshots.0.state : SUCCESS } + + # Index more documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 4 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 5 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 6 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 7 + - snapshot: two + + - do: + count: + index: docs + + - match: {count: 7} + + # Create a second snapshot + - do: + snapshot.create: + repository: repository + snapshot: snapshot-two + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-two } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.get: + repository: repository + snapshot: snapshot-one,snapshot-two + + - is_true: snapshots + - match: { snapshots.0.state : SUCCESS } + - match: { snapshots.1.state : SUCCESS } + + # Delete the index + - do: + indices.delete: + index: docs + + # Restore the second snapshot + - do: + snapshot.restore: + repository: repository + snapshot: snapshot-two + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 7} + + # Delete the index again + - do: + indices.delete: + index: docs + + # Restore the first snapshot + - do: + snapshot.restore: + repository: repository + snapshot: snapshot-one + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 3} + + # Remove the snapshots + - do: + snapshot.delete: + repository: repository + snapshot: snapshot-two + master_timeout: 5m + + - do: + snapshot.delete: + repository: repository + snapshot: snapshot-one + master_timeout: 5m diff --git a/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/10_basic.yml b/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/10_basic.yml index 25726fa8f9b..3a754a34a8a 100644 --- a/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/10_basic.yml +++ b/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/10_basic.yml @@ -11,177 +11,3 @@ nodes.info: {} - match: { nodes.$master.plugins.0.name: repository-azure } ---- -"Snapshot/Restore with repository-azure": - - # Register repository - - do: - snapshot.create_repository: - repository: repository - body: - type: azure - settings: - container: "container_test" - client: "integration_test" - - - match: { acknowledged: true } - - # Get repository - - do: - snapshot.get_repository: - repository: repository - - - match: {repository.settings.container : "container_test"} - - match: {repository.settings.client : "integration_test"} - - # Index documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _type: doc - _id: 1 - - snapshot: one - - index: - _index: docs - _type: doc - _id: 2 - - snapshot: one - - index: - _index: docs - _type: doc - _id: 3 - - snapshot: one - - - do: - count: - index: docs - - - match: {count: 3} - - # Create a first snapshot - - do: - snapshot.create: - repository: repository - snapshot: snapshot-one - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-one } - - match: { snapshot.state : SUCCESS } - - match: { snapshot.include_global_state: true } - - match: { snapshot.shards.failed : 0 } - - - do: - snapshot.status: - repository: repository - snapshot: snapshot-one - - - is_true: snapshots - - match: { snapshots.0.snapshot: snapshot-one } - - match: { snapshots.0.state : SUCCESS } - - # Index more documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _type: doc - _id: 4 - - snapshot: two - - index: - _index: docs - _type: doc - _id: 5 - - snapshot: two - - index: - _index: docs - _type: doc - _id: 6 - - snapshot: two - - index: - _index: docs - _type: doc - _id: 7 - - snapshot: two - - - do: - count: - index: docs - - - match: {count: 7} - - # Create a second snapshot - - do: - snapshot.create: - repository: repository - snapshot: snapshot-two - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-two } - - match: { snapshot.state : SUCCESS } - - match: { snapshot.shards.failed : 0 } - - - do: - snapshot.get: - repository: repository - snapshot: snapshot-one,snapshot-two - - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } - - # Delete the index - - do: - indices.delete: - index: docs - - # Restore the second snapshot - - do: - snapshot.restore: - repository: repository - snapshot: snapshot-two - wait_for_completion: true - - - do: - count: - index: docs - - - match: {count: 7} - - # Delete the index again - - do: - indices.delete: - index: docs - - # Restore the first snapshot - - do: - snapshot.restore: - repository: repository - snapshot: snapshot-one - wait_for_completion: true - - - do: - count: - index: docs - - - match: {count: 3} - - # Remove the snapshots - - do: - snapshot.delete: - repository: repository - snapshot: snapshot-two - - - do: - snapshot.delete: - repository: repository - snapshot: snapshot-one - - # Remove our repository - - do: - snapshot.delete_repository: - repository: repository diff --git a/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml b/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml deleted file mode 100644 index 2bd98c0a363..00000000000 --- a/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml +++ /dev/null @@ -1,82 +0,0 @@ -"Deprecated Repository can be registered": - - skip: - features: warnings - - do: - warnings: - - "[account] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." - snapshot.create_repository: - repository: test_repo_azure - verify: false - body: - type: azure - settings: - account : "my_test_account" - container : "backup-container" - base_path : "backups" - chunk_size: "32m" - compress : true - - - is_true: acknowledged - - - do: - snapshot.get_repository: - repository: test_repo_azure - - - is_true : test_repo_azure - - match : { test_repo_azure.settings.account : "my_test_account" } - - match : { test_repo_azure.settings.container : "backup-container" } - - match : { test_repo_azure.settings.base_path : "backups" } - - match : { test_repo_azure.settings.chunk_size: "32m" } - - match : { test_repo_azure.settings.compress : "true" } ---- -"Default repository can be registered": - - do: - snapshot.create_repository: - repository: test_repo_azure - verify: false - body: - type: azure - settings: - container : "backup-container" - base_path : "backups" - chunk_size: "32m" - compress : true - - - is_true: acknowledged - - - do: - snapshot.get_repository: - repository: test_repo_azure - - - is_true : test_repo_azure - - match : { test_repo_azure.settings.container : "backup-container" } - - match : { test_repo_azure.settings.base_path : "backups" } - - match : { test_repo_azure.settings.chunk_size: "32m" } - - match : { test_repo_azure.settings.compress : "true" } ---- -"Named client repository can be registered": - - do: - snapshot.create_repository: - repository: test_repo_azure - verify: false - body: - type: azure - settings: - client : "secondary" - container : "backup-container" - base_path : "backups" - chunk_size: "32m" - compress : true - - - is_true: acknowledged - - - do: - snapshot.get_repository: - repository: test_repo_azure - - - is_true : test_repo_azure - - match : { test_repo_azure.settings.client : "secondary" } - - match : { test_repo_azure.settings.container : "backup-container" } - - match : { test_repo_azure.settings.base_path : "backups" } - - match : { test_repo_azure.settings.chunk_size: "32m" } - - match : { test_repo_azure.settings.compress : "true" }