honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use 3rd party task to run integration tests on external service (#56588) 

Backport of #56587 for 7.x
This commit is contained in:
Tanguy Leroux 2020-06-02 11:26:58 +02:00 committed by GitHub
parent 52c555e286
commit b4a2cd810a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 327 additions and 510 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
plugins/repository-azure/build.gradle
View File

@ -1,5 +1,9 @@
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.info.BuildParams
import static org.elasticsearch.gradle.PropertyNormalization.DEFAULT
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
@ -34,7 +38,7 @@ dependencies {
restResources {
restApi {
includeCore '_common', 'cluster', 'nodes'
includeCore '_common', 'cluster', 'nodes', 'snapshot', 'bulk', 'count', 'indices'
}
}
@ -69,16 +73,13 @@ thirdPartyAudit {
)
}
check {
// also execute the QA tests when testing the plugin
dependsOn 'qa:microsoft-azure-storage:check'
}
boolean useFixture = false
testClusters {
integTest {
keystore 'azure.client.integration_test.account', 'azure_account'
keystore 'azure.client.integration_test.key', 'azure_key'
}
def azureAddress = {
assert useFixture: 'closure should not be used without a fixture'
int ephemeralPort = project(':test:fixtures:azure-fixture').postProcessFixture.ext."test.fixtures.azure-fixture.tcp.8091"
assert ephemeralPort > 0
return 'ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:' + ephemeralPort + '/'
}
String azureAccount = System.getenv("azure_storage_account")
@ -87,19 +88,60 @@ String azureContainer = System.getenv("azure_storage_container")
String azureBasePath = System.getenv("azure_storage_base_path")
String azureSasToken = System.getenv("azure_storage_sas_token")
if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
azureAccount = 'azure_integration_test_account'
azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
azureContainer = 'container'
azureBasePath = ''
azureSasToken = ''
useFixture = true
apply plugin: 'elasticsearch.test.fixtures'
testFixtures.useFixture ':test:fixtures:azure-fixture', 'azure-fixture'
}
Map<String, Object> expansions = [
'container': azureContainer,
'base_path': azureBasePath + "_integration_tests"
]
processTestResources {
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
test {
// this is tested explicitly in a separate test task
exclude '**/AzureStorageCleanupThirdPartyTests.class'
}
task thirdPartyTest(type: Test) {
testClusters {
integTest {
keystore 'azure.client.integration_test.account', azureAccount
if (azureKey != null && azureKey.isEmpty() == false) {
keystore 'azure.client.integration_test.key', azureKey
}
if (azureSasToken != null && azureSasToken.isEmpty() == false) {
keystore 'azure.client.integration_test.sas_token', azureSasToken
}
if (useFixture) {
setting 'azure.client.integration_test.endpoint_suffix', azureAddress
String firstPartOfSeed = BuildParams.testSeed.tokenize(':').get(0)
setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), System.getProperty('ignore.tests.seed') == null ? DEFAULT : IGNORE_VALUE
}
}
}
task azureThirdPartyTest(type: Test) {
dependsOn tasks.integTest
include '**/AzureStorageCleanupThirdPartyTests.class'
systemProperty 'test.azure.account', azureAccount ? azureAccount : ""
systemProperty 'test.azure.key', azureKey ? azureKey : ""
systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : ""
systemProperty 'test.azure.container', azureContainer ? azureContainer : ""
systemProperty 'test.azure.base', (azureBasePath ? azureBasePath : "") + "_third_party_tests_" + BuildParams.testSeed
if (useFixture) {
nonInputProperties.systemProperty 'test.azure.endpoint_suffix', "${-> azureAddress.call() }"
}
}
if (azureAccount || azureKey || azureContainer || azureBasePath || azureSasToken) {
check.dependsOn(thirdPartyTest)
}
check.dependsOn(azureThirdPartyTest)

1
plugins/repository-azure/qa/build.gradle
View File

@ -1 +0,0 @@
group = "${group}.plugins.repository-azure.qa"

94
plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
View File

@ -1,94 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.test.AntFixture
import static org.elasticsearch.gradle.PropertyNormalization.DEFAULT
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
apply plugin: 'elasticsearch.test.fixtures'
restResources {
restApi {
includeCore '_common', 'snapshot', 'bulk', 'count', 'indices'
}
}
testFixtures.useFixture ":test:fixtures:azure-fixture", "azure-fixture"
boolean useFixture = false
String azureAccount = System.getenv("azure_storage_account")
String azureKey = System.getenv("azure_storage_key")
String azureContainer = System.getenv("azure_storage_container")
String azureBasePath = System.getenv("azure_storage_base_path")
String azureSasToken = System.getenv("azure_storage_sas_token")
if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
azureAccount = 'azure_integration_test_account'
azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
azureContainer = 'container'
azureBasePath = ''
azureSasToken = ''
useFixture = true
}
Map<String, Object> expansions = [
'container': azureContainer,
'base_path': azureBasePath + "_integration_tests"
]
processTestResources {
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
integTest {
dependsOn project(':plugins:repository-azure').bundlePlugin
}
testClusters.integTest {
plugin project(':plugins:repository-azure').bundlePlugin.archiveFile
keystore 'azure.client.integration_test.account', azureAccount
if (azureKey != null && azureKey.isEmpty() == false) {
keystore 'azure.client.integration_test.key', azureKey
}
if (azureSasToken != null && azureSasToken.isEmpty() == false) {
keystore 'azure.client.integration_test.sas_token', azureSasToken
}
if (useFixture) {
def azureAddress = {
int ephemeralPort = project(':test:fixtures:azure-fixture').postProcessFixture.ext."test.fixtures.azure-fixture.tcp.8091"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}
// Use a closure on the string to delay evaluation until tests are executed. The endpoint_suffix is used
// in a hacky way to change the protocol and endpoint. We must fix that.
setting 'azure.client.integration_test.endpoint_suffix',
{ "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=${-> azureAddress()}" }, IGNORE_VALUE
String firstPartOfSeed = BuildParams.testSeed.tokenize(':').get(0)
setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), System.getProperty('ignore.tests.seed') == null ? DEFAULT : IGNORE_VALUE
}
}

47
plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java
View File

@ -1,47 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.azure;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
public class AzureStorageRepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
public AzureStorageRepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws Exception {
return ESClientYamlSuiteTestCase.createParameters();
}
@Override
protected Settings restClientSettings() {
// Give more time to repository-azure to complete the snapshot operations
return Settings.builder().put(super.restClientSettings())
.put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "60s")
.build();
}
}

12
plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
View File

@ -50,6 +50,18 @@ public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyReposi
return pluginList(AzureRepositoryPlugin.class);
}
@Override
protected Settings nodeSettings() {
final String endpoint = System.getProperty("test.azure.endpoint_suffix");
if (Strings.hasText(endpoint)) {
return Settings.builder()
.put(super.nodeSettings())
.put("azure.client.default.endpoint_suffix", endpoint)
.build();
}
return super.nodeSettings();
}
@Override
protected SecureSettings credentials() {
assertThat(System.getProperty("test.azure.account"), not(blankOrNullString()));

0
plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml → plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml
View File

143
plugins/repository-gcs/build.gradle
View File

@ -1,5 +1,15 @@
import java.nio.file.Files
import java.security.KeyPair
import java.security.KeyPairGenerator
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.test.RestIntegTestTask
import java.nio.file.Files
import java.security.KeyPair
import java.security.KeyPairGenerator
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
@ -58,7 +68,7 @@ dependencies {
restResources {
restApi {
includeCore '_common', 'cluster', 'nodes'
includeCore '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'
}
}
@ -184,7 +194,132 @@ thirdPartyAudit {
)
}
check {
// also execute the QA tests when testing the plugin
dependsOn 'qa:google-cloud-storage:check'
boolean useFixture = false
def fixtureAddress = { fixture ->
assert useFixture: 'closure should not be used without a fixture'
int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}
String gcsServiceAccount = System.getenv("google_storage_service_account")
String gcsBucket = System.getenv("google_storage_bucket")
String gcsBasePath = System.getenv("google_storage_base_path")
File serviceAccountFile = null
if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
gcsBucket = 'bucket'
gcsBasePath = 'integration_test'
useFixture = true
apply plugin: 'elasticsearch.test.fixtures'
testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture')
testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-third-party')
} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present")
} else {
serviceAccountFile = new File(gcsServiceAccount)
}
def encodedCredentials = {
Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath()))
}
/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
task createServiceAccountFile() {
doLast {
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
keyPairGenerator.initialize(1024)
KeyPair keyPair = keyPairGenerator.generateKeyPair()
String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
serviceAccountFile.parentFile.mkdirs()
serviceAccountFile.setText("{\n" +
' "type": "service_account",\n' +
' "project_id": "integration_test",\n' +
' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
' "client_email": "integration_test@appspot.gserviceaccount.com",\n' +
' "client_id": "123456789101112130594"\n' +
'}', 'UTF-8')
}
}
Map<String, Object> expansions = [
'bucket' : gcsBucket,
'base_path': gcsBasePath + "_integration_tests"
]
processTestResources {
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
test {
// this is tested explicitly in a separate test task
exclude '**/GoogleCloudStorageThirdPartyTests.class'
}
final Closure testClustersConfiguration = {
keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE
if (useFixture) {
/* Use a closure on the string to delay evaluation until tests are executed */
setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE
setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-gcs plugin"
}
}
integTest {
if (useFixture) {
dependsOn createServiceAccountFile
}
}
check.dependsOn integTest
testClusters {
integTest testClustersConfiguration
}
/*
* We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add
* an additional test that forces the large blob threshold to be small to exercise the resumable upload path.
*/
task largeBlobIntegTest(type: RestIntegTestTask) {
mustRunAfter integTest
dependsOn project(':plugins:repository-gcs').bundlePlugin
if (useFixture) {
dependsOn createServiceAccountFile
}
}
check.dependsOn largeBlobIntegTest
testClusters.largeBlobIntegTest testClustersConfiguration
testClusters {
largeBlobIntegTest {
plugin project(':plugins:repository-gcs').bundlePlugin.archiveFile
// force large blob uploads by setting the threshold small, forcing this code path to be tested
systemProperty 'es.repository_gcs.large_blob_threshold_byte_size', '256'
}
}
task gcsThirdPartyTest(type: Test) {
dependsOn integTest,largeBlobIntegTest
include '**/GoogleCloudStorageThirdPartyTests.class'
systemProperty 'tests.security.manager', false
systemProperty 'test.google.bucket', gcsBucket
nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed
nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}"
if (useFixture) {
dependsOn createServiceAccountFile
nonInputProperties.systemProperty 'test.google.endpoint', "${-> fixtureAddress('gcs-fixture-third-party')}"
nonInputProperties.systemProperty 'test.google.tokenURI', "${-> fixtureAddress('gcs-fixture-third-party')}/o/oauth2/token"
}
}
check.dependsOn(gcsThirdPartyTest)

1
plugins/repository-gcs/qa/build.gradle
View File

@ -1 +0,0 @@
group = "${group}.plugins.repository-gcs.qa"

177
plugins/repository-gcs/qa/google-cloud-storage/build.gradle
View File

@ -1,177 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.test.RestIntegTestTask
import java.nio.file.Files
import java.security.KeyPair
import java.security.KeyPairGenerator
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
apply plugin: 'elasticsearch.test.fixtures'
// TODO think about flattening qa:google-cloud-storage project into parent
dependencies {
testCompile project(path: ':plugins:repository-gcs')
}
restResources {
restApi {
includeCore '_common', 'snapshot','indices', 'index', 'bulk', 'count'
}
}
testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture')
testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-third-party')
boolean useFixture = false
String gcsServiceAccount = System.getenv("google_storage_service_account")
String gcsBucket = System.getenv("google_storage_bucket")
String gcsBasePath = System.getenv("google_storage_base_path")
File serviceAccountFile = null
if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
gcsBucket = 'bucket'
gcsBasePath = 'integration_test'
useFixture = true
} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present")
} else {
serviceAccountFile = new File(gcsServiceAccount)
}
def encodedCredentials = {
Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath()))
}
def fixtureAddress = { fixture ->
assert useFixture: 'closure should not be used without a fixture'
int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}
/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
task createServiceAccountFile() {
doLast {
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
keyPairGenerator.initialize(1024)
KeyPair keyPair = keyPairGenerator.generateKeyPair()
String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
serviceAccountFile.parentFile.mkdirs()
serviceAccountFile.setText("{\n" +
' "type": "service_account",\n' +
' "project_id": "integration_test",\n' +
' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
' "client_email": "integration_test@appspot.gserviceaccount.com",\n' +
' "client_id": "123456789101112130594"\n' +
'}', 'UTF-8')
}
}
task thirdPartyTest(type: Test) {
if (useFixture) {
thirdPartyTest.dependsOn createServiceAccountFile
nonInputProperties.systemProperty 'test.google.endpoint', "${-> fixtureAddress('gcs-fixture-third-party')}"
nonInputProperties.systemProperty 'test.google.tokenURI', "${-> fixtureAddress('gcs-fixture-third-party')}/o/oauth2/token"
gradle.taskGraph.whenReady {
if (it.hasTask(gcsThirdPartyTests)) {
throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables " +
"'google_storage_service_account', 'google_storage_bucket', 'google_storage_base_path' are set.")
}
}
}
include '**/GoogleCloudStorageThirdPartyTests.class'
systemProperty 'tests.security.manager', false
systemProperty 'test.google.bucket', gcsBucket
nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed
nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}"
}
task gcsThirdPartyTests {
dependsOn check
}
check.dependsOn thirdPartyTest
integTest.mustRunAfter(thirdPartyTest)
/*
* We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add
* an additional test that forces the large blob threshold to be small to exercise the resumable upload path.
*/
task largeBlobIntegTest(type: RestIntegTestTask) {
mustRunAfter(thirdPartyTest, integTest)
}
check.dependsOn integTest
check.dependsOn largeBlobIntegTest
Map<String, Object> expansions = [
'bucket': gcsBucket,
'base_path': gcsBasePath + "_integration_tests"
]
processTestResources {
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
final Closure integTestConfiguration = {
dependsOn project(':plugins:repository-gcs').bundlePlugin
}
integTest integTestConfiguration
largeBlobIntegTest integTestConfiguration
final Closure testClustersConfiguration = {
plugin project(':plugins:repository-gcs').bundlePlugin.archiveFile
keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE
if (useFixture) {
tasks.integTest.dependsOn createServiceAccountFile
tasks.largeBlobIntegTest.dependsOn createServiceAccountFile
/* Use a closure on the string to delay evaluation until tests are executed */
setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE
setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-gcs plugin"
}
}
testClusters.integTest testClustersConfiguration
testClusters.largeBlobIntegTest testClustersConfiguration
testClusters.largeBlobIntegTest {
// force large blob uploads by setting the threshold small, forcing this code path to be tested
systemProperty 'es.repository_gcs.large_blob_threshold_byte_size', '256'
}

37
plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryClientYamlTestSuiteIT.java
View File

@ -1,37 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.gcs;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
public class GoogleCloudStorageRepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
public GoogleCloudStorageRepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws Exception {
return createParameters();
}
}

0
plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java → plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java
View File

0
plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml → plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/20_repository.yml
View File

204
plugins/repository-s3/build.gradle
View File

@ -22,8 +22,6 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
* specific language governing permissions and limitations
* under the License.
*/
apply plugin: 'elasticsearch.test.fixtures'
esplugin {
description 'The S3 repository plugin adds S3 repositories'
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
@ -75,13 +73,20 @@ task testRepositoryCreds(type: Test) {
check.dependsOn(testRepositoryCreds)
test {
// these are tested explicitly in separate test tasks
// this is tested explicitly in separate test tasks
exclude '**/RepositoryCredentialsTests.class'
exclude '**/S3RepositoryThirdPartyTests.class'
}
boolean useFixture = false
def fixtureAddress = { fixture, name, port ->
assert useFixture: 'closure should not be used without a fixture'
int ephemeralPort = project(":test:fixtures:${fixture}").postProcessFixture.ext."test.fixtures.${name}.tcp.${port}"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}
// We test against two repositories, one which uses the usual two-part "permanent" credentials and
// the other which uses three-part "temporary" or "session" credentials.
@ -113,6 +118,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
s3PermanentBucket = 'bucket'
s3PermanentBasePath = 'base_path'
apply plugin: 'elasticsearch.test.fixtures'
useFixture = true
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
@ -139,35 +145,78 @@ if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
}
task thirdPartyTest(type: Test) {
include '**/S3RepositoryThirdPartyTests.class'
systemProperty 'test.s3.account', s3PermanentAccessKey
systemProperty 'test.s3.key', s3PermanentSecretKey
systemProperty 'test.s3.bucket', s3PermanentBucket
nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed
processTestResources {
Map<String, Object> expansions = [
'permanent_bucket': s3PermanentBucket,
'permanent_base_path': s3PermanentBasePath + "_integration_tests",
'temporary_bucket': s3TemporaryBucket,
'temporary_base_path': s3TemporaryBasePath + "_integration_tests",
'ec2_bucket': s3EC2Bucket,
'ec2_base_path': s3EC2BasePath,
'ecs_bucket': s3ECSBucket,
'ecs_base_path': s3ECSBasePath,
'disable_chunked_encoding': s3DisableChunkedEncoding,
]
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
test {
// this is tested explicitly in a separate test task
exclude '**/S3RepositoryThirdPartyTests.class'
}
// IntegTest
integTest {
runner {
systemProperty 'tests.rest.blacklist', (
useFixture ?
['repository_s3/50_repository_ecs_credentials/*']
:
[
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
]
).join(",")
}
}
testClusters.integTest {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
keystore 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
keystore 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
if (useFixture) {
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture')
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-session-token')
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ec2')
normalization {
runtimeClasspath {
// ignore generated address file for the purposes of build avoidance
ignore 's3Fixture.address'
}
}
setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture', '80')}" }, IGNORE_VALUE
setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-session-token', '80')}" }, IGNORE_VALUE
setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE
// to redirect InstanceProfileCredentialsProvider to custom auth point
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-s3 plugin"
}
}
// MinIO
if (useFixture) {
testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture')
def minioAddress = {
int minioPort = project(':test:fixtures:minio-fixture').postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
assert minioPort > 0
'http://127.0.0.1:' + minioPort
}
normalization {
runtimeClasspath {
// ignore generated address file for the purposes of build avoidance
ignore 's3Fixture.address'
}
}
thirdPartyTest {
dependsOn tasks.bundlePlugin
nonInputProperties.systemProperty 'test.s3.endpoint', "${-> minioAddress.call()}"
}
task integTestMinio(type: RestIntegTestTask) {
description = "Runs REST tests using the Minio repository."
dependsOn tasks.bundlePlugin
@ -185,81 +234,15 @@ if (useFixture) {
testClusters.integTestMinio {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
setting 's3.client.integration_test_permanent.endpoint', minioAddress, IGNORE_VALUE
setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000')}" }, IGNORE_VALUE
plugin tasks.bundlePlugin.archiveFile
}
integTest.runner {
systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
}
} else {
integTest.runner {
systemProperty 'tests.rest.blacklist',
[
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
].join(",")
}
}
check.dependsOn(thirdPartyTest)
processTestResources {
Map<String, Object> expansions = [
'permanent_bucket': s3PermanentBucket,
'permanent_base_path': s3PermanentBasePath + "_integration_tests",
'temporary_bucket': s3TemporaryBucket,
'temporary_base_path': s3TemporaryBasePath + "_integration_tests",
'ec2_bucket': s3EC2Bucket,
'ec2_base_path': s3EC2BasePath,
'ecs_bucket': s3ECSBucket,
'ecs_base_path': s3ECSBasePath,
'disable_chunked_encoding': s3DisableChunkedEncoding,
]
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
[
's3-fixture',
's3-fixture-with-session-token',
's3-fixture-with-ec2',
's3-fixture-with-ecs',
].forEach { fixture -> testFixtures.useFixture(':test:fixtures:s3-fixture', fixture) }
def fixtureAddress = { fixture ->
assert useFixture: 'closure should not be used without a fixture'
int ephemeralPort = project(':test:fixtures:s3-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}
testClusters.integTest {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
keystore 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
keystore 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
if (useFixture) {
setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture')}" }, IGNORE_VALUE
setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture-with-session-token')}" }, IGNORE_VALUE
setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
// to redirect InstanceProfileCredentialsProvider to custom auth point
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-s3 plugin"
}
}
task s3ThirdPartyTests {
dependsOn check
}
// ECS
if (useFixture) {
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ecs')
task integTestECS(type: RestIntegTestTask.class) {
description = "Runs tests using the ECS repository."
dependsOn('bundlePlugin')
@ -275,19 +258,28 @@ if (useFixture) {
check.dependsOn(integTestECS)
testClusters.integTestECS {
setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture-with-ecs')}" }, IGNORE_VALUE
setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}" }, IGNORE_VALUE
plugin tasks.bundlePlugin.archiveFile
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture-with-ecs')}/ecs_credentials_endpoint" }, IGNORE_VALUE
}
gradle.taskGraph.whenReady {
if (it.hasTask(s3ThirdPartyTests)) {
throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'amazon_s3_access_key', " +
"'amazon_s3_secret_key', 'amazon_s3_bucket', and 'amazon_s3_base_path' are set.");
}
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}/ecs_credentials_endpoint" }, IGNORE_VALUE
}
}
// 3rd Party Tests
task s3ThirdPartyTest(type: Test) {
include '**/S3RepositoryThirdPartyTests.class'
systemProperty 'test.s3.account', s3PermanentAccessKey
systemProperty 'test.s3.key', s3PermanentSecretKey
systemProperty 'test.s3.bucket', s3PermanentBucket
nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed
if (useFixture) {
dependsOn tasks.integTestMinio
nonInputProperties.systemProperty 'test.s3.endpoint', "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000') }"
} else {
dependsOn tasks.integTest
}
}
check.dependsOn(s3ThirdPartyTest)
thirdPartyAudit.ignoreMissingClasses(
// classes are missing
'javax.servlet.ServletContextEvent',

6
x-pack/plugin/searchable-snapshots/qa/azure/build.gradle
View File

@ -63,7 +63,7 @@ integTest {
dependsOn repositoryPlugin.bundlePlugin
runner {
systemProperty 'test.azure.container', azureContainer
systemProperty 'test.azure.base_path', azureBasePath + "/searchable_snapshots_tests"
systemProperty 'test.azure.base_path', azureBasePath + "_searchable_snapshots_tests"
}
}
@ -100,3 +100,7 @@ testClusters.integTest {
}
}
task azureThirdPartyTest {
dependsOn integTest
}

9
x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle
View File

@ -61,10 +61,6 @@ if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
serviceAccountFile = new File(gcsServiceAccount)
}
def encodedCredentials = {
Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath()))
}
/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
task createServiceAccountFile() {
doLast {
@ -111,7 +107,7 @@ integTest {
dependsOn repositoryPlugin.bundlePlugin
runner {
systemProperty 'test.gcs.bucket', gcsBucket
systemProperty 'test.gcs.base_path', gcsBasePath + "/searchable_snapshots_tests"
systemProperty 'test.gcs.base_path', gcsBasePath + "_searchable_snapshots_tests"
}
}
@ -136,3 +132,6 @@ testClusters.integTest {
setting 'xpack.license.self_generated.type', 'trial'
}
task gcsThirdPartyTest {
dependsOn integTest
}

8
x-pack/plugin/searchable-snapshots/qa/s3/build.gradle
View File

@ -29,7 +29,7 @@ if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) {
s3AccessKey = 'access_key'
s3SecretKey = 'secret_key'
s3Bucket = 'bucket'
s3BasePath = 'base_path'
s3BasePath = null
useFixture = true
} else if (!s3AccessKey || !s3SecretKey || !s3Bucket || !s3BasePath) {
@ -45,7 +45,7 @@ integTest {
dependsOn repositoryPlugin.bundlePlugin
runner {
systemProperty 'test.s3.bucket', s3Bucket
systemProperty 'test.s3.base_path', s3BasePath + "/searchable_snapshots_tests"
systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_searchable_snapshots_tests" : 'base_path'
}
}
@ -68,7 +68,6 @@ testClusters.integTest {
assert ephemeralPort > 0
'127.0.0.1:' + ephemeralPort
}
setting 's3.client.searchable_snapshots.protocol', 'http'
setting 's3.client.searchable_snapshots.endpoint', { "${-> fixtureAddress('s3-fixture-other')}" }, IGNORE_VALUE
@ -77,3 +76,6 @@ testClusters.integTest {
}
}
task s3ThirdPartyTest {
dependsOn integTest
}

8
x-pack/snapshot-tool/qa/google-cloud-storage/build.gradle
View File

@ -27,7 +27,7 @@ String gcsBucket = System.getenv("google_storage_bucket")
String gcsBasePath = System.getenv("google_storage_base_path")
if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
gcsServiceAccount = new File(project(':plugins:repository-gcs:qa:google-cloud-storage').buildDir,
gcsServiceAccount = new File(project(':plugins:repository-gcs').buildDir,
'generated-resources/service_account_test.json').path
gcsBucket = 'bucket_test'
gcsBasePath = 'integration_test'
@ -41,7 +41,7 @@ def encodedCredentials = {
Base64.encoder.encodeToString(Files.readAllBytes(file(gcsServiceAccount).toPath()))
}
task thirdPartyTest(type: Test) {
task gcsThirdPartyTest(type: Test) {
include '**/GCSCleanupTests.class'
systemProperty 'tests.security.manager', false
@ -52,7 +52,7 @@ task thirdPartyTest(type: Test) {
}
if (useGCSFixture) {
thirdPartyTest.enabled = false;
gcsThirdPartyTest.enabled = false;
testingConventions.enabled = false;
/*
@ -85,4 +85,4 @@ if (useGCSFixture) {
*/
}
check.dependsOn(thirdPartyTest)
check.dependsOn(gcsThirdPartyTest)

18
x-pack/snapshot-tool/qa/s3/build.gradle
View File

@ -14,10 +14,6 @@ dependencies {
test.enabled = false
task s3ThirdPartyTests {
dependsOn check
}
boolean useS3Fixture = false
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
@ -36,7 +32,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
}
task thirdPartyTest(type: Test) {
task s3ThirdPartyTest(type: Test) {
include '**/*.class'
systemProperty 'tests.security.manager', false
@ -58,17 +54,9 @@ if (useS3Fixture) {
'http://127.0.0.1:' + minioPort
}
thirdPartyTest {
s3ThirdPartyTest {
dependsOn project(':test:fixtures:minio-fixture').postProcessFixture
nonInputProperties.systemProperty 'test.s3.endpoint', "${-> minioAddress.call()}"
}
gradle.taskGraph.whenReady {
if (it.hasTask(s3ThirdPartyTests)) {
throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'amazon_s3_access_key', " +
"'amazon_s3_secret_key', 'amazon_s3_bucket', and 'amazon_s3_base_path' are set.");
}
}
}
check.dependsOn(thirdPartyTest)
check.dependsOn(s3ThirdPartyTest)