Use 3rd party task to run integration tests on external service (#56588)

Backport of #56587 for 7.x
This commit is contained in:
Tanguy Leroux 2020-06-02 11:26:58 +02:00 committed by GitHub
parent 52c555e286
commit b4a2cd810a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 327 additions and 510 deletions

View File

@ -1,5 +1,9 @@
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.info.BuildParams
import static org.elasticsearch.gradle.PropertyNormalization.DEFAULT
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
/* /*
* Licensed to Elasticsearch under one or more contributor * Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with * license agreements. See the NOTICE file distributed with
@ -34,7 +38,7 @@ dependencies {
restResources { restResources {
restApi { restApi {
includeCore '_common', 'cluster', 'nodes' includeCore '_common', 'cluster', 'nodes', 'snapshot', 'bulk', 'count', 'indices'
} }
} }
@ -69,16 +73,13 @@ thirdPartyAudit {
) )
} }
check { boolean useFixture = false
// also execute the QA tests when testing the plugin
dependsOn 'qa:microsoft-azure-storage:check'
}
testClusters { def azureAddress = {
integTest { assert useFixture: 'closure should not be used without a fixture'
keystore 'azure.client.integration_test.account', 'azure_account' int ephemeralPort = project(':test:fixtures:azure-fixture').postProcessFixture.ext."test.fixtures.azure-fixture.tcp.8091"
keystore 'azure.client.integration_test.key', 'azure_key' assert ephemeralPort > 0
} return 'ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:' + ephemeralPort + '/'
} }
String azureAccount = System.getenv("azure_storage_account") String azureAccount = System.getenv("azure_storage_account")
@ -87,19 +88,60 @@ String azureContainer = System.getenv("azure_storage_container")
String azureBasePath = System.getenv("azure_storage_base_path") String azureBasePath = System.getenv("azure_storage_base_path")
String azureSasToken = System.getenv("azure_storage_sas_token") String azureSasToken = System.getenv("azure_storage_sas_token")
if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
azureAccount = 'azure_integration_test_account'
azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
azureContainer = 'container'
azureBasePath = ''
azureSasToken = ''
useFixture = true
apply plugin: 'elasticsearch.test.fixtures'
testFixtures.useFixture ':test:fixtures:azure-fixture', 'azure-fixture'
}
Map<String, Object> expansions = [
'container': azureContainer,
'base_path': azureBasePath + "_integration_tests"
]
processTestResources {
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
test { test {
// this is tested explicitly in a separate test task
exclude '**/AzureStorageCleanupThirdPartyTests.class' exclude '**/AzureStorageCleanupThirdPartyTests.class'
} }
task thirdPartyTest(type: Test) { testClusters {
integTest {
keystore 'azure.client.integration_test.account', azureAccount
if (azureKey != null && azureKey.isEmpty() == false) {
keystore 'azure.client.integration_test.key', azureKey
}
if (azureSasToken != null && azureSasToken.isEmpty() == false) {
keystore 'azure.client.integration_test.sas_token', azureSasToken
}
if (useFixture) {
setting 'azure.client.integration_test.endpoint_suffix', azureAddress
String firstPartOfSeed = BuildParams.testSeed.tokenize(':').get(0)
setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), System.getProperty('ignore.tests.seed') == null ? DEFAULT : IGNORE_VALUE
}
}
}
task azureThirdPartyTest(type: Test) {
dependsOn tasks.integTest
include '**/AzureStorageCleanupThirdPartyTests.class' include '**/AzureStorageCleanupThirdPartyTests.class'
systemProperty 'test.azure.account', azureAccount ? azureAccount : "" systemProperty 'test.azure.account', azureAccount ? azureAccount : ""
systemProperty 'test.azure.key', azureKey ? azureKey : "" systemProperty 'test.azure.key', azureKey ? azureKey : ""
systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : "" systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : ""
systemProperty 'test.azure.container', azureContainer ? azureContainer : "" systemProperty 'test.azure.container', azureContainer ? azureContainer : ""
systemProperty 'test.azure.base', (azureBasePath ? azureBasePath : "") + "_third_party_tests_" + BuildParams.testSeed systemProperty 'test.azure.base', (azureBasePath ? azureBasePath : "") + "_third_party_tests_" + BuildParams.testSeed
if (useFixture) {
nonInputProperties.systemProperty 'test.azure.endpoint_suffix', "${-> azureAddress.call() }"
}
} }
check.dependsOn(azureThirdPartyTest)
if (azureAccount || azureKey || azureContainer || azureBasePath || azureSasToken) {
check.dependsOn(thirdPartyTest)
}

View File

@ -1 +0,0 @@
group = "${group}.plugins.repository-azure.qa"

View File

@ -1,94 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.test.AntFixture
import static org.elasticsearch.gradle.PropertyNormalization.DEFAULT
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
apply plugin: 'elasticsearch.test.fixtures'
restResources {
restApi {
includeCore '_common', 'snapshot', 'bulk', 'count', 'indices'
}
}
testFixtures.useFixture ":test:fixtures:azure-fixture", "azure-fixture"
boolean useFixture = false
String azureAccount = System.getenv("azure_storage_account")
String azureKey = System.getenv("azure_storage_key")
String azureContainer = System.getenv("azure_storage_container")
String azureBasePath = System.getenv("azure_storage_base_path")
String azureSasToken = System.getenv("azure_storage_sas_token")
if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
azureAccount = 'azure_integration_test_account'
azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
azureContainer = 'container'
azureBasePath = ''
azureSasToken = ''
useFixture = true
}
Map<String, Object> expansions = [
'container': azureContainer,
'base_path': azureBasePath + "_integration_tests"
]
processTestResources {
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
integTest {
dependsOn project(':plugins:repository-azure').bundlePlugin
}
testClusters.integTest {
plugin project(':plugins:repository-azure').bundlePlugin.archiveFile
keystore 'azure.client.integration_test.account', azureAccount
if (azureKey != null && azureKey.isEmpty() == false) {
keystore 'azure.client.integration_test.key', azureKey
}
if (azureSasToken != null && azureSasToken.isEmpty() == false) {
keystore 'azure.client.integration_test.sas_token', azureSasToken
}
if (useFixture) {
def azureAddress = {
int ephemeralPort = project(':test:fixtures:azure-fixture').postProcessFixture.ext."test.fixtures.azure-fixture.tcp.8091"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}
// Use a closure on the string to delay evaluation until tests are executed. The endpoint_suffix is used
// in a hacky way to change the protocol and endpoint. We must fix that.
setting 'azure.client.integration_test.endpoint_suffix',
{ "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=${-> azureAddress()}" }, IGNORE_VALUE
String firstPartOfSeed = BuildParams.testSeed.tokenize(':').get(0)
setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), System.getProperty('ignore.tests.seed') == null ? DEFAULT : IGNORE_VALUE
}
}

View File

@ -1,47 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.azure;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
public class AzureStorageRepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
public AzureStorageRepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws Exception {
return ESClientYamlSuiteTestCase.createParameters();
}
@Override
protected Settings restClientSettings() {
// Give more time to repository-azure to complete the snapshot operations
return Settings.builder().put(super.restClientSettings())
.put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "60s")
.build();
}
}

View File

@ -50,6 +50,18 @@ public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyReposi
return pluginList(AzureRepositoryPlugin.class); return pluginList(AzureRepositoryPlugin.class);
} }
@Override
protected Settings nodeSettings() {
final String endpoint = System.getProperty("test.azure.endpoint_suffix");
if (Strings.hasText(endpoint)) {
return Settings.builder()
.put(super.nodeSettings())
.put("azure.client.default.endpoint_suffix", endpoint)
.build();
}
return super.nodeSettings();
}
@Override @Override
protected SecureSettings credentials() { protected SecureSettings credentials() {
assertThat(System.getProperty("test.azure.account"), not(blankOrNullString())); assertThat(System.getProperty("test.azure.account"), not(blankOrNullString()));

View File

@ -1,5 +1,15 @@
import java.nio.file.Files import java.nio.file.Files
import java.security.KeyPair
import java.security.KeyPairGenerator
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.test.RestIntegTestTask
import java.nio.file.Files
import java.security.KeyPair
import java.security.KeyPairGenerator
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
/* /*
* Licensed to Elasticsearch under one or more contributor * Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with * license agreements. See the NOTICE file distributed with
@ -58,7 +68,7 @@ dependencies {
restResources { restResources {
restApi { restApi {
includeCore '_common', 'cluster', 'nodes' includeCore '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'
} }
} }
@ -184,7 +194,132 @@ thirdPartyAudit {
) )
} }
check { boolean useFixture = false
// also execute the QA tests when testing the plugin
dependsOn 'qa:google-cloud-storage:check' def fixtureAddress = { fixture ->
assert useFixture: 'closure should not be used without a fixture'
int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
} }
String gcsServiceAccount = System.getenv("google_storage_service_account")
String gcsBucket = System.getenv("google_storage_bucket")
String gcsBasePath = System.getenv("google_storage_base_path")
File serviceAccountFile = null
if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
gcsBucket = 'bucket'
gcsBasePath = 'integration_test'
useFixture = true
apply plugin: 'elasticsearch.test.fixtures'
testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture')
testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-third-party')
} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present")
} else {
serviceAccountFile = new File(gcsServiceAccount)
}
def encodedCredentials = {
Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath()))
}
/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
task createServiceAccountFile() {
doLast {
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
keyPairGenerator.initialize(1024)
KeyPair keyPair = keyPairGenerator.generateKeyPair()
String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
serviceAccountFile.parentFile.mkdirs()
serviceAccountFile.setText("{\n" +
' "type": "service_account",\n' +
' "project_id": "integration_test",\n' +
' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
' "client_email": "integration_test@appspot.gserviceaccount.com",\n' +
' "client_id": "123456789101112130594"\n' +
'}', 'UTF-8')
}
}
Map<String, Object> expansions = [
'bucket' : gcsBucket,
'base_path': gcsBasePath + "_integration_tests"
]
processTestResources {
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
test {
// this is tested explicitly in a separate test task
exclude '**/GoogleCloudStorageThirdPartyTests.class'
}
final Closure testClustersConfiguration = {
keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE
if (useFixture) {
/* Use a closure on the string to delay evaluation until tests are executed */
setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE
setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-gcs plugin"
}
}
integTest {
if (useFixture) {
dependsOn createServiceAccountFile
}
}
check.dependsOn integTest
testClusters {
integTest testClustersConfiguration
}
/*
* We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add
* an additional test that forces the large blob threshold to be small to exercise the resumable upload path.
*/
task largeBlobIntegTest(type: RestIntegTestTask) {
mustRunAfter integTest
dependsOn project(':plugins:repository-gcs').bundlePlugin
if (useFixture) {
dependsOn createServiceAccountFile
}
}
check.dependsOn largeBlobIntegTest
testClusters.largeBlobIntegTest testClustersConfiguration
testClusters {
largeBlobIntegTest {
plugin project(':plugins:repository-gcs').bundlePlugin.archiveFile
// force large blob uploads by setting the threshold small, forcing this code path to be tested
systemProperty 'es.repository_gcs.large_blob_threshold_byte_size', '256'
}
}
task gcsThirdPartyTest(type: Test) {
dependsOn integTest,largeBlobIntegTest
include '**/GoogleCloudStorageThirdPartyTests.class'
systemProperty 'tests.security.manager', false
systemProperty 'test.google.bucket', gcsBucket
nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed
nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}"
if (useFixture) {
dependsOn createServiceAccountFile
nonInputProperties.systemProperty 'test.google.endpoint', "${-> fixtureAddress('gcs-fixture-third-party')}"
nonInputProperties.systemProperty 'test.google.tokenURI', "${-> fixtureAddress('gcs-fixture-third-party')}/o/oauth2/token"
}
}
check.dependsOn(gcsThirdPartyTest)

View File

@ -1 +0,0 @@
group = "${group}.plugins.repository-gcs.qa"

View File

@ -1,177 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.test.RestIntegTestTask
import java.nio.file.Files
import java.security.KeyPair
import java.security.KeyPairGenerator
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
apply plugin: 'elasticsearch.test.fixtures'
// TODO think about flattening qa:google-cloud-storage project into parent
dependencies {
testCompile project(path: ':plugins:repository-gcs')
}
restResources {
restApi {
includeCore '_common', 'snapshot','indices', 'index', 'bulk', 'count'
}
}
testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture')
testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-third-party')
boolean useFixture = false
String gcsServiceAccount = System.getenv("google_storage_service_account")
String gcsBucket = System.getenv("google_storage_bucket")
String gcsBasePath = System.getenv("google_storage_base_path")
File serviceAccountFile = null
if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
gcsBucket = 'bucket'
gcsBasePath = 'integration_test'
useFixture = true
} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present")
} else {
serviceAccountFile = new File(gcsServiceAccount)
}
def encodedCredentials = {
Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath()))
}
def fixtureAddress = { fixture ->
assert useFixture: 'closure should not be used without a fixture'
int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}
/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
task createServiceAccountFile() {
doLast {
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
keyPairGenerator.initialize(1024)
KeyPair keyPair = keyPairGenerator.generateKeyPair()
String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
serviceAccountFile.parentFile.mkdirs()
serviceAccountFile.setText("{\n" +
' "type": "service_account",\n' +
' "project_id": "integration_test",\n' +
' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
' "client_email": "integration_test@appspot.gserviceaccount.com",\n' +
' "client_id": "123456789101112130594"\n' +
'}', 'UTF-8')
}
}
task thirdPartyTest(type: Test) {
if (useFixture) {
thirdPartyTest.dependsOn createServiceAccountFile
nonInputProperties.systemProperty 'test.google.endpoint', "${-> fixtureAddress('gcs-fixture-third-party')}"
nonInputProperties.systemProperty 'test.google.tokenURI', "${-> fixtureAddress('gcs-fixture-third-party')}/o/oauth2/token"
gradle.taskGraph.whenReady {
if (it.hasTask(gcsThirdPartyTests)) {
throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables " +
"'google_storage_service_account', 'google_storage_bucket', 'google_storage_base_path' are set.")
}
}
}
include '**/GoogleCloudStorageThirdPartyTests.class'
systemProperty 'tests.security.manager', false
systemProperty 'test.google.bucket', gcsBucket
nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed
nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}"
}
task gcsThirdPartyTests {
dependsOn check
}
check.dependsOn thirdPartyTest
integTest.mustRunAfter(thirdPartyTest)
/*
* We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add
* an additional test that forces the large blob threshold to be small to exercise the resumable upload path.
*/
task largeBlobIntegTest(type: RestIntegTestTask) {
mustRunAfter(thirdPartyTest, integTest)
}
check.dependsOn integTest
check.dependsOn largeBlobIntegTest
Map<String, Object> expansions = [
'bucket': gcsBucket,
'base_path': gcsBasePath + "_integration_tests"
]
processTestResources {
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
final Closure integTestConfiguration = {
dependsOn project(':plugins:repository-gcs').bundlePlugin
}
integTest integTestConfiguration
largeBlobIntegTest integTestConfiguration
final Closure testClustersConfiguration = {
plugin project(':plugins:repository-gcs').bundlePlugin.archiveFile
keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE
if (useFixture) {
tasks.integTest.dependsOn createServiceAccountFile
tasks.largeBlobIntegTest.dependsOn createServiceAccountFile
/* Use a closure on the string to delay evaluation until tests are executed */
setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE
setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-gcs plugin"
}
}
testClusters.integTest testClustersConfiguration
testClusters.largeBlobIntegTest testClustersConfiguration
testClusters.largeBlobIntegTest {
// force large blob uploads by setting the threshold small, forcing this code path to be tested
systemProperty 'es.repository_gcs.large_blob_threshold_byte_size', '256'
}

View File

@ -1,37 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.gcs;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
public class GoogleCloudStorageRepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
public GoogleCloudStorageRepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws Exception {
return createParameters();
}
}

View File

@ -22,8 +22,6 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
* specific language governing permissions and limitations * specific language governing permissions and limitations
* under the License. * under the License.
*/ */
apply plugin: 'elasticsearch.test.fixtures'
esplugin { esplugin {
description 'The S3 repository plugin adds S3 repositories' description 'The S3 repository plugin adds S3 repositories'
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin' classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
@ -75,13 +73,20 @@ task testRepositoryCreds(type: Test) {
check.dependsOn(testRepositoryCreds) check.dependsOn(testRepositoryCreds)
test { test {
// these are tested explicitly in separate test tasks // this is tested explicitly in separate test tasks
exclude '**/RepositoryCredentialsTests.class' exclude '**/RepositoryCredentialsTests.class'
exclude '**/S3RepositoryThirdPartyTests.class' exclude '**/S3RepositoryThirdPartyTests.class'
} }
boolean useFixture = false boolean useFixture = false
def fixtureAddress = { fixture, name, port ->
assert useFixture: 'closure should not be used without a fixture'
int ephemeralPort = project(":test:fixtures:${fixture}").postProcessFixture.ext."test.fixtures.${name}.tcp.${port}"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}
// We test against two repositories, one which uses the usual two-part "permanent" credentials and // We test against two repositories, one which uses the usual two-part "permanent" credentials and
// the other which uses three-part "temporary" or "session" credentials. // the other which uses three-part "temporary" or "session" credentials.
@ -113,6 +118,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
s3PermanentBucket = 'bucket' s3PermanentBucket = 'bucket'
s3PermanentBasePath = 'base_path' s3PermanentBasePath = 'base_path'
apply plugin: 'elasticsearch.test.fixtures'
useFixture = true useFixture = true
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) { } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
@ -139,35 +145,78 @@ if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present") throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
} }
task thirdPartyTest(type: Test) { processTestResources {
include '**/S3RepositoryThirdPartyTests.class' Map<String, Object> expansions = [
systemProperty 'test.s3.account', s3PermanentAccessKey 'permanent_bucket': s3PermanentBucket,
systemProperty 'test.s3.key', s3PermanentSecretKey 'permanent_base_path': s3PermanentBasePath + "_integration_tests",
systemProperty 'test.s3.bucket', s3PermanentBucket 'temporary_bucket': s3TemporaryBucket,
nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed 'temporary_base_path': s3TemporaryBasePath + "_integration_tests",
'ec2_bucket': s3EC2Bucket,
'ec2_base_path': s3EC2BasePath,
'ecs_bucket': s3ECSBucket,
'ecs_base_path': s3ECSBasePath,
'disable_chunked_encoding': s3DisableChunkedEncoding,
]
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
} }
test {
// this is tested explicitly in a separate test task
exclude '**/S3RepositoryThirdPartyTests.class'
}
// IntegTest
integTest {
runner {
systemProperty 'tests.rest.blacklist', (
useFixture ?
['repository_s3/50_repository_ecs_credentials/*']
:
[
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
]
).join(",")
}
}
testClusters.integTest {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
keystore 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
keystore 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
if (useFixture) {
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture')
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-session-token')
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ec2')
normalization {
runtimeClasspath {
// ignore generated address file for the purposes of build avoidance
ignore 's3Fixture.address'
}
}
setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture', '80')}" }, IGNORE_VALUE
setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-session-token', '80')}" }, IGNORE_VALUE
setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE
// to redirect InstanceProfileCredentialsProvider to custom auth point
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-s3 plugin"
}
}
// MinIO
if (useFixture) { if (useFixture) {
testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture') testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture')
def minioAddress = {
int minioPort = project(':test:fixtures:minio-fixture').postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
assert minioPort > 0
'http://127.0.0.1:' + minioPort
}
normalization {
runtimeClasspath {
// ignore generated address file for the purposes of build avoidance
ignore 's3Fixture.address'
}
}
thirdPartyTest {
dependsOn tasks.bundlePlugin
nonInputProperties.systemProperty 'test.s3.endpoint', "${-> minioAddress.call()}"
}
task integTestMinio(type: RestIntegTestTask) { task integTestMinio(type: RestIntegTestTask) {
description = "Runs REST tests using the Minio repository." description = "Runs REST tests using the Minio repository."
dependsOn tasks.bundlePlugin dependsOn tasks.bundlePlugin
@ -185,81 +234,15 @@ if (useFixture) {
testClusters.integTestMinio { testClusters.integTestMinio {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
setting 's3.client.integration_test_permanent.endpoint', minioAddress, IGNORE_VALUE setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000')}" }, IGNORE_VALUE
plugin tasks.bundlePlugin.archiveFile plugin tasks.bundlePlugin.archiveFile
} }
integTest.runner {
systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
}
} else {
integTest.runner {
systemProperty 'tests.rest.blacklist',
[
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
].join(",")
}
}
check.dependsOn(thirdPartyTest)
processTestResources {
Map<String, Object> expansions = [
'permanent_bucket': s3PermanentBucket,
'permanent_base_path': s3PermanentBasePath + "_integration_tests",
'temporary_bucket': s3TemporaryBucket,
'temporary_base_path': s3TemporaryBasePath + "_integration_tests",
'ec2_bucket': s3EC2Bucket,
'ec2_base_path': s3EC2BasePath,
'ecs_bucket': s3ECSBucket,
'ecs_base_path': s3ECSBasePath,
'disable_chunked_encoding': s3DisableChunkedEncoding,
]
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
[
's3-fixture',
's3-fixture-with-session-token',
's3-fixture-with-ec2',
's3-fixture-with-ecs',
].forEach { fixture -> testFixtures.useFixture(':test:fixtures:s3-fixture', fixture) }
def fixtureAddress = { fixture ->
assert useFixture: 'closure should not be used without a fixture'
int ephemeralPort = project(':test:fixtures:s3-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}
testClusters.integTest {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
keystore 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
keystore 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
if (useFixture) {
setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture')}" }, IGNORE_VALUE
setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture-with-session-token')}" }, IGNORE_VALUE
setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
// to redirect InstanceProfileCredentialsProvider to custom auth point
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-s3 plugin"
}
}
task s3ThirdPartyTests {
dependsOn check
} }
// ECS
if (useFixture) { if (useFixture) {
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ecs')
task integTestECS(type: RestIntegTestTask.class) { task integTestECS(type: RestIntegTestTask.class) {
description = "Runs tests using the ECS repository." description = "Runs tests using the ECS repository."
dependsOn('bundlePlugin') dependsOn('bundlePlugin')
@ -275,19 +258,28 @@ if (useFixture) {
check.dependsOn(integTestECS) check.dependsOn(integTestECS)
testClusters.integTestECS { testClusters.integTestECS {
setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture-with-ecs')}" }, IGNORE_VALUE setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}" }, IGNORE_VALUE
plugin tasks.bundlePlugin.archiveFile plugin tasks.bundlePlugin.archiveFile
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture-with-ecs')}/ecs_credentials_endpoint" }, IGNORE_VALUE environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}/ecs_credentials_endpoint" }, IGNORE_VALUE
}
gradle.taskGraph.whenReady {
if (it.hasTask(s3ThirdPartyTests)) {
throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'amazon_s3_access_key', " +
"'amazon_s3_secret_key', 'amazon_s3_bucket', and 'amazon_s3_base_path' are set.");
}
} }
} }
// 3rd Party Tests
task s3ThirdPartyTest(type: Test) {
include '**/S3RepositoryThirdPartyTests.class'
systemProperty 'test.s3.account', s3PermanentAccessKey
systemProperty 'test.s3.key', s3PermanentSecretKey
systemProperty 'test.s3.bucket', s3PermanentBucket
nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed
if (useFixture) {
dependsOn tasks.integTestMinio
nonInputProperties.systemProperty 'test.s3.endpoint', "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000') }"
} else {
dependsOn tasks.integTest
}
}
check.dependsOn(s3ThirdPartyTest)
thirdPartyAudit.ignoreMissingClasses( thirdPartyAudit.ignoreMissingClasses(
// classes are missing // classes are missing
'javax.servlet.ServletContextEvent', 'javax.servlet.ServletContextEvent',

View File

@ -63,7 +63,7 @@ integTest {
dependsOn repositoryPlugin.bundlePlugin dependsOn repositoryPlugin.bundlePlugin
runner { runner {
systemProperty 'test.azure.container', azureContainer systemProperty 'test.azure.container', azureContainer
systemProperty 'test.azure.base_path', azureBasePath + "/searchable_snapshots_tests" systemProperty 'test.azure.base_path', azureBasePath + "_searchable_snapshots_tests"
} }
} }
@ -100,3 +100,7 @@ testClusters.integTest {
} }
} }
task azureThirdPartyTest {
dependsOn integTest
}

View File

@ -61,10 +61,6 @@ if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
serviceAccountFile = new File(gcsServiceAccount) serviceAccountFile = new File(gcsServiceAccount)
} }
def encodedCredentials = {
Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath()))
}
/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/ /** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
task createServiceAccountFile() { task createServiceAccountFile() {
doLast { doLast {
@ -111,7 +107,7 @@ integTest {
dependsOn repositoryPlugin.bundlePlugin dependsOn repositoryPlugin.bundlePlugin
runner { runner {
systemProperty 'test.gcs.bucket', gcsBucket systemProperty 'test.gcs.bucket', gcsBucket
systemProperty 'test.gcs.base_path', gcsBasePath + "/searchable_snapshots_tests" systemProperty 'test.gcs.base_path', gcsBasePath + "_searchable_snapshots_tests"
} }
} }
@ -136,3 +132,6 @@ testClusters.integTest {
setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.license.self_generated.type', 'trial'
} }
task gcsThirdPartyTest {
dependsOn integTest
}

View File

@ -29,7 +29,7 @@ if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) {
s3AccessKey = 'access_key' s3AccessKey = 'access_key'
s3SecretKey = 'secret_key' s3SecretKey = 'secret_key'
s3Bucket = 'bucket' s3Bucket = 'bucket'
s3BasePath = 'base_path' s3BasePath = null
useFixture = true useFixture = true
} else if (!s3AccessKey || !s3SecretKey || !s3Bucket || !s3BasePath) { } else if (!s3AccessKey || !s3SecretKey || !s3Bucket || !s3BasePath) {
@ -45,7 +45,7 @@ integTest {
dependsOn repositoryPlugin.bundlePlugin dependsOn repositoryPlugin.bundlePlugin
runner { runner {
systemProperty 'test.s3.bucket', s3Bucket systemProperty 'test.s3.bucket', s3Bucket
systemProperty 'test.s3.base_path', s3BasePath + "/searchable_snapshots_tests" systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_searchable_snapshots_tests" : 'base_path'
} }
} }
@ -68,7 +68,6 @@ testClusters.integTest {
assert ephemeralPort > 0 assert ephemeralPort > 0
'127.0.0.1:' + ephemeralPort '127.0.0.1:' + ephemeralPort
} }
setting 's3.client.searchable_snapshots.protocol', 'http' setting 's3.client.searchable_snapshots.protocol', 'http'
setting 's3.client.searchable_snapshots.endpoint', { "${-> fixtureAddress('s3-fixture-other')}" }, IGNORE_VALUE setting 's3.client.searchable_snapshots.endpoint', { "${-> fixtureAddress('s3-fixture-other')}" }, IGNORE_VALUE
@ -77,3 +76,6 @@ testClusters.integTest {
} }
} }
task s3ThirdPartyTest {
dependsOn integTest
}

View File

@ -27,7 +27,7 @@ String gcsBucket = System.getenv("google_storage_bucket")
String gcsBasePath = System.getenv("google_storage_base_path") String gcsBasePath = System.getenv("google_storage_base_path")
if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
gcsServiceAccount = new File(project(':plugins:repository-gcs:qa:google-cloud-storage').buildDir, gcsServiceAccount = new File(project(':plugins:repository-gcs').buildDir,
'generated-resources/service_account_test.json').path 'generated-resources/service_account_test.json').path
gcsBucket = 'bucket_test' gcsBucket = 'bucket_test'
gcsBasePath = 'integration_test' gcsBasePath = 'integration_test'
@ -41,7 +41,7 @@ def encodedCredentials = {
Base64.encoder.encodeToString(Files.readAllBytes(file(gcsServiceAccount).toPath())) Base64.encoder.encodeToString(Files.readAllBytes(file(gcsServiceAccount).toPath()))
} }
task thirdPartyTest(type: Test) { task gcsThirdPartyTest(type: Test) {
include '**/GCSCleanupTests.class' include '**/GCSCleanupTests.class'
systemProperty 'tests.security.manager', false systemProperty 'tests.security.manager', false
@ -52,7 +52,7 @@ task thirdPartyTest(type: Test) {
} }
if (useGCSFixture) { if (useGCSFixture) {
thirdPartyTest.enabled = false; gcsThirdPartyTest.enabled = false;
testingConventions.enabled = false; testingConventions.enabled = false;
/* /*
@ -85,4 +85,4 @@ if (useGCSFixture) {
*/ */
} }
check.dependsOn(thirdPartyTest) check.dependsOn(gcsThirdPartyTest)

View File

@ -14,10 +14,6 @@ dependencies {
test.enabled = false test.enabled = false
task s3ThirdPartyTests {
dependsOn check
}
boolean useS3Fixture = false boolean useS3Fixture = false
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key") String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
@ -36,7 +32,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present") throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
} }
task thirdPartyTest(type: Test) { task s3ThirdPartyTest(type: Test) {
include '**/*.class' include '**/*.class'
systemProperty 'tests.security.manager', false systemProperty 'tests.security.manager', false
@ -58,17 +54,9 @@ if (useS3Fixture) {
'http://127.0.0.1:' + minioPort 'http://127.0.0.1:' + minioPort
} }
thirdPartyTest { s3ThirdPartyTest {
dependsOn project(':test:fixtures:minio-fixture').postProcessFixture dependsOn project(':test:fixtures:minio-fixture').postProcessFixture
nonInputProperties.systemProperty 'test.s3.endpoint', "${-> minioAddress.call()}" nonInputProperties.systemProperty 'test.s3.endpoint', "${-> minioAddress.call()}"
} }
gradle.taskGraph.whenReady {
if (it.hasTask(s3ThirdPartyTests)) {
throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'amazon_s3_access_key', " +
"'amazon_s3_secret_key', 'amazon_s3_bucket', and 'amazon_s3_base_path' are set.");
}
}
} }
check.dependsOn(s3ThirdPartyTest)
check.dependsOn(thirdPartyTest)