mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-09 06:25:07 +00:00
a2fafa6af4
This change converts the module and plugin parameters for testClusters to be lazy. Meaning that the values are not resolved until they are actually used. This removes the requirement to use project.afterEvaluate to be able to resolve the bundle artifact. Note - this does not completely remove the need for afterEvaluate since it is still needed for the custom resource extension.
426 lines
17 KiB
Groovy
426 lines
17 KiB
Groovy
import org.elasticsearch.gradle.MavenFilteringHack
|
|
import org.elasticsearch.gradle.info.BuildParams
|
|
import org.elasticsearch.gradle.test.RestIntegTestTask
|
|
|
|
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
|
|
|
|
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
apply plugin: 'elasticsearch.test.fixtures'
|
|
|
|
esplugin {
|
|
description 'The S3 repository plugin adds S3 repositories'
|
|
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
|
|
}
|
|
|
|
versions << [
|
|
'aws': '1.11.749'
|
|
]
|
|
|
|
dependencies {
|
|
compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
|
|
compile "com.amazonaws:aws-java-sdk-core:${versions.aws}"
|
|
compile "com.amazonaws:jmespath-java:${versions.aws}"
|
|
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
|
|
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
|
|
compile "commons-logging:commons-logging:${versions.commonslogging}"
|
|
compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
|
|
compile "commons-codec:commons-codec:${versions.commonscodec}"
|
|
compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
|
|
compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
|
|
compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
|
|
compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
|
|
compile "joda-time:joda-time:${versions.joda}"
|
|
|
|
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
|
|
// and whitelist this hack in JarHell
|
|
compile 'javax.xml.bind:jaxb-api:2.2.2'
|
|
|
|
testCompile project(':test:fixtures:s3-fixture')
|
|
}
|
|
|
|
dependencyLicenses {
|
|
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
|
|
mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
|
|
mapping from: /jackson-.*/, to: 'jackson'
|
|
mapping from: /jaxb-.*/, to: 'jaxb'
|
|
}
|
|
|
|
bundlePlugin {
|
|
from('config/repository-s3') {
|
|
into 'config'
|
|
}
|
|
}
|
|
|
|
task testRepositoryCreds(type: Test) {
|
|
include '**/RepositoryCredentialsTests.class'
|
|
systemProperty 'es.allow_insecure_settings', 'true'
|
|
}
|
|
check.dependsOn(testRepositoryCreds)
|
|
|
|
test {
|
|
// these are tested explicitly in separate test tasks
|
|
exclude '**/RepositoryCredentialsTests.class'
|
|
exclude '**/S3RepositoryThirdPartyTests.class'
|
|
}
|
|
|
|
boolean useFixture = false
|
|
|
|
// We test against two repositories, one which uses the usual two-part "permanent" credentials and
|
|
// the other which uses three-part "temporary" or "session" credentials.
|
|
|
|
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
|
|
String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
|
|
String s3PermanentBucket = System.getenv("amazon_s3_bucket")
|
|
String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
|
|
|
|
String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
|
|
String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
|
|
String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
|
|
String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
|
|
String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
|
|
|
|
String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
|
|
String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
|
|
|
|
String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
|
|
String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
|
|
|
|
boolean s3DisableChunkedEncoding = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()
|
|
|
|
// If all these variables are missing then we are testing against the internal fixture instead, which has the following
|
|
// credentials hard-coded in.
|
|
|
|
if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) {
|
|
s3PermanentAccessKey = 'access_key'
|
|
s3PermanentSecretKey = 'secret_key'
|
|
s3PermanentBucket = 'bucket'
|
|
s3PermanentBasePath = 'base_path'
|
|
|
|
useFixture = true
|
|
|
|
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
|
|
throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
|
|
}
|
|
|
|
if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
|
|
s3TemporaryAccessKey = 'session_token_access_key'
|
|
s3TemporarySecretKey = 'session_token_secret_key'
|
|
s3TemporaryBucket = 'session_token_bucket'
|
|
s3TemporaryBasePath = 'session_token_base_path'
|
|
s3TemporarySessionToken = 'session_token'
|
|
|
|
} else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
|
|
throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
|
|
}
|
|
|
|
if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
|
|
s3EC2Bucket = 'ec2_bucket'
|
|
s3EC2BasePath = 'ec2_base_path'
|
|
s3ECSBucket = 'ecs_bucket'
|
|
s3ECSBasePath = 'ecs_base_path'
|
|
} else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath) {
|
|
throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
|
|
}
|
|
|
|
task thirdPartyTest(type: Test) {
|
|
include '**/S3RepositoryThirdPartyTests.class'
|
|
systemProperty 'test.s3.account', s3PermanentAccessKey
|
|
systemProperty 'test.s3.key', s3PermanentSecretKey
|
|
systemProperty 'test.s3.bucket', s3PermanentBucket
|
|
nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed
|
|
}
|
|
|
|
if (useFixture) {
|
|
testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture')
|
|
|
|
def minioAddress = {
|
|
int minioPort = project(':test:fixtures:minio-fixture').postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
|
|
assert minioPort > 0
|
|
'http://127.0.0.1:' + minioPort
|
|
}
|
|
|
|
normalization {
|
|
runtimeClasspath {
|
|
// ignore generated address file for the purposes of build avoidance
|
|
ignore 's3Fixture.address'
|
|
}
|
|
}
|
|
|
|
thirdPartyTest {
|
|
dependsOn tasks.bundlePlugin
|
|
nonInputProperties.systemProperty 'test.s3.endpoint', "${-> minioAddress.call()}"
|
|
}
|
|
|
|
task integTestMinio(type: RestIntegTestTask) {
|
|
description = "Runs REST tests using the Minio repository."
|
|
dependsOn tasks.bundlePlugin
|
|
runner {
|
|
// Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
|
|
systemProperty 'tests.rest.blacklist', [
|
|
'repository_s3/30_repository_temporary_credentials/*',
|
|
'repository_s3/40_repository_ec2_credentials/*',
|
|
'repository_s3/50_repository_ecs_credentials/*'
|
|
].join(",")
|
|
}
|
|
}
|
|
check.dependsOn(integTestMinio)
|
|
|
|
testClusters.integTestMinio {
|
|
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
|
|
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
|
|
setting 's3.client.integration_test_permanent.endpoint', minioAddress, IGNORE_VALUE
|
|
plugin tasks.bundlePlugin.archiveFile
|
|
}
|
|
|
|
integTest.runner {
|
|
systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
|
|
}
|
|
} else {
|
|
integTest.runner {
|
|
systemProperty 'tests.rest.blacklist',
|
|
[
|
|
'repository_s3/30_repository_temporary_credentials/*',
|
|
'repository_s3/40_repository_ec2_credentials/*',
|
|
'repository_s3/50_repository_ecs_credentials/*'
|
|
].join(",")
|
|
}
|
|
}
|
|
|
|
check.dependsOn(thirdPartyTest)
|
|
|
|
processTestResources {
|
|
Map<String, Object> expansions = [
|
|
'permanent_bucket': s3PermanentBucket,
|
|
'permanent_base_path': s3PermanentBasePath + "_integration_tests",
|
|
'temporary_bucket': s3TemporaryBucket,
|
|
'temporary_base_path': s3TemporaryBasePath + "_integration_tests",
|
|
'ec2_bucket': s3EC2Bucket,
|
|
'ec2_base_path': s3EC2BasePath,
|
|
'ecs_bucket': s3ECSBucket,
|
|
'ecs_base_path': s3ECSBasePath,
|
|
'disable_chunked_encoding': s3DisableChunkedEncoding,
|
|
]
|
|
inputs.properties(expansions)
|
|
MavenFilteringHack.filter(it, expansions)
|
|
}
|
|
|
|
[
|
|
's3-fixture',
|
|
's3-fixture-with-session-token',
|
|
's3-fixture-with-ec2',
|
|
's3-fixture-with-ecs',
|
|
].forEach { fixture -> testFixtures.useFixture(':test:fixtures:s3-fixture', fixture) }
|
|
|
|
def fixtureAddress = { fixture ->
|
|
assert useFixture: 'closure should not be used without a fixture'
|
|
int ephemeralPort = project(':test:fixtures:s3-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
|
|
assert ephemeralPort > 0
|
|
'http://127.0.0.1:' + ephemeralPort
|
|
}
|
|
|
|
testClusters.integTest {
|
|
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
|
|
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
|
|
|
|
keystore 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
|
|
keystore 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
|
|
keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
|
|
|
|
if (useFixture) {
|
|
setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture')}" }, IGNORE_VALUE
|
|
setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture-with-session-token')}" }, IGNORE_VALUE
|
|
setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
|
|
|
|
// to redirect InstanceProfileCredentialsProvider to custom auth point
|
|
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
|
|
} else {
|
|
println "Using an external service to test the repository-s3 plugin"
|
|
}
|
|
}
|
|
|
|
task s3ThirdPartyTests {
|
|
dependsOn check
|
|
}
|
|
|
|
if (useFixture) {
|
|
task integTestECS(type: RestIntegTestTask.class) {
|
|
description = "Runs tests using the ECS repository."
|
|
dependsOn('bundlePlugin')
|
|
runner {
|
|
systemProperty 'tests.rest.blacklist', [
|
|
'repository_s3/10_basic/*',
|
|
'repository_s3/20_repository_permanent_credentials/*',
|
|
'repository_s3/30_repository_temporary_credentials/*',
|
|
'repository_s3/40_repository_ec2_credentials/*'
|
|
].join(",")
|
|
}
|
|
}
|
|
check.dependsOn(integTestECS)
|
|
|
|
testClusters.integTestECS {
|
|
setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture-with-ecs')}" }, IGNORE_VALUE
|
|
plugin tasks.bundlePlugin.archiveFile
|
|
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture-with-ecs')}/ecs_credentials_endpoint" }, IGNORE_VALUE
|
|
}
|
|
|
|
gradle.taskGraph.whenReady {
|
|
if (it.hasTask(s3ThirdPartyTests)) {
|
|
throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'amazon_s3_access_key', " +
|
|
"'amazon_s3_secret_key', 'amazon_s3_bucket', and 'amazon_s3_base_path' are set.");
|
|
}
|
|
}
|
|
}
|
|
|
|
thirdPartyAudit.ignoreMissingClasses(
|
|
// classes are missing
|
|
'javax.servlet.ServletContextEvent',
|
|
'javax.servlet.ServletContextListener',
|
|
'org.apache.avalon.framework.logger.Logger',
|
|
'org.apache.log.Hierarchy',
|
|
'org.apache.log.Logger',
|
|
'software.amazon.ion.IonReader',
|
|
'software.amazon.ion.IonSystem',
|
|
'software.amazon.ion.IonType',
|
|
'software.amazon.ion.IonWriter',
|
|
'software.amazon.ion.Timestamp',
|
|
'software.amazon.ion.system.IonBinaryWriterBuilder',
|
|
'software.amazon.ion.system.IonSystemBuilder',
|
|
'software.amazon.ion.system.IonTextWriterBuilder',
|
|
'software.amazon.ion.system.IonWriterBuilder',
|
|
// We don't use the kms dependency
|
|
'com.amazonaws.services.kms.AWSKMS',
|
|
'com.amazonaws.services.kms.AWSKMSClient',
|
|
'com.amazonaws.services.kms.model.DecryptRequest',
|
|
'com.amazonaws.services.kms.model.DecryptResult',
|
|
'com.amazonaws.services.kms.model.EncryptRequest',
|
|
'com.amazonaws.services.kms.model.EncryptResult',
|
|
'com.amazonaws.services.kms.model.GenerateDataKeyRequest',
|
|
'com.amazonaws.services.kms.model.GenerateDataKeyResult'
|
|
)
|
|
|
|
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
|
|
if (BuildParams.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
|
|
thirdPartyAudit.ignoreJarHellWithJDK(
|
|
'javax.xml.bind.Binder',
|
|
'javax.xml.bind.ContextFinder$1',
|
|
'javax.xml.bind.ContextFinder',
|
|
'javax.xml.bind.DataBindingException',
|
|
'javax.xml.bind.DatatypeConverter',
|
|
'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
|
|
'javax.xml.bind.DatatypeConverterImpl',
|
|
'javax.xml.bind.DatatypeConverterInterface',
|
|
'javax.xml.bind.Element',
|
|
'javax.xml.bind.GetPropertyAction',
|
|
'javax.xml.bind.JAXB$Cache',
|
|
'javax.xml.bind.JAXB',
|
|
'javax.xml.bind.JAXBContext',
|
|
'javax.xml.bind.JAXBElement$GlobalScope',
|
|
'javax.xml.bind.JAXBElement',
|
|
'javax.xml.bind.JAXBException',
|
|
'javax.xml.bind.JAXBIntrospector',
|
|
'javax.xml.bind.JAXBPermission',
|
|
'javax.xml.bind.MarshalException',
|
|
'javax.xml.bind.Marshaller$Listener',
|
|
'javax.xml.bind.Marshaller',
|
|
'javax.xml.bind.Messages',
|
|
'javax.xml.bind.NotIdentifiableEvent',
|
|
'javax.xml.bind.ParseConversionEvent',
|
|
'javax.xml.bind.PrintConversionEvent',
|
|
'javax.xml.bind.PropertyException',
|
|
'javax.xml.bind.SchemaOutputResolver',
|
|
'javax.xml.bind.TypeConstraintException',
|
|
'javax.xml.bind.UnmarshalException',
|
|
'javax.xml.bind.Unmarshaller$Listener',
|
|
'javax.xml.bind.Unmarshaller',
|
|
'javax.xml.bind.UnmarshallerHandler',
|
|
'javax.xml.bind.ValidationEvent',
|
|
'javax.xml.bind.ValidationEventHandler',
|
|
'javax.xml.bind.ValidationEventLocator',
|
|
'javax.xml.bind.ValidationException',
|
|
'javax.xml.bind.Validator',
|
|
'javax.xml.bind.WhiteSpaceProcessor',
|
|
'javax.xml.bind.annotation.DomHandler',
|
|
'javax.xml.bind.annotation.W3CDomHandler',
|
|
'javax.xml.bind.annotation.XmlAccessOrder',
|
|
'javax.xml.bind.annotation.XmlAccessType',
|
|
'javax.xml.bind.annotation.XmlAccessorOrder',
|
|
'javax.xml.bind.annotation.XmlAccessorType',
|
|
'javax.xml.bind.annotation.XmlAnyAttribute',
|
|
'javax.xml.bind.annotation.XmlAnyElement',
|
|
'javax.xml.bind.annotation.XmlAttachmentRef',
|
|
'javax.xml.bind.annotation.XmlAttribute',
|
|
'javax.xml.bind.annotation.XmlElement$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlElement',
|
|
'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
|
|
'javax.xml.bind.annotation.XmlElementDecl',
|
|
'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlElementRef',
|
|
'javax.xml.bind.annotation.XmlElementRefs',
|
|
'javax.xml.bind.annotation.XmlElementWrapper',
|
|
'javax.xml.bind.annotation.XmlElements',
|
|
'javax.xml.bind.annotation.XmlEnum',
|
|
'javax.xml.bind.annotation.XmlEnumValue',
|
|
'javax.xml.bind.annotation.XmlID',
|
|
'javax.xml.bind.annotation.XmlIDREF',
|
|
'javax.xml.bind.annotation.XmlInlineBinaryData',
|
|
'javax.xml.bind.annotation.XmlList',
|
|
'javax.xml.bind.annotation.XmlMimeType',
|
|
'javax.xml.bind.annotation.XmlMixed',
|
|
'javax.xml.bind.annotation.XmlNs',
|
|
'javax.xml.bind.annotation.XmlNsForm',
|
|
'javax.xml.bind.annotation.XmlRegistry',
|
|
'javax.xml.bind.annotation.XmlRootElement',
|
|
'javax.xml.bind.annotation.XmlSchema',
|
|
'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlSchemaType',
|
|
'javax.xml.bind.annotation.XmlSchemaTypes',
|
|
'javax.xml.bind.annotation.XmlSeeAlso',
|
|
'javax.xml.bind.annotation.XmlTransient',
|
|
'javax.xml.bind.annotation.XmlType$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlType',
|
|
'javax.xml.bind.annotation.XmlValue',
|
|
'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
|
|
'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
|
|
'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
|
|
'javax.xml.bind.attachment.AttachmentMarshaller',
|
|
'javax.xml.bind.attachment.AttachmentUnmarshaller',
|
|
'javax.xml.bind.helpers.AbstractMarshallerImpl',
|
|
'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
|
|
'javax.xml.bind.helpers.DefaultValidationEventHandler',
|
|
'javax.xml.bind.helpers.Messages',
|
|
'javax.xml.bind.helpers.NotIdentifiableEventImpl',
|
|
'javax.xml.bind.helpers.ParseConversionEventImpl',
|
|
'javax.xml.bind.helpers.PrintConversionEventImpl',
|
|
'javax.xml.bind.helpers.ValidationEventImpl',
|
|
'javax.xml.bind.helpers.ValidationEventLocatorImpl',
|
|
'javax.xml.bind.util.JAXBResult',
|
|
'javax.xml.bind.util.JAXBSource$1',
|
|
'javax.xml.bind.util.JAXBSource',
|
|
'javax.xml.bind.util.Messages',
|
|
'javax.xml.bind.util.ValidationEventCollector'
|
|
)
|
|
} else {
|
|
thirdPartyAudit.ignoreMissingClasses 'javax.activation.DataHandler'
|
|
}
|