426 lines
17 KiB
Groovy
426 lines
17 KiB
Groovy
import org.elasticsearch.gradle.MavenFilteringHack
|
|
import org.elasticsearch.gradle.info.BuildParams
|
|
import org.elasticsearch.gradle.test.RestIntegTestTask
|
|
|
|
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
|
|
|
|
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
apply plugin: 'elasticsearch.test.fixtures'
|
|
|
|
esplugin {
|
|
description 'The S3 repository plugin adds S3 repositories'
|
|
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
|
|
}
|
|
|
|
versions << [
|
|
'aws': '1.11.749'
|
|
]
|
|
|
|
dependencies {
|
|
compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
|
|
compile "com.amazonaws:aws-java-sdk-core:${versions.aws}"
|
|
compile "com.amazonaws:jmespath-java:${versions.aws}"
|
|
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
|
|
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
|
|
compile "commons-logging:commons-logging:${versions.commonslogging}"
|
|
compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
|
|
compile "commons-codec:commons-codec:${versions.commonscodec}"
|
|
compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
|
|
compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
|
|
compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
|
|
compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
|
|
compile "joda-time:joda-time:${versions.joda}"
|
|
|
|
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
|
|
// and whitelist this hack in JarHell
|
|
compile 'javax.xml.bind:jaxb-api:2.2.2'
|
|
|
|
testCompile project(':test:fixtures:s3-fixture')
|
|
}
|
|
|
|
dependencyLicenses {
|
|
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
|
|
mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
|
|
mapping from: /jackson-.*/, to: 'jackson'
|
|
mapping from: /jaxb-.*/, to: 'jaxb'
|
|
}
|
|
|
|
bundlePlugin {
|
|
from('config/repository-s3') {
|
|
into 'config'
|
|
}
|
|
}
|
|
|
|
task testRepositoryCreds(type: Test) {
|
|
include '**/RepositoryCredentialsTests.class'
|
|
systemProperty 'es.allow_insecure_settings', 'true'
|
|
}
|
|
check.dependsOn(testRepositoryCreds)
|
|
|
|
test {
|
|
// these are tested explicitly in separate test tasks
|
|
exclude '**/RepositoryCredentialsTests.class'
|
|
exclude '**/S3RepositoryThirdPartyTests.class'
|
|
}
|
|
|
|
boolean useFixture = false
|
|
|
|
// We test against two repositories, one which uses the usual two-part "permanent" credentials and
|
|
// the other which uses three-part "temporary" or "session" credentials.
|
|
|
|
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
|
|
String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
|
|
String s3PermanentBucket = System.getenv("amazon_s3_bucket")
|
|
String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
|
|
|
|
String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
|
|
String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
|
|
String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
|
|
String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
|
|
String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
|
|
|
|
String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
|
|
String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
|
|
|
|
String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
|
|
String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
|
|
|
|
boolean s3DisableChunkedEncoding = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()
|
|
|
|
// If all these variables are missing then we are testing against the internal fixture instead, which has the following
|
|
// credentials hard-coded in.
|
|
|
|
if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) {
|
|
s3PermanentAccessKey = 'access_key'
|
|
s3PermanentSecretKey = 'secret_key'
|
|
s3PermanentBucket = 'bucket'
|
|
s3PermanentBasePath = 'base_path'
|
|
|
|
useFixture = true
|
|
|
|
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
|
|
throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
|
|
}
|
|
|
|
if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
|
|
s3TemporaryAccessKey = 'session_token_access_key'
|
|
s3TemporarySecretKey = 'session_token_secret_key'
|
|
s3TemporaryBucket = 'session_token_bucket'
|
|
s3TemporaryBasePath = 'session_token_base_path'
|
|
s3TemporarySessionToken = 'session_token'
|
|
|
|
} else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
|
|
throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
|
|
}
|
|
|
|
if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
|
|
s3EC2Bucket = 'ec2_bucket'
|
|
s3EC2BasePath = 'ec2_base_path'
|
|
s3ECSBucket = 'ecs_bucket'
|
|
s3ECSBasePath = 'ecs_base_path'
|
|
} else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath) {
|
|
throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
|
|
}
|
|
|
|
task thirdPartyTest(type: Test) {
|
|
include '**/S3RepositoryThirdPartyTests.class'
|
|
systemProperty 'test.s3.account', s3PermanentAccessKey
|
|
systemProperty 'test.s3.key', s3PermanentSecretKey
|
|
systemProperty 'test.s3.bucket', s3PermanentBucket
|
|
nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed
|
|
}
|
|
|
|
if (useFixture) {
|
|
testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture')
|
|
|
|
def minioAddress = {
|
|
int minioPort = project(':test:fixtures:minio-fixture').postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
|
|
assert minioPort > 0
|
|
'http://127.0.0.1:' + minioPort
|
|
}
|
|
|
|
normalization {
|
|
runtimeClasspath {
|
|
// ignore generated address file for the purposes of build avoidance
|
|
ignore 's3Fixture.address'
|
|
}
|
|
}
|
|
|
|
thirdPartyTest {
|
|
dependsOn tasks.bundlePlugin
|
|
nonInputProperties.systemProperty 'test.s3.endpoint', "${-> minioAddress.call()}"
|
|
}
|
|
|
|
task integTestMinio(type: RestIntegTestTask) {
|
|
description = "Runs REST tests using the Minio repository."
|
|
dependsOn tasks.bundlePlugin
|
|
runner {
|
|
// Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
|
|
systemProperty 'tests.rest.blacklist', [
|
|
'repository_s3/30_repository_temporary_credentials/*',
|
|
'repository_s3/40_repository_ec2_credentials/*',
|
|
'repository_s3/50_repository_ecs_credentials/*'
|
|
].join(",")
|
|
}
|
|
}
|
|
check.dependsOn(integTestMinio)
|
|
|
|
testClusters.integTestMinio {
|
|
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
|
|
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
|
|
setting 's3.client.integration_test_permanent.endpoint', minioAddress, IGNORE_VALUE
|
|
plugin tasks.bundlePlugin.archiveFile
|
|
}
|
|
|
|
integTest.runner {
|
|
systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
|
|
}
|
|
} else {
|
|
integTest.runner {
|
|
systemProperty 'tests.rest.blacklist',
|
|
[
|
|
'repository_s3/30_repository_temporary_credentials/*',
|
|
'repository_s3/40_repository_ec2_credentials/*',
|
|
'repository_s3/50_repository_ecs_credentials/*'
|
|
].join(",")
|
|
}
|
|
}
|
|
|
|
check.dependsOn(thirdPartyTest)
|
|
|
|
processTestResources {
|
|
Map<String, Object> expansions = [
|
|
'permanent_bucket': s3PermanentBucket,
|
|
'permanent_base_path': s3PermanentBasePath + "_integration_tests",
|
|
'temporary_bucket': s3TemporaryBucket,
|
|
'temporary_base_path': s3TemporaryBasePath + "_integration_tests",
|
|
'ec2_bucket': s3EC2Bucket,
|
|
'ec2_base_path': s3EC2BasePath,
|
|
'ecs_bucket': s3ECSBucket,
|
|
'ecs_base_path': s3ECSBasePath,
|
|
'disable_chunked_encoding': s3DisableChunkedEncoding,
|
|
]
|
|
inputs.properties(expansions)
|
|
MavenFilteringHack.filter(it, expansions)
|
|
}
|
|
|
|
[
|
|
's3-fixture',
|
|
's3-fixture-with-session-token',
|
|
's3-fixture-with-ec2',
|
|
's3-fixture-with-ecs',
|
|
].forEach { fixture -> testFixtures.useFixture(':test:fixtures:s3-fixture', fixture) }
|
|
|
|
def fixtureAddress = { fixture ->
|
|
assert useFixture: 'closure should not be used without a fixture'
|
|
int ephemeralPort = project(':test:fixtures:s3-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
|
|
assert ephemeralPort > 0
|
|
'http://127.0.0.1:' + ephemeralPort
|
|
}
|
|
|
|
testClusters.integTest {
|
|
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
|
|
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
|
|
|
|
keystore 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
|
|
keystore 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
|
|
keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
|
|
|
|
if (useFixture) {
|
|
setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture')}" }, IGNORE_VALUE
|
|
setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture-with-session-token')}" }, IGNORE_VALUE
|
|
setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
|
|
|
|
// to redirect InstanceProfileCredentialsProvider to custom auth point
|
|
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
|
|
} else {
|
|
println "Using an external service to test the repository-s3 plugin"
|
|
}
|
|
}
|
|
|
|
task s3ThirdPartyTests {
|
|
dependsOn check
|
|
}
|
|
|
|
if (useFixture) {
|
|
task integTestECS(type: RestIntegTestTask.class) {
|
|
description = "Runs tests using the ECS repository."
|
|
dependsOn('bundlePlugin')
|
|
runner {
|
|
systemProperty 'tests.rest.blacklist', [
|
|
'repository_s3/10_basic/*',
|
|
'repository_s3/20_repository_permanent_credentials/*',
|
|
'repository_s3/30_repository_temporary_credentials/*',
|
|
'repository_s3/40_repository_ec2_credentials/*'
|
|
].join(",")
|
|
}
|
|
}
|
|
check.dependsOn(integTestECS)
|
|
|
|
testClusters.integTestECS {
|
|
setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture-with-ecs')}" }, IGNORE_VALUE
|
|
plugin tasks.bundlePlugin.archiveFile
|
|
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture-with-ecs')}/ecs_credentials_endpoint" }, IGNORE_VALUE
|
|
}
|
|
|
|
gradle.taskGraph.whenReady {
|
|
if (it.hasTask(s3ThirdPartyTests)) {
|
|
throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'amazon_s3_access_key', " +
|
|
"'amazon_s3_secret_key', 'amazon_s3_bucket', and 'amazon_s3_base_path' are set.");
|
|
}
|
|
}
|
|
}
|
|
|
|
thirdPartyAudit.ignoreMissingClasses(
|
|
// classes are missing
|
|
'javax.servlet.ServletContextEvent',
|
|
'javax.servlet.ServletContextListener',
|
|
'org.apache.avalon.framework.logger.Logger',
|
|
'org.apache.log.Hierarchy',
|
|
'org.apache.log.Logger',
|
|
'software.amazon.ion.IonReader',
|
|
'software.amazon.ion.IonSystem',
|
|
'software.amazon.ion.IonType',
|
|
'software.amazon.ion.IonWriter',
|
|
'software.amazon.ion.Timestamp',
|
|
'software.amazon.ion.system.IonBinaryWriterBuilder',
|
|
'software.amazon.ion.system.IonSystemBuilder',
|
|
'software.amazon.ion.system.IonTextWriterBuilder',
|
|
'software.amazon.ion.system.IonWriterBuilder',
|
|
// We don't use the kms dependency
|
|
'com.amazonaws.services.kms.AWSKMS',
|
|
'com.amazonaws.services.kms.AWSKMSClient',
|
|
'com.amazonaws.services.kms.model.DecryptRequest',
|
|
'com.amazonaws.services.kms.model.DecryptResult',
|
|
'com.amazonaws.services.kms.model.EncryptRequest',
|
|
'com.amazonaws.services.kms.model.EncryptResult',
|
|
'com.amazonaws.services.kms.model.GenerateDataKeyRequest',
|
|
'com.amazonaws.services.kms.model.GenerateDataKeyResult'
|
|
)
|
|
|
|
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
|
|
if (BuildParams.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
|
|
thirdPartyAudit.ignoreJarHellWithJDK(
|
|
'javax.xml.bind.Binder',
|
|
'javax.xml.bind.ContextFinder$1',
|
|
'javax.xml.bind.ContextFinder',
|
|
'javax.xml.bind.DataBindingException',
|
|
'javax.xml.bind.DatatypeConverter',
|
|
'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
|
|
'javax.xml.bind.DatatypeConverterImpl',
|
|
'javax.xml.bind.DatatypeConverterInterface',
|
|
'javax.xml.bind.Element',
|
|
'javax.xml.bind.GetPropertyAction',
|
|
'javax.xml.bind.JAXB$Cache',
|
|
'javax.xml.bind.JAXB',
|
|
'javax.xml.bind.JAXBContext',
|
|
'javax.xml.bind.JAXBElement$GlobalScope',
|
|
'javax.xml.bind.JAXBElement',
|
|
'javax.xml.bind.JAXBException',
|
|
'javax.xml.bind.JAXBIntrospector',
|
|
'javax.xml.bind.JAXBPermission',
|
|
'javax.xml.bind.MarshalException',
|
|
'javax.xml.bind.Marshaller$Listener',
|
|
'javax.xml.bind.Marshaller',
|
|
'javax.xml.bind.Messages',
|
|
'javax.xml.bind.NotIdentifiableEvent',
|
|
'javax.xml.bind.ParseConversionEvent',
|
|
'javax.xml.bind.PrintConversionEvent',
|
|
'javax.xml.bind.PropertyException',
|
|
'javax.xml.bind.SchemaOutputResolver',
|
|
'javax.xml.bind.TypeConstraintException',
|
|
'javax.xml.bind.UnmarshalException',
|
|
'javax.xml.bind.Unmarshaller$Listener',
|
|
'javax.xml.bind.Unmarshaller',
|
|
'javax.xml.bind.UnmarshallerHandler',
|
|
'javax.xml.bind.ValidationEvent',
|
|
'javax.xml.bind.ValidationEventHandler',
|
|
'javax.xml.bind.ValidationEventLocator',
|
|
'javax.xml.bind.ValidationException',
|
|
'javax.xml.bind.Validator',
|
|
'javax.xml.bind.WhiteSpaceProcessor',
|
|
'javax.xml.bind.annotation.DomHandler',
|
|
'javax.xml.bind.annotation.W3CDomHandler',
|
|
'javax.xml.bind.annotation.XmlAccessOrder',
|
|
'javax.xml.bind.annotation.XmlAccessType',
|
|
'javax.xml.bind.annotation.XmlAccessorOrder',
|
|
'javax.xml.bind.annotation.XmlAccessorType',
|
|
'javax.xml.bind.annotation.XmlAnyAttribute',
|
|
'javax.xml.bind.annotation.XmlAnyElement',
|
|
'javax.xml.bind.annotation.XmlAttachmentRef',
|
|
'javax.xml.bind.annotation.XmlAttribute',
|
|
'javax.xml.bind.annotation.XmlElement$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlElement',
|
|
'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
|
|
'javax.xml.bind.annotation.XmlElementDecl',
|
|
'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlElementRef',
|
|
'javax.xml.bind.annotation.XmlElementRefs',
|
|
'javax.xml.bind.annotation.XmlElementWrapper',
|
|
'javax.xml.bind.annotation.XmlElements',
|
|
'javax.xml.bind.annotation.XmlEnum',
|
|
'javax.xml.bind.annotation.XmlEnumValue',
|
|
'javax.xml.bind.annotation.XmlID',
|
|
'javax.xml.bind.annotation.XmlIDREF',
|
|
'javax.xml.bind.annotation.XmlInlineBinaryData',
|
|
'javax.xml.bind.annotation.XmlList',
|
|
'javax.xml.bind.annotation.XmlMimeType',
|
|
'javax.xml.bind.annotation.XmlMixed',
|
|
'javax.xml.bind.annotation.XmlNs',
|
|
'javax.xml.bind.annotation.XmlNsForm',
|
|
'javax.xml.bind.annotation.XmlRegistry',
|
|
'javax.xml.bind.annotation.XmlRootElement',
|
|
'javax.xml.bind.annotation.XmlSchema',
|
|
'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlSchemaType',
|
|
'javax.xml.bind.annotation.XmlSchemaTypes',
|
|
'javax.xml.bind.annotation.XmlSeeAlso',
|
|
'javax.xml.bind.annotation.XmlTransient',
|
|
'javax.xml.bind.annotation.XmlType$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlType',
|
|
'javax.xml.bind.annotation.XmlValue',
|
|
'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
|
|
'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
|
|
'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
|
|
'javax.xml.bind.attachment.AttachmentMarshaller',
|
|
'javax.xml.bind.attachment.AttachmentUnmarshaller',
|
|
'javax.xml.bind.helpers.AbstractMarshallerImpl',
|
|
'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
|
|
'javax.xml.bind.helpers.DefaultValidationEventHandler',
|
|
'javax.xml.bind.helpers.Messages',
|
|
'javax.xml.bind.helpers.NotIdentifiableEventImpl',
|
|
'javax.xml.bind.helpers.ParseConversionEventImpl',
|
|
'javax.xml.bind.helpers.PrintConversionEventImpl',
|
|
'javax.xml.bind.helpers.ValidationEventImpl',
|
|
'javax.xml.bind.helpers.ValidationEventLocatorImpl',
|
|
'javax.xml.bind.util.JAXBResult',
|
|
'javax.xml.bind.util.JAXBSource$1',
|
|
'javax.xml.bind.util.JAXBSource',
|
|
'javax.xml.bind.util.Messages',
|
|
'javax.xml.bind.util.ValidationEventCollector'
|
|
)
|
|
} else {
|
|
thirdPartyAudit.ignoreMissingClasses 'javax.activation.DataHandler'
|
|
}
|