mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-05 20:48:22 +00:00
35d96c22c0
* Fix 3rd pary S3 tests This is allready excluded on line 186, by doing this again here, the other exclusion from arround that line are removed causing the tests to fail. * Fix blacklisting with the fixture
414 lines
17 KiB
Groovy
414 lines
17 KiB
Groovy
import org.elasticsearch.gradle.BuildPlugin
|
|
import org.elasticsearch.gradle.MavenFilteringHack
|
|
import org.elasticsearch.gradle.test.AntFixture
|
|
import org.elasticsearch.gradle.test.RestIntegTestTask
|
|
import com.carrotsearch.gradle.junit4.RandomizedTestingTask
|
|
|
|
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
esplugin {
|
|
description 'The S3 repository plugin adds S3 repositories'
|
|
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
|
|
}
|
|
|
|
versions << [
|
|
'aws': '1.11.505'
|
|
]
|
|
|
|
dependencies {
|
|
compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
|
|
compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}"
|
|
compile "com.amazonaws:aws-java-sdk-core:${versions.aws}"
|
|
compile "com.amazonaws:jmespath-java:${versions.aws}"
|
|
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
|
|
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
|
|
compile "commons-logging:commons-logging:${versions.commonslogging}"
|
|
compile "commons-codec:commons-codec:${versions.commonscodec}"
|
|
compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
|
|
compile 'com.fasterxml.jackson.core:jackson-databind:2.8.11.3'
|
|
compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
|
|
compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
|
|
compile "joda-time:joda-time:${versions.joda}"
|
|
|
|
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
|
|
// and whitelist this hack in JarHell
|
|
compile 'javax.xml.bind:jaxb-api:2.2.2'
|
|
}
|
|
|
|
dependencyLicenses {
|
|
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
|
|
mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
|
|
mapping from: /jackson-.*/, to: 'jackson'
|
|
mapping from: /jaxb-.*/, to: 'jaxb'
|
|
}
|
|
|
|
bundlePlugin {
|
|
from('config/repository-s3') {
|
|
into 'config'
|
|
}
|
|
}
|
|
|
|
task testRepositoryCreds(type: RandomizedTestingTask) {
|
|
include '**/RepositoryCredentialsTests.class'
|
|
include '**/S3BlobStoreRepositoryTests.class'
|
|
systemProperty 'es.allow_insecure_settings', 'true'
|
|
}
|
|
check.dependsOn(testRepositoryCreds)
|
|
|
|
unitTest {
|
|
// these are tested explicitly in separate test tasks
|
|
exclude '**/*CredentialsTests.class'
|
|
exclude '**/S3BlobStoreRepositoryTests.class'
|
|
}
|
|
|
|
boolean useFixture = false
|
|
|
|
// We test against two repositories, one which uses the usual two-part "permanent" credentials and
|
|
// the other which uses three-part "temporary" or "session" credentials.
|
|
|
|
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
|
|
String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
|
|
String s3PermanentBucket = System.getenv("amazon_s3_bucket")
|
|
String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
|
|
|
|
String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
|
|
String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
|
|
String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
|
|
String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
|
|
String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
|
|
|
|
String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
|
|
String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
|
|
|
|
String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
|
|
String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
|
|
|
|
// If all these variables are missing then we are testing against the internal fixture instead, which has the following
|
|
// credentials hard-coded in.
|
|
|
|
if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) {
|
|
s3PermanentAccessKey = 's3_integration_test_permanent_access_key'
|
|
s3PermanentSecretKey = 's3_integration_test_permanent_secret_key'
|
|
s3PermanentBucket = 'permanent-bucket-test'
|
|
s3PermanentBasePath = 'integration_test'
|
|
|
|
useFixture = true
|
|
|
|
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
|
|
throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
|
|
}
|
|
|
|
if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
|
|
s3TemporaryAccessKey = 's3_integration_test_temporary_access_key'
|
|
s3TemporarySecretKey = 's3_integration_test_temporary_secret_key'
|
|
s3TemporaryBucket = 'temporary-bucket-test'
|
|
s3TemporaryBasePath = 'integration_test'
|
|
s3TemporarySessionToken = 's3_integration_test_temporary_session_token'
|
|
|
|
} else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
|
|
throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
|
|
}
|
|
|
|
if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
|
|
s3EC2Bucket = 'ec2-bucket-test'
|
|
s3EC2BasePath = 'integration_test'
|
|
s3ECSBucket = 'ecs-bucket-test'
|
|
s3ECSBasePath = 'integration_test'
|
|
} else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath) {
|
|
throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
|
|
}
|
|
|
|
if (useFixture) {
|
|
apply plugin: 'elasticsearch.test.fixtures'
|
|
task writeDockerFile {
|
|
File minioDockerfile = new File("${project.buildDir}/minio-docker/Dockerfile")
|
|
outputs.file(minioDockerfile)
|
|
doLast {
|
|
minioDockerfile.parentFile.mkdirs()
|
|
minioDockerfile.text = "FROM minio/minio:RELEASE.2019-01-23T23-18-58Z\n" +
|
|
"RUN mkdir -p /minio/data/${s3PermanentBucket}\n" +
|
|
"ENV MINIO_ACCESS_KEY ${s3PermanentAccessKey}\n" +
|
|
"ENV MINIO_SECRET_KEY ${s3PermanentSecretKey}"
|
|
}
|
|
}
|
|
preProcessFixture {
|
|
dependsOn(writeDockerFile)
|
|
}
|
|
|
|
task integTestMinio(type: RestIntegTestTask) {
|
|
description = "Runs REST tests using the Minio repository."
|
|
dependsOn tasks.bundlePlugin, tasks.postProcessFixture
|
|
runner {
|
|
// Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
|
|
systemProperty 'tests.rest.blacklist', [
|
|
'repository_s3/30_repository_temporary_credentials/*',
|
|
'repository_s3/40_repository_ec2_credentials/*',
|
|
'repository_s3/50_repository_ecs_credentials/*'
|
|
].join(",")
|
|
}
|
|
}
|
|
check.dependsOn(integTestMinio)
|
|
BuildPlugin.requireDocker(tasks.integTestMinio)
|
|
|
|
testClusters.integTestMinio {
|
|
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
|
|
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
|
|
setting 's3.client.integration_test_permanent.endpoint', {
|
|
int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
|
|
assert minioPort > 0
|
|
return 'http://127.0.0.1:' + minioPort
|
|
}
|
|
plugin file(tasks.bundlePlugin.archiveFile)
|
|
}
|
|
|
|
integTest.runner {
|
|
systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
|
|
}
|
|
} else {
|
|
integTest.runner {
|
|
systemProperty 'tests.rest.blacklist',
|
|
[
|
|
'repository_s3/30_repository_temporary_credentials/*',
|
|
'repository_s3/40_repository_ec2_credentials/*',
|
|
'repository_s3/50_repository_ecs_credentials/*'
|
|
].join(",")
|
|
}
|
|
}
|
|
|
|
File parentFixtures = new File(project.buildDir, "fixtures")
|
|
File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties')
|
|
|
|
task s3FixtureProperties {
|
|
outputs.file(s3FixtureFile)
|
|
def s3FixtureOptions = [
|
|
"tests.seed" : project.testSeed,
|
|
"s3Fixture.permanent_bucket_name" : s3PermanentBucket,
|
|
"s3Fixture.permanent_key" : s3PermanentAccessKey,
|
|
"s3Fixture.temporary_bucket_name" : s3TemporaryBucket,
|
|
"s3Fixture.temporary_key" : s3TemporaryAccessKey,
|
|
"s3Fixture.temporary_session_token": s3TemporarySessionToken,
|
|
"s3Fixture.ec2_bucket_name" : s3EC2Bucket,
|
|
"s3Fixture.ecs_bucket_name" : s3ECSBucket
|
|
]
|
|
|
|
doLast {
|
|
file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n")
|
|
}
|
|
}
|
|
|
|
/** A task to start the AmazonS3Fixture which emulates an S3 service **/
|
|
task s3Fixture(type: AntFixture) {
|
|
dependsOn testClasses
|
|
dependsOn s3FixtureProperties
|
|
inputs.file(s3FixtureFile)
|
|
|
|
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
|
|
executable = new File(project.runtimeJavaHome, 'bin/java')
|
|
args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3FixtureFile.getAbsolutePath()
|
|
}
|
|
|
|
processTestResources {
|
|
Map<String, Object> expansions = [
|
|
'permanent_bucket': s3PermanentBucket,
|
|
'permanent_base_path': s3PermanentBasePath,
|
|
'temporary_bucket': s3TemporaryBucket,
|
|
'temporary_base_path': s3TemporaryBasePath,
|
|
'ec2_bucket': s3EC2Bucket,
|
|
'ec2_base_path': s3EC2BasePath,
|
|
'ecs_bucket': s3ECSBucket,
|
|
'ecs_base_path': s3ECSBasePath
|
|
]
|
|
inputs.properties(expansions)
|
|
MavenFilteringHack.filter(it, expansions)
|
|
}
|
|
|
|
integTest {
|
|
dependsOn s3Fixture
|
|
}
|
|
|
|
testClusters.integTest {
|
|
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
|
|
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
|
|
|
|
keystore 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
|
|
keystore 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
|
|
keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
|
|
|
|
if (useFixture) {
|
|
setting 's3.client.integration_test_permanent.endpoint', { "http://${s3Fixture.addressAndPort}" }
|
|
setting 's3.client.integration_test_temporary.endpoint', { "http://${s3Fixture.addressAndPort}" }
|
|
setting 's3.client.integration_test_ec2.endpoint', { "http://${s3Fixture.addressAndPort}" }
|
|
|
|
// to redirect InstanceProfileCredentialsProvider to custom auth point
|
|
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${s3Fixture.addressAndPort}" }
|
|
} else {
|
|
println "Using an external service to test the repository-s3 plugin"
|
|
}
|
|
}
|
|
|
|
if (useFixture) {
|
|
task integTestECS(type: RestIntegTestTask.class) {
|
|
description = "Runs tests using the ECS repository."
|
|
dependsOn(project.s3Fixture)
|
|
runner {
|
|
systemProperty 'tests.rest.blacklist', [
|
|
'repository_s3/10_basic/*',
|
|
'repository_s3/20_repository_permanent_credentials/*',
|
|
'repository_s3/30_repository_temporary_credentials/*',
|
|
'repository_s3/40_repository_ec2_credentials/*'
|
|
].join(",")
|
|
}
|
|
}
|
|
check.dependsOn(integTestECS)
|
|
|
|
testClusters.integTestECS {
|
|
setting 's3.client.integration_test_ecs.endpoint', { "http://${s3Fixture.addressAndPort}" }
|
|
plugin file(tasks.bundlePlugin.archiveFile)
|
|
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "http://${s3Fixture.addressAndPort}/ecs_credentials_endpoint" }
|
|
}
|
|
}
|
|
|
|
thirdPartyAudit.ignoreMissingClasses (
|
|
// classes are missing
|
|
'javax.servlet.ServletContextEvent',
|
|
'javax.servlet.ServletContextListener',
|
|
'org.apache.avalon.framework.logger.Logger',
|
|
'org.apache.log.Hierarchy',
|
|
'org.apache.log.Logger',
|
|
'software.amazon.ion.IonReader',
|
|
'software.amazon.ion.IonSystem',
|
|
'software.amazon.ion.IonType',
|
|
'software.amazon.ion.IonWriter',
|
|
'software.amazon.ion.Timestamp',
|
|
'software.amazon.ion.system.IonBinaryWriterBuilder',
|
|
'software.amazon.ion.system.IonSystemBuilder',
|
|
'software.amazon.ion.system.IonTextWriterBuilder',
|
|
'software.amazon.ion.system.IonWriterBuilder'
|
|
)
|
|
|
|
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
|
|
if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
|
|
thirdPartyAudit.ignoreJarHellWithJDK (
|
|
'javax.xml.bind.Binder',
|
|
'javax.xml.bind.ContextFinder$1',
|
|
'javax.xml.bind.ContextFinder',
|
|
'javax.xml.bind.DataBindingException',
|
|
'javax.xml.bind.DatatypeConverter',
|
|
'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
|
|
'javax.xml.bind.DatatypeConverterImpl',
|
|
'javax.xml.bind.DatatypeConverterInterface',
|
|
'javax.xml.bind.Element',
|
|
'javax.xml.bind.GetPropertyAction',
|
|
'javax.xml.bind.JAXB$Cache',
|
|
'javax.xml.bind.JAXB',
|
|
'javax.xml.bind.JAXBContext',
|
|
'javax.xml.bind.JAXBElement$GlobalScope',
|
|
'javax.xml.bind.JAXBElement',
|
|
'javax.xml.bind.JAXBException',
|
|
'javax.xml.bind.JAXBIntrospector',
|
|
'javax.xml.bind.JAXBPermission',
|
|
'javax.xml.bind.MarshalException',
|
|
'javax.xml.bind.Marshaller$Listener',
|
|
'javax.xml.bind.Marshaller',
|
|
'javax.xml.bind.Messages',
|
|
'javax.xml.bind.NotIdentifiableEvent',
|
|
'javax.xml.bind.ParseConversionEvent',
|
|
'javax.xml.bind.PrintConversionEvent',
|
|
'javax.xml.bind.PropertyException',
|
|
'javax.xml.bind.SchemaOutputResolver',
|
|
'javax.xml.bind.TypeConstraintException',
|
|
'javax.xml.bind.UnmarshalException',
|
|
'javax.xml.bind.Unmarshaller$Listener',
|
|
'javax.xml.bind.Unmarshaller',
|
|
'javax.xml.bind.UnmarshallerHandler',
|
|
'javax.xml.bind.ValidationEvent',
|
|
'javax.xml.bind.ValidationEventHandler',
|
|
'javax.xml.bind.ValidationEventLocator',
|
|
'javax.xml.bind.ValidationException',
|
|
'javax.xml.bind.Validator',
|
|
'javax.xml.bind.WhiteSpaceProcessor',
|
|
'javax.xml.bind.annotation.DomHandler',
|
|
'javax.xml.bind.annotation.W3CDomHandler',
|
|
'javax.xml.bind.annotation.XmlAccessOrder',
|
|
'javax.xml.bind.annotation.XmlAccessType',
|
|
'javax.xml.bind.annotation.XmlAccessorOrder',
|
|
'javax.xml.bind.annotation.XmlAccessorType',
|
|
'javax.xml.bind.annotation.XmlAnyAttribute',
|
|
'javax.xml.bind.annotation.XmlAnyElement',
|
|
'javax.xml.bind.annotation.XmlAttachmentRef',
|
|
'javax.xml.bind.annotation.XmlAttribute',
|
|
'javax.xml.bind.annotation.XmlElement$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlElement',
|
|
'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
|
|
'javax.xml.bind.annotation.XmlElementDecl',
|
|
'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlElementRef',
|
|
'javax.xml.bind.annotation.XmlElementRefs',
|
|
'javax.xml.bind.annotation.XmlElementWrapper',
|
|
'javax.xml.bind.annotation.XmlElements',
|
|
'javax.xml.bind.annotation.XmlEnum',
|
|
'javax.xml.bind.annotation.XmlEnumValue',
|
|
'javax.xml.bind.annotation.XmlID',
|
|
'javax.xml.bind.annotation.XmlIDREF',
|
|
'javax.xml.bind.annotation.XmlInlineBinaryData',
|
|
'javax.xml.bind.annotation.XmlList',
|
|
'javax.xml.bind.annotation.XmlMimeType',
|
|
'javax.xml.bind.annotation.XmlMixed',
|
|
'javax.xml.bind.annotation.XmlNs',
|
|
'javax.xml.bind.annotation.XmlNsForm',
|
|
'javax.xml.bind.annotation.XmlRegistry',
|
|
'javax.xml.bind.annotation.XmlRootElement',
|
|
'javax.xml.bind.annotation.XmlSchema',
|
|
'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlSchemaType',
|
|
'javax.xml.bind.annotation.XmlSchemaTypes',
|
|
'javax.xml.bind.annotation.XmlSeeAlso',
|
|
'javax.xml.bind.annotation.XmlTransient',
|
|
'javax.xml.bind.annotation.XmlType$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlType',
|
|
'javax.xml.bind.annotation.XmlValue',
|
|
'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
|
|
'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
|
|
'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
|
|
'javax.xml.bind.attachment.AttachmentMarshaller',
|
|
'javax.xml.bind.attachment.AttachmentUnmarshaller',
|
|
'javax.xml.bind.helpers.AbstractMarshallerImpl',
|
|
'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
|
|
'javax.xml.bind.helpers.DefaultValidationEventHandler',
|
|
'javax.xml.bind.helpers.Messages',
|
|
'javax.xml.bind.helpers.NotIdentifiableEventImpl',
|
|
'javax.xml.bind.helpers.ParseConversionEventImpl',
|
|
'javax.xml.bind.helpers.PrintConversionEventImpl',
|
|
'javax.xml.bind.helpers.ValidationEventImpl',
|
|
'javax.xml.bind.helpers.ValidationEventLocatorImpl',
|
|
'javax.xml.bind.util.JAXBResult',
|
|
'javax.xml.bind.util.JAXBSource$1',
|
|
'javax.xml.bind.util.JAXBSource',
|
|
'javax.xml.bind.util.Messages',
|
|
'javax.xml.bind.util.ValidationEventCollector'
|
|
)
|
|
} else {
|
|
thirdPartyAudit.ignoreMissingClasses 'javax.activation.DataHandler'
|
|
}
|