mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-10 06:55:32 +00:00
32d67ef504
Adds a Minio fixture to run the S3 repository tests against Minio. Also collapses the single qa subproject into the s3-repository project, which simplifies the code structure (having it all in one place) and helps to avoid having too many Gradle subprojects.
448 lines
18 KiB
Groovy
448 lines
18 KiB
Groovy
import org.apache.tools.ant.taskdefs.condition.Os
|
|
import org.elasticsearch.gradle.LoggedExec
|
|
import org.elasticsearch.gradle.MavenFilteringHack
|
|
import org.elasticsearch.gradle.test.AntFixture
|
|
import org.elasticsearch.gradle.test.ClusterConfiguration
|
|
import org.elasticsearch.gradle.test.RestIntegTestTask
|
|
|
|
import java.lang.reflect.Field
|
|
|
|
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
esplugin {
|
|
description 'The S3 repository plugin adds S3 repositories'
|
|
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
|
|
}
|
|
|
|
versions << [
|
|
'aws': '1.11.223'
|
|
]
|
|
|
|
dependencies {
|
|
compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
|
|
compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}"
|
|
compile "com.amazonaws:aws-java-sdk-core:${versions.aws}"
|
|
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
|
|
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
|
|
compile "commons-logging:commons-logging:${versions.commonslogging}"
|
|
compile "commons-codec:commons-codec:${versions.commonscodec}"
|
|
compile 'com.fasterxml.jackson.core:jackson-databind:2.6.7.1'
|
|
compile 'com.fasterxml.jackson.core:jackson-annotations:2.6.0'
|
|
|
|
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
|
|
// and whitelist this hack in JarHell
|
|
compile 'javax.xml.bind:jaxb-api:2.2.2'
|
|
}
|
|
|
|
dependencyLicenses {
|
|
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
|
|
mapping from: /jackson-.*/, to: 'jackson'
|
|
mapping from: /jaxb-.*/, to: 'jaxb'
|
|
}
|
|
|
|
bundlePlugin {
|
|
from('config/repository-s3') {
|
|
into 'config'
|
|
}
|
|
}
|
|
|
|
additionalTest('testRepositoryCreds'){
|
|
include '**/RepositoryCredentialsTests.class'
|
|
systemProperty 'es.allow_insecure_settings', 'true'
|
|
}
|
|
|
|
test {
|
|
// these are tested explicitly in separate test tasks
|
|
exclude '**/*CredentialsTests.class'
|
|
}
|
|
|
|
boolean useFixture = false
|
|
|
|
// We test against two repositories, one which uses the usual two-part "permanent" credentials and
|
|
// the other which uses three-part "temporary" or "session" credentials.
|
|
|
|
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
|
|
String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
|
|
String s3PermanentBucket = System.getenv("amazon_s3_bucket")
|
|
String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
|
|
|
|
String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
|
|
String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
|
|
String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
|
|
String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
|
|
String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
|
|
|
|
// If all these variables are missing then we are testing against the internal fixture instead, which has the following
|
|
// credentials hard-coded in.
|
|
|
|
if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath
|
|
&& !s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
|
|
|
|
s3PermanentAccessKey = 's3_integration_test_permanent_access_key'
|
|
s3PermanentSecretKey = 's3_integration_test_permanent_secret_key'
|
|
s3PermanentBucket = 'permanent-bucket-test'
|
|
s3PermanentBasePath = 'integration_test'
|
|
|
|
s3TemporaryAccessKey = 's3_integration_test_temporary_access_key'
|
|
s3TemporarySecretKey = 's3_integration_test_temporary_secret_key'
|
|
s3TemporaryBucket = 'temporary-bucket-test'
|
|
s3TemporaryBasePath = 'integration_test'
|
|
s3TemporarySessionToken = 's3_integration_test_temporary_session_token'
|
|
|
|
useFixture = true
|
|
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath
|
|
|| !s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
|
|
throw new IllegalArgumentException("not all options specified to run against external S3 service")
|
|
}
|
|
|
|
final String minioVersion = 'RELEASE.2018-06-22T23-48-46Z'
|
|
final String minioBinDir = "${buildDir}/minio/bin"
|
|
final String minioDataDir = "${buildDir}/minio/data"
|
|
final String minioAddress = "127.0.0.1:60920"
|
|
|
|
final String minioDistribution
|
|
final String minioCheckSum
|
|
if (Os.isFamily(Os.FAMILY_MAC)) {
|
|
minioDistribution = 'darwin-amd64'
|
|
minioCheckSum = '96b0bcb2f590e8e65fb83d5c3e221f9bd1106b49fa6f22c6b726b80b845d7c60'
|
|
} else if (Os.isFamily(Os.FAMILY_UNIX)) {
|
|
minioDistribution = 'linux-amd64'
|
|
minioCheckSum = '713dac7c105285eab3b92649be92b5e793b29d3525c7929fa7aaed99374fad99'
|
|
} else {
|
|
minioDistribution = null
|
|
minioCheckSum = null
|
|
}
|
|
|
|
buildscript {
|
|
repositories {
|
|
maven {
|
|
url 'https://plugins.gradle.org/m2/'
|
|
}
|
|
}
|
|
dependencies {
|
|
classpath 'de.undercouch:gradle-download-task:3.4.3'
|
|
}
|
|
}
|
|
|
|
if (useFixture && minioDistribution) {
|
|
apply plugin: 'de.undercouch.download'
|
|
|
|
final String minioFileName = "minio.${minioVersion}"
|
|
final String minioDownloadURL = "https://dl.minio.io/server/minio/release/${minioDistribution}/archive/${minioFileName}"
|
|
final String minioFilePath = "${gradle.gradleUserHomeDir}/downloads/minio/${minioDistribution}/${minioFileName}"
|
|
|
|
task downloadMinio(type: Download) {
|
|
src minioDownloadURL
|
|
dest minioFilePath
|
|
onlyIfModified true
|
|
}
|
|
|
|
task verifyMinioChecksum(type: Verify, dependsOn: downloadMinio) {
|
|
src minioFilePath
|
|
algorithm 'SHA-256'
|
|
checksum minioCheckSum
|
|
}
|
|
|
|
task installMinio(type: Sync, dependsOn: verifyMinioChecksum) {
|
|
from minioFilePath
|
|
into minioBinDir
|
|
fileMode 0755
|
|
}
|
|
|
|
task startMinio {
|
|
dependsOn installMinio
|
|
|
|
ext.minioPid = 0L
|
|
|
|
doLast {
|
|
new File("${minioDataDir}/${s3PermanentBucket}").mkdirs()
|
|
// we skip these tests on Windows so we do no need to worry about compatibility here
|
|
final ProcessBuilder minio = new ProcessBuilder(
|
|
"${minioBinDir}/${minioFileName}",
|
|
"server",
|
|
"--address",
|
|
minioAddress,
|
|
minioDataDir)
|
|
minio.environment().put('MINIO_ACCESS_KEY', s3PermanentAccessKey)
|
|
minio.environment().put('MINIO_SECRET_KEY', s3PermanentSecretKey)
|
|
final Process process = minio.start()
|
|
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
|
|
try {
|
|
Class<?> cProcessImpl = process.getClass()
|
|
Field fPid = cProcessImpl.getDeclaredField("pid")
|
|
if (!fPid.isAccessible()) {
|
|
fPid.setAccessible(true)
|
|
}
|
|
minioPid = fPid.getInt(process)
|
|
} catch (Exception e) {
|
|
logger.error("failed to read pid from minio process", e)
|
|
process.destroyForcibly()
|
|
throw e
|
|
}
|
|
} else {
|
|
minioPid = process.pid()
|
|
}
|
|
|
|
new BufferedReader(new InputStreamReader(process.getInputStream())).withReader { br ->
|
|
String line
|
|
int httpPort = 0
|
|
while ((line = br.readLine()) != null) {
|
|
logger.info(line)
|
|
if (line.matches('.*Endpoint.*:\\d+$')) {
|
|
assert httpPort == 0
|
|
final int index = line.lastIndexOf(":")
|
|
assert index >= 0
|
|
httpPort = Integer.parseInt(line.substring(index + 1))
|
|
|
|
final File script = new File(project.buildDir, "minio/minio.killer.sh")
|
|
script.setText(
|
|
["function shutdown {",
|
|
" kill ${minioPid}",
|
|
"}",
|
|
"trap shutdown EXIT",
|
|
// will wait indefinitely for input, but we never pass input, and the pipe is only closed when the build dies
|
|
"read line\n"].join('\n'), 'UTF-8')
|
|
final ProcessBuilder killer = new ProcessBuilder("bash", script.absolutePath)
|
|
killer.start()
|
|
break
|
|
}
|
|
}
|
|
|
|
assert httpPort > 0
|
|
}
|
|
}
|
|
}
|
|
|
|
task stopMinio(type: LoggedExec) {
|
|
onlyIf { startMinio.minioPid > 0 }
|
|
|
|
doFirst {
|
|
logger.info("Shutting down minio with pid ${startMinio.minioPid}")
|
|
}
|
|
|
|
final Object pid = "${ -> startMinio.minioPid }"
|
|
|
|
// we skip these tests on Windows so we do no need to worry about compatibility here
|
|
executable = 'kill'
|
|
args('-9', pid)
|
|
}
|
|
|
|
RestIntegTestTask integTestMinio = project.tasks.create('integTestMinio', RestIntegTestTask.class) {
|
|
description = "Runs REST tests using the Minio repository."
|
|
}
|
|
|
|
// The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
|
|
project.afterEvaluate {
|
|
ClusterConfiguration cluster = project.extensions.getByName('integTestMinioCluster') as ClusterConfiguration
|
|
cluster.dependsOn(project.bundlePlugin)
|
|
cluster.keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
|
|
cluster.keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
|
|
|
|
cluster.setting 's3.client.integration_test_permanent.endpoint', "http://${minioAddress}"
|
|
|
|
Task restIntegTestTask = project.tasks.getByName('integTestMinio')
|
|
restIntegTestTask.clusterConfig.plugin(project.path)
|
|
|
|
// Default jvm arguments for all test clusters
|
|
String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
|
|
" " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
|
|
" " + System.getProperty('tests.jvm.argline', '')
|
|
|
|
restIntegTestTask.clusterConfig.jvmArgs = jvmArgs
|
|
}
|
|
|
|
integTestMinioRunner.dependsOn(startMinio)
|
|
integTestMinioRunner.finalizedBy(stopMinio)
|
|
// Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
|
|
integTestMinioRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/30_repository_temporary_credentials/*'
|
|
|
|
project.check.dependsOn(integTestMinio)
|
|
}
|
|
|
|
/** A task to start the AmazonS3Fixture which emulates an S3 service **/
|
|
task s3Fixture(type: AntFixture) {
|
|
dependsOn testClasses
|
|
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
|
|
executable = new File(project.runtimeJavaHome, 'bin/java')
|
|
args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3PermanentBucket, s3TemporaryBucket
|
|
}
|
|
|
|
Map<String, Object> expansions = [
|
|
'permanent_bucket': s3PermanentBucket,
|
|
'permanent_base_path': s3PermanentBasePath,
|
|
'temporary_bucket': s3TemporaryBucket,
|
|
'temporary_base_path': s3TemporaryBasePath
|
|
]
|
|
|
|
processTestResources {
|
|
inputs.properties(expansions)
|
|
MavenFilteringHack.filter(it, expansions)
|
|
}
|
|
|
|
integTestCluster {
|
|
keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
|
|
keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
|
|
|
|
keystoreSetting 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
|
|
keystoreSetting 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
|
|
keystoreSetting 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
|
|
|
|
if (useFixture) {
|
|
dependsOn s3Fixture
|
|
/* Use a closure on the string to delay evaluation until tests are executed */
|
|
setting 's3.client.integration_test_permanent.endpoint', "http://${-> s3Fixture.addressAndPort}"
|
|
setting 's3.client.integration_test_temporary.endpoint', "http://${-> s3Fixture.addressAndPort}"
|
|
} else {
|
|
println "Using an external service to test the repository-s3 plugin"
|
|
}
|
|
}
|
|
|
|
thirdPartyAudit.excludes = [
|
|
// classes are missing
|
|
'javax.servlet.ServletContextEvent',
|
|
'javax.servlet.ServletContextListener',
|
|
'org.apache.avalon.framework.logger.Logger',
|
|
'org.apache.log.Hierarchy',
|
|
'org.apache.log.Logger',
|
|
'software.amazon.ion.IonReader',
|
|
'software.amazon.ion.IonSystem',
|
|
'software.amazon.ion.IonType',
|
|
'software.amazon.ion.IonWriter',
|
|
'software.amazon.ion.Timestamp',
|
|
'software.amazon.ion.system.IonBinaryWriterBuilder',
|
|
'software.amazon.ion.system.IonSystemBuilder',
|
|
'software.amazon.ion.system.IonTextWriterBuilder',
|
|
'software.amazon.ion.system.IonWriterBuilder',
|
|
]
|
|
|
|
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
|
|
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
|
|
thirdPartyAudit.excludes += [
|
|
'javax.xml.bind.Binder',
|
|
'javax.xml.bind.ContextFinder$1',
|
|
'javax.xml.bind.ContextFinder',
|
|
'javax.xml.bind.DataBindingException',
|
|
'javax.xml.bind.DatatypeConverter',
|
|
'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
|
|
'javax.xml.bind.DatatypeConverterImpl',
|
|
'javax.xml.bind.DatatypeConverterInterface',
|
|
'javax.xml.bind.Element',
|
|
'javax.xml.bind.GetPropertyAction',
|
|
'javax.xml.bind.JAXB$Cache',
|
|
'javax.xml.bind.JAXB',
|
|
'javax.xml.bind.JAXBContext',
|
|
'javax.xml.bind.JAXBElement$GlobalScope',
|
|
'javax.xml.bind.JAXBElement',
|
|
'javax.xml.bind.JAXBException',
|
|
'javax.xml.bind.JAXBIntrospector',
|
|
'javax.xml.bind.JAXBPermission',
|
|
'javax.xml.bind.MarshalException',
|
|
'javax.xml.bind.Marshaller$Listener',
|
|
'javax.xml.bind.Marshaller',
|
|
'javax.xml.bind.Messages',
|
|
'javax.xml.bind.NotIdentifiableEvent',
|
|
'javax.xml.bind.ParseConversionEvent',
|
|
'javax.xml.bind.PrintConversionEvent',
|
|
'javax.xml.bind.PropertyException',
|
|
'javax.xml.bind.SchemaOutputResolver',
|
|
'javax.xml.bind.TypeConstraintException',
|
|
'javax.xml.bind.UnmarshalException',
|
|
'javax.xml.bind.Unmarshaller$Listener',
|
|
'javax.xml.bind.Unmarshaller',
|
|
'javax.xml.bind.UnmarshallerHandler',
|
|
'javax.xml.bind.ValidationEvent',
|
|
'javax.xml.bind.ValidationEventHandler',
|
|
'javax.xml.bind.ValidationEventLocator',
|
|
'javax.xml.bind.ValidationException',
|
|
'javax.xml.bind.Validator',
|
|
'javax.xml.bind.WhiteSpaceProcessor',
|
|
'javax.xml.bind.annotation.DomHandler',
|
|
'javax.xml.bind.annotation.W3CDomHandler',
|
|
'javax.xml.bind.annotation.XmlAccessOrder',
|
|
'javax.xml.bind.annotation.XmlAccessType',
|
|
'javax.xml.bind.annotation.XmlAccessorOrder',
|
|
'javax.xml.bind.annotation.XmlAccessorType',
|
|
'javax.xml.bind.annotation.XmlAnyAttribute',
|
|
'javax.xml.bind.annotation.XmlAnyElement',
|
|
'javax.xml.bind.annotation.XmlAttachmentRef',
|
|
'javax.xml.bind.annotation.XmlAttribute',
|
|
'javax.xml.bind.annotation.XmlElement$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlElement',
|
|
'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
|
|
'javax.xml.bind.annotation.XmlElementDecl',
|
|
'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlElementRef',
|
|
'javax.xml.bind.annotation.XmlElementRefs',
|
|
'javax.xml.bind.annotation.XmlElementWrapper',
|
|
'javax.xml.bind.annotation.XmlElements',
|
|
'javax.xml.bind.annotation.XmlEnum',
|
|
'javax.xml.bind.annotation.XmlEnumValue',
|
|
'javax.xml.bind.annotation.XmlID',
|
|
'javax.xml.bind.annotation.XmlIDREF',
|
|
'javax.xml.bind.annotation.XmlInlineBinaryData',
|
|
'javax.xml.bind.annotation.XmlList',
|
|
'javax.xml.bind.annotation.XmlMimeType',
|
|
'javax.xml.bind.annotation.XmlMixed',
|
|
'javax.xml.bind.annotation.XmlNs',
|
|
'javax.xml.bind.annotation.XmlNsForm',
|
|
'javax.xml.bind.annotation.XmlRegistry',
|
|
'javax.xml.bind.annotation.XmlRootElement',
|
|
'javax.xml.bind.annotation.XmlSchema',
|
|
'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlSchemaType',
|
|
'javax.xml.bind.annotation.XmlSchemaTypes',
|
|
'javax.xml.bind.annotation.XmlSeeAlso',
|
|
'javax.xml.bind.annotation.XmlTransient',
|
|
'javax.xml.bind.annotation.XmlType$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlType',
|
|
'javax.xml.bind.annotation.XmlValue',
|
|
'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
|
|
'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
|
|
'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
|
|
'javax.xml.bind.attachment.AttachmentMarshaller',
|
|
'javax.xml.bind.attachment.AttachmentUnmarshaller',
|
|
'javax.xml.bind.helpers.AbstractMarshallerImpl',
|
|
'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
|
|
'javax.xml.bind.helpers.DefaultValidationEventHandler',
|
|
'javax.xml.bind.helpers.Messages',
|
|
'javax.xml.bind.helpers.NotIdentifiableEventImpl',
|
|
'javax.xml.bind.helpers.ParseConversionEventImpl',
|
|
'javax.xml.bind.helpers.PrintConversionEventImpl',
|
|
'javax.xml.bind.helpers.ValidationEventImpl',
|
|
'javax.xml.bind.helpers.ValidationEventLocatorImpl',
|
|
'javax.xml.bind.util.JAXBResult',
|
|
'javax.xml.bind.util.JAXBSource$1',
|
|
'javax.xml.bind.util.JAXBSource',
|
|
'javax.xml.bind.util.Messages',
|
|
'javax.xml.bind.util.ValidationEventCollector'
|
|
]
|
|
} else {
|
|
thirdPartyAudit.excludes += ['javax.activation.DataHandler']
|
|
}
|
|
|
|
// AWS SDK is exposing some deprecated methods which we call using a delegate:
|
|
// * setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation)
|
|
// * changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass)
|
|
compileTestJava.options.compilerArgs << "-Xlint:-deprecation"
|