448 lines
18 KiB
Groovy
448 lines
18 KiB
Groovy
import org.apache.tools.ant.taskdefs.condition.Os
|
|
import org.elasticsearch.gradle.LoggedExec
|
|
import org.elasticsearch.gradle.MavenFilteringHack
|
|
import org.elasticsearch.gradle.test.AntFixture
|
|
import org.elasticsearch.gradle.test.ClusterConfiguration
|
|
import org.elasticsearch.gradle.test.RestIntegTestTask
|
|
|
|
import java.lang.reflect.Field
|
|
|
|
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
esplugin {
|
|
description 'The S3 repository plugin adds S3 repositories'
|
|
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
|
|
}
|
|
|
|
versions << [
|
|
'aws': '1.11.223'
|
|
]
|
|
|
|
dependencies {
|
|
compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
|
|
compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}"
|
|
compile "com.amazonaws:aws-java-sdk-core:${versions.aws}"
|
|
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
|
|
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
|
|
compile "commons-logging:commons-logging:${versions.commonslogging}"
|
|
compile "commons-codec:commons-codec:${versions.commonscodec}"
|
|
compile 'com.fasterxml.jackson.core:jackson-databind:2.6.7.1'
|
|
compile 'com.fasterxml.jackson.core:jackson-annotations:2.6.0'
|
|
|
|
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
|
|
// and whitelist this hack in JarHell
|
|
compile 'javax.xml.bind:jaxb-api:2.2.2'
|
|
}
|
|
|
|
dependencyLicenses {
|
|
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
|
|
mapping from: /jackson-.*/, to: 'jackson'
|
|
mapping from: /jaxb-.*/, to: 'jaxb'
|
|
}
|
|
|
|
bundlePlugin {
|
|
from('config/repository-s3') {
|
|
into 'config'
|
|
}
|
|
}
|
|
|
|
additionalTest('testRepositoryCreds'){
|
|
include '**/RepositoryCredentialsTests.class'
|
|
systemProperty 'es.allow_insecure_settings', 'true'
|
|
}
|
|
|
|
test {
|
|
// these are tested explicitly in separate test tasks
|
|
exclude '**/*CredentialsTests.class'
|
|
}
|
|
|
|
boolean useFixture = false
|
|
|
|
// We test against two repositories, one which uses the usual two-part "permanent" credentials and
|
|
// the other which uses three-part "temporary" or "session" credentials.
|
|
|
|
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
|
|
String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
|
|
String s3PermanentBucket = System.getenv("amazon_s3_bucket")
|
|
String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
|
|
|
|
String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
|
|
String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
|
|
String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
|
|
String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
|
|
String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
|
|
|
|
// If all these variables are missing then we are testing against the internal fixture instead, which has the following
|
|
// credentials hard-coded in.
|
|
|
|
if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath
|
|
&& !s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
|
|
|
|
s3PermanentAccessKey = 's3_integration_test_permanent_access_key'
|
|
s3PermanentSecretKey = 's3_integration_test_permanent_secret_key'
|
|
s3PermanentBucket = 'permanent-bucket-test'
|
|
s3PermanentBasePath = 'integration_test'
|
|
|
|
s3TemporaryAccessKey = 's3_integration_test_temporary_access_key'
|
|
s3TemporarySecretKey = 's3_integration_test_temporary_secret_key'
|
|
s3TemporaryBucket = 'temporary-bucket-test'
|
|
s3TemporaryBasePath = 'integration_test'
|
|
s3TemporarySessionToken = 's3_integration_test_temporary_session_token'
|
|
|
|
useFixture = true
|
|
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath
|
|
|| !s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
|
|
throw new IllegalArgumentException("not all options specified to run against external S3 service")
|
|
}
|
|
|
|
final String minioVersion = 'RELEASE.2018-06-22T23-48-46Z'
|
|
final String minioBinDir = "${buildDir}/minio/bin"
|
|
final String minioDataDir = "${buildDir}/minio/data"
|
|
final String minioAddress = "127.0.0.1:60920"
|
|
|
|
final String minioDistribution
|
|
final String minioCheckSum
|
|
if (Os.isFamily(Os.FAMILY_MAC)) {
|
|
minioDistribution = 'darwin-amd64'
|
|
minioCheckSum = '96b0bcb2f590e8e65fb83d5c3e221f9bd1106b49fa6f22c6b726b80b845d7c60'
|
|
} else if (Os.isFamily(Os.FAMILY_UNIX)) {
|
|
minioDistribution = 'linux-amd64'
|
|
minioCheckSum = '713dac7c105285eab3b92649be92b5e793b29d3525c7929fa7aaed99374fad99'
|
|
} else {
|
|
minioDistribution = null
|
|
minioCheckSum = null
|
|
}
|
|
|
|
buildscript {
|
|
repositories {
|
|
maven {
|
|
url 'https://plugins.gradle.org/m2/'
|
|
}
|
|
}
|
|
dependencies {
|
|
classpath 'de.undercouch:gradle-download-task:3.4.3'
|
|
}
|
|
}
|
|
|
|
if (useFixture && minioDistribution) {
|
|
apply plugin: 'de.undercouch.download'
|
|
|
|
final String minioFileName = "minio.${minioVersion}"
|
|
final String minioDownloadURL = "https://dl.minio.io/server/minio/release/${minioDistribution}/archive/${minioFileName}"
|
|
final String minioFilePath = "${gradle.gradleUserHomeDir}/downloads/minio/${minioDistribution}/${minioFileName}"
|
|
|
|
task downloadMinio(type: Download) {
|
|
src minioDownloadURL
|
|
dest minioFilePath
|
|
onlyIfModified true
|
|
}
|
|
|
|
task verifyMinioChecksum(type: Verify, dependsOn: downloadMinio) {
|
|
src minioFilePath
|
|
algorithm 'SHA-256'
|
|
checksum minioCheckSum
|
|
}
|
|
|
|
task installMinio(type: Sync, dependsOn: verifyMinioChecksum) {
|
|
from minioFilePath
|
|
into minioBinDir
|
|
fileMode 0755
|
|
}
|
|
|
|
task startMinio {
|
|
dependsOn installMinio
|
|
|
|
ext.minioPid = 0L
|
|
|
|
doLast {
|
|
new File("${minioDataDir}/${s3PermanentBucket}").mkdirs()
|
|
// we skip these tests on Windows so we do no need to worry about compatibility here
|
|
final ProcessBuilder minio = new ProcessBuilder(
|
|
"${minioBinDir}/${minioFileName}",
|
|
"server",
|
|
"--address",
|
|
minioAddress,
|
|
minioDataDir)
|
|
minio.environment().put('MINIO_ACCESS_KEY', s3PermanentAccessKey)
|
|
minio.environment().put('MINIO_SECRET_KEY', s3PermanentSecretKey)
|
|
final Process process = minio.start()
|
|
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
|
|
try {
|
|
Class<?> cProcessImpl = process.getClass()
|
|
Field fPid = cProcessImpl.getDeclaredField("pid")
|
|
if (!fPid.isAccessible()) {
|
|
fPid.setAccessible(true)
|
|
}
|
|
minioPid = fPid.getInt(process)
|
|
} catch (Exception e) {
|
|
logger.error("failed to read pid from minio process", e)
|
|
process.destroyForcibly()
|
|
throw e
|
|
}
|
|
} else {
|
|
minioPid = process.pid()
|
|
}
|
|
|
|
new BufferedReader(new InputStreamReader(process.getInputStream())).withReader { br ->
|
|
String line
|
|
int httpPort = 0
|
|
while ((line = br.readLine()) != null) {
|
|
logger.info(line)
|
|
if (line.matches('.*Endpoint.*:\\d+$')) {
|
|
assert httpPort == 0
|
|
final int index = line.lastIndexOf(":")
|
|
assert index >= 0
|
|
httpPort = Integer.parseInt(line.substring(index + 1))
|
|
|
|
final File script = new File(project.buildDir, "minio/minio.killer.sh")
|
|
script.setText(
|
|
["function shutdown {",
|
|
" kill ${minioPid}",
|
|
"}",
|
|
"trap shutdown EXIT",
|
|
// will wait indefinitely for input, but we never pass input, and the pipe is only closed when the build dies
|
|
"read line\n"].join('\n'), 'UTF-8')
|
|
final ProcessBuilder killer = new ProcessBuilder("bash", script.absolutePath)
|
|
killer.start()
|
|
break
|
|
}
|
|
}
|
|
|
|
assert httpPort > 0
|
|
}
|
|
}
|
|
}
|
|
|
|
task stopMinio(type: LoggedExec) {
|
|
onlyIf { startMinio.minioPid > 0 }
|
|
|
|
doFirst {
|
|
logger.info("Shutting down minio with pid ${startMinio.minioPid}")
|
|
}
|
|
|
|
final Object pid = "${ -> startMinio.minioPid }"
|
|
|
|
// we skip these tests on Windows so we do no need to worry about compatibility here
|
|
executable = 'kill'
|
|
args('-9', pid)
|
|
}
|
|
|
|
RestIntegTestTask integTestMinio = project.tasks.create('integTestMinio', RestIntegTestTask.class) {
|
|
description = "Runs REST tests using the Minio repository."
|
|
}
|
|
|
|
// The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
|
|
project.afterEvaluate {
|
|
ClusterConfiguration cluster = project.extensions.getByName('integTestMinioCluster') as ClusterConfiguration
|
|
cluster.dependsOn(project.bundlePlugin)
|
|
cluster.keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
|
|
cluster.keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
|
|
|
|
cluster.setting 's3.client.integration_test_permanent.endpoint', "http://${minioAddress}"
|
|
|
|
Task restIntegTestTask = project.tasks.getByName('integTestMinio')
|
|
restIntegTestTask.clusterConfig.plugin(project.path)
|
|
|
|
// Default jvm arguments for all test clusters
|
|
String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
|
|
" " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
|
|
" " + System.getProperty('tests.jvm.argline', '')
|
|
|
|
restIntegTestTask.clusterConfig.jvmArgs = jvmArgs
|
|
}
|
|
|
|
integTestMinioRunner.dependsOn(startMinio)
|
|
integTestMinioRunner.finalizedBy(stopMinio)
|
|
// Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
|
|
integTestMinioRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/30_repository_temporary_credentials/*'
|
|
|
|
project.check.dependsOn(integTestMinio)
|
|
}
|
|
|
|
/** A task to start the AmazonS3Fixture which emulates an S3 service **/
|
|
task s3Fixture(type: AntFixture) {
|
|
dependsOn testClasses
|
|
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
|
|
executable = new File(project.runtimeJavaHome, 'bin/java')
|
|
args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3PermanentBucket, s3TemporaryBucket
|
|
}
|
|
|
|
Map<String, Object> expansions = [
|
|
'permanent_bucket': s3PermanentBucket,
|
|
'permanent_base_path': s3PermanentBasePath,
|
|
'temporary_bucket': s3TemporaryBucket,
|
|
'temporary_base_path': s3TemporaryBasePath
|
|
]
|
|
|
|
processTestResources {
|
|
inputs.properties(expansions)
|
|
MavenFilteringHack.filter(it, expansions)
|
|
}
|
|
|
|
integTestCluster {
|
|
keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
|
|
keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
|
|
|
|
keystoreSetting 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
|
|
keystoreSetting 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
|
|
keystoreSetting 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
|
|
|
|
if (useFixture) {
|
|
dependsOn s3Fixture
|
|
/* Use a closure on the string to delay evaluation until tests are executed */
|
|
setting 's3.client.integration_test_permanent.endpoint', "http://${-> s3Fixture.addressAndPort}"
|
|
setting 's3.client.integration_test_temporary.endpoint', "http://${-> s3Fixture.addressAndPort}"
|
|
} else {
|
|
println "Using an external service to test the repository-s3 plugin"
|
|
}
|
|
}
|
|
|
|
thirdPartyAudit.excludes = [
|
|
// classes are missing
|
|
'javax.servlet.ServletContextEvent',
|
|
'javax.servlet.ServletContextListener',
|
|
'org.apache.avalon.framework.logger.Logger',
|
|
'org.apache.log.Hierarchy',
|
|
'org.apache.log.Logger',
|
|
'software.amazon.ion.IonReader',
|
|
'software.amazon.ion.IonSystem',
|
|
'software.amazon.ion.IonType',
|
|
'software.amazon.ion.IonWriter',
|
|
'software.amazon.ion.Timestamp',
|
|
'software.amazon.ion.system.IonBinaryWriterBuilder',
|
|
'software.amazon.ion.system.IonSystemBuilder',
|
|
'software.amazon.ion.system.IonTextWriterBuilder',
|
|
'software.amazon.ion.system.IonWriterBuilder',
|
|
]
|
|
|
|
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
|
|
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
|
|
thirdPartyAudit.excludes += [
|
|
'javax.xml.bind.Binder',
|
|
'javax.xml.bind.ContextFinder$1',
|
|
'javax.xml.bind.ContextFinder',
|
|
'javax.xml.bind.DataBindingException',
|
|
'javax.xml.bind.DatatypeConverter',
|
|
'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
|
|
'javax.xml.bind.DatatypeConverterImpl',
|
|
'javax.xml.bind.DatatypeConverterInterface',
|
|
'javax.xml.bind.Element',
|
|
'javax.xml.bind.GetPropertyAction',
|
|
'javax.xml.bind.JAXB$Cache',
|
|
'javax.xml.bind.JAXB',
|
|
'javax.xml.bind.JAXBContext',
|
|
'javax.xml.bind.JAXBElement$GlobalScope',
|
|
'javax.xml.bind.JAXBElement',
|
|
'javax.xml.bind.JAXBException',
|
|
'javax.xml.bind.JAXBIntrospector',
|
|
'javax.xml.bind.JAXBPermission',
|
|
'javax.xml.bind.MarshalException',
|
|
'javax.xml.bind.Marshaller$Listener',
|
|
'javax.xml.bind.Marshaller',
|
|
'javax.xml.bind.Messages',
|
|
'javax.xml.bind.NotIdentifiableEvent',
|
|
'javax.xml.bind.ParseConversionEvent',
|
|
'javax.xml.bind.PrintConversionEvent',
|
|
'javax.xml.bind.PropertyException',
|
|
'javax.xml.bind.SchemaOutputResolver',
|
|
'javax.xml.bind.TypeConstraintException',
|
|
'javax.xml.bind.UnmarshalException',
|
|
'javax.xml.bind.Unmarshaller$Listener',
|
|
'javax.xml.bind.Unmarshaller',
|
|
'javax.xml.bind.UnmarshallerHandler',
|
|
'javax.xml.bind.ValidationEvent',
|
|
'javax.xml.bind.ValidationEventHandler',
|
|
'javax.xml.bind.ValidationEventLocator',
|
|
'javax.xml.bind.ValidationException',
|
|
'javax.xml.bind.Validator',
|
|
'javax.xml.bind.WhiteSpaceProcessor',
|
|
'javax.xml.bind.annotation.DomHandler',
|
|
'javax.xml.bind.annotation.W3CDomHandler',
|
|
'javax.xml.bind.annotation.XmlAccessOrder',
|
|
'javax.xml.bind.annotation.XmlAccessType',
|
|
'javax.xml.bind.annotation.XmlAccessorOrder',
|
|
'javax.xml.bind.annotation.XmlAccessorType',
|
|
'javax.xml.bind.annotation.XmlAnyAttribute',
|
|
'javax.xml.bind.annotation.XmlAnyElement',
|
|
'javax.xml.bind.annotation.XmlAttachmentRef',
|
|
'javax.xml.bind.annotation.XmlAttribute',
|
|
'javax.xml.bind.annotation.XmlElement$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlElement',
|
|
'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
|
|
'javax.xml.bind.annotation.XmlElementDecl',
|
|
'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlElementRef',
|
|
'javax.xml.bind.annotation.XmlElementRefs',
|
|
'javax.xml.bind.annotation.XmlElementWrapper',
|
|
'javax.xml.bind.annotation.XmlElements',
|
|
'javax.xml.bind.annotation.XmlEnum',
|
|
'javax.xml.bind.annotation.XmlEnumValue',
|
|
'javax.xml.bind.annotation.XmlID',
|
|
'javax.xml.bind.annotation.XmlIDREF',
|
|
'javax.xml.bind.annotation.XmlInlineBinaryData',
|
|
'javax.xml.bind.annotation.XmlList',
|
|
'javax.xml.bind.annotation.XmlMimeType',
|
|
'javax.xml.bind.annotation.XmlMixed',
|
|
'javax.xml.bind.annotation.XmlNs',
|
|
'javax.xml.bind.annotation.XmlNsForm',
|
|
'javax.xml.bind.annotation.XmlRegistry',
|
|
'javax.xml.bind.annotation.XmlRootElement',
|
|
'javax.xml.bind.annotation.XmlSchema',
|
|
'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlSchemaType',
|
|
'javax.xml.bind.annotation.XmlSchemaTypes',
|
|
'javax.xml.bind.annotation.XmlSeeAlso',
|
|
'javax.xml.bind.annotation.XmlTransient',
|
|
'javax.xml.bind.annotation.XmlType$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlType',
|
|
'javax.xml.bind.annotation.XmlValue',
|
|
'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
|
|
'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
|
|
'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
|
|
'javax.xml.bind.attachment.AttachmentMarshaller',
|
|
'javax.xml.bind.attachment.AttachmentUnmarshaller',
|
|
'javax.xml.bind.helpers.AbstractMarshallerImpl',
|
|
'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
|
|
'javax.xml.bind.helpers.DefaultValidationEventHandler',
|
|
'javax.xml.bind.helpers.Messages',
|
|
'javax.xml.bind.helpers.NotIdentifiableEventImpl',
|
|
'javax.xml.bind.helpers.ParseConversionEventImpl',
|
|
'javax.xml.bind.helpers.PrintConversionEventImpl',
|
|
'javax.xml.bind.helpers.ValidationEventImpl',
|
|
'javax.xml.bind.helpers.ValidationEventLocatorImpl',
|
|
'javax.xml.bind.util.JAXBResult',
|
|
'javax.xml.bind.util.JAXBSource$1',
|
|
'javax.xml.bind.util.JAXBSource',
|
|
'javax.xml.bind.util.Messages',
|
|
'javax.xml.bind.util.ValidationEventCollector'
|
|
]
|
|
} else {
|
|
thirdPartyAudit.excludes += ['javax.activation.DataHandler']
|
|
}
|
|
|
|
// AWS SDK is exposing some deprecated methods which we call using a delegate:
|
|
// * setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation)
|
|
// * changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass)
|
|
compileTestJava.options.compilerArgs << "-Xlint:-deprecation"
|