OpenSearch/plugins/repository-s3/build.gradle

562 lines
22 KiB
Groovy

import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.test.AntFixture
import org.elasticsearch.gradle.test.ClusterConfiguration
import org.elasticsearch.gradle.test.RestIntegTestTask
import java.lang.reflect.Field
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
esplugin {
description 'The S3 repository plugin adds S3 repositories'
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
}
versions << [
'aws': '1.11.223'
]
dependencies {
compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}"
compile "com.amazonaws:aws-java-sdk-core:${versions.aws}"
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
compile "commons-logging:commons-logging:${versions.commonslogging}"
compile "commons-codec:commons-codec:${versions.commonscodec}"
compile 'com.fasterxml.jackson.core:jackson-databind:2.6.7.1'
compile 'com.fasterxml.jackson.core:jackson-annotations:2.6.0'
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
// and whitelist this hack in JarHell
compile 'javax.xml.bind:jaxb-api:2.2.2'
}
dependencyLicenses {
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
mapping from: /jackson-.*/, to: 'jackson'
mapping from: /jaxb-.*/, to: 'jaxb'
}
bundlePlugin {
from('config/repository-s3') {
into 'config'
}
}
additionalTest('testRepositoryCreds'){
include '**/RepositoryCredentialsTests.class'
systemProperty 'es.allow_insecure_settings', 'true'
}
test {
// these are tested explicitly in separate test tasks
exclude '**/*CredentialsTests.class'
}
boolean useFixture = false
// We test against two repositories, one which uses the usual two-part "permanent" credentials and
// the other which uses three-part "temporary" or "session" credentials.
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
String s3PermanentBucket = System.getenv("amazon_s3_bucket")
String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
// If all these variables are missing then we are testing against the internal fixture instead, which has the following
// credentials hard-coded in.
if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath
&& !s3EC2Bucket && !s3EC2BasePath
&& !s3ECSBucket && !s3ECSBasePath) {
s3PermanentAccessKey = 's3_integration_test_permanent_access_key'
s3PermanentSecretKey = 's3_integration_test_permanent_secret_key'
s3PermanentBucket = 'permanent-bucket-test'
s3PermanentBasePath = 'integration_test'
s3EC2Bucket = 'ec2-bucket-test'
s3EC2BasePath = 'integration_test'
s3ECSBucket = 'ecs-bucket-test'
s3ECSBasePath = 'integration_test'
useFixture = true
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
}
if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
s3TemporaryAccessKey = 's3_integration_test_temporary_access_key'
s3TemporarySecretKey = 's3_integration_test_temporary_secret_key'
s3TemporaryBucket = 'temporary-bucket-test'
s3TemporaryBasePath = 'integration_test'
s3TemporarySessionToken = 's3_integration_test_temporary_session_token'
} else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
}
final String minioVersion = 'RELEASE.2018-06-22T23-48-46Z'
final String minioBinDir = "${buildDir}/minio/bin"
final String minioDataDir = "${buildDir}/minio/data"
final String minioAddress = "127.0.0.1"
final String minioDistribution
final String minioCheckSum
if (Os.isFamily(Os.FAMILY_MAC)) {
minioDistribution = 'darwin-amd64'
minioCheckSum = '96b0bcb2f590e8e65fb83d5c3e221f9bd1106b49fa6f22c6b726b80b845d7c60'
} else if (Os.isFamily(Os.FAMILY_UNIX)) {
minioDistribution = 'linux-amd64'
minioCheckSum = '713dac7c105285eab3b92649be92b5e793b29d3525c7929fa7aaed99374fad99'
} else {
minioDistribution = null
minioCheckSum = null
}
buildscript {
repositories {
maven {
url 'https://plugins.gradle.org/m2/'
}
}
dependencies {
classpath 'de.undercouch:gradle-download-task:3.4.3'
}
}
if (useFixture && minioDistribution) {
apply plugin: 'de.undercouch.download'
final String minioFileName = "minio.${minioVersion}"
final String minioDownloadURL = "https://dl.minio.io/server/minio/release/${minioDistribution}/archive/${minioFileName}"
final String minioFilePath = "${gradle.gradleUserHomeDir}/downloads/minio/${minioDistribution}/${minioFileName}"
task downloadMinio(type: Download) {
src minioDownloadURL
dest minioFilePath
onlyIfModified true
}
task verifyMinioChecksum(type: Verify, dependsOn: downloadMinio) {
src minioFilePath
algorithm 'SHA-256'
checksum minioCheckSum
}
task installMinio(type: Sync, dependsOn: verifyMinioChecksum) {
from minioFilePath
into minioBinDir
fileMode 0755
}
task startMinio {
dependsOn installMinio
ext.minioPid = 0L
ext.minioPort = 0
doLast {
// get free port
for (int port = 60920; port < 60940; port++) {
try {
javax.net.ServerSocketFactory.getDefault().createServerSocket(port, 1, InetAddress.getByName(minioAddress)).close()
minioPort = port
break
} catch (BindException e) {
logger.info("Port " + port + " for Minio process is already taken", e)
}
}
if (minioPort == 0) {
throw new GradleException("Could not find a free port for Minio")
}
new File("${minioDataDir}/${s3PermanentBucket}").mkdirs()
// we skip these tests on Windows so we do no need to worry about compatibility here
final ProcessBuilder minio = new ProcessBuilder(
"${minioBinDir}/${minioFileName}",
"server",
"--address",
minioAddress + ":" + minioPort,
minioDataDir)
minio.environment().put('MINIO_ACCESS_KEY', s3PermanentAccessKey)
minio.environment().put('MINIO_SECRET_KEY', s3PermanentSecretKey)
final Process process = minio.start()
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
try {
Class<?> cProcessImpl = process.getClass()
Field fPid = cProcessImpl.getDeclaredField("pid")
if (!fPid.isAccessible()) {
fPid.setAccessible(true)
}
minioPid = fPid.getInt(process)
} catch (Exception e) {
logger.error("failed to read pid from minio process", e)
process.destroyForcibly()
throw e
}
} else {
minioPid = process.pid()
}
new BufferedReader(new InputStreamReader(process.getInputStream())).withReader { br ->
String line
int httpPort = 0
while ((line = br.readLine()) != null) {
logger.info(line)
if (line.matches('.*Endpoint.*:\\d+$')) {
assert httpPort == 0
final int index = line.lastIndexOf(":")
assert index >= 0
httpPort = Integer.parseInt(line.substring(index + 1))
assert httpPort == minioPort : "Port mismatch, expected ${minioPort} but was ${httpPort}"
final File script = new File(project.buildDir, "minio/minio.killer.sh")
script.setText(
["function shutdown {",
" kill ${minioPid}",
"}",
"trap shutdown EXIT",
// will wait indefinitely for input, but we never pass input, and the pipe is only closed when the build dies
"read line\n"].join('\n'), 'UTF-8')
final ProcessBuilder killer = new ProcessBuilder("bash", script.absolutePath)
killer.start()
break
}
}
assert httpPort > 0
}
}
}
task stopMinio(type: LoggedExec) {
onlyIf { startMinio.minioPid > 0 }
doFirst {
logger.info("Shutting down minio with pid ${startMinio.minioPid}")
}
final Object pid = "${ -> startMinio.minioPid }"
// we skip these tests on Windows so we do no need to worry about compatibility here
executable = 'kill'
args('-9', pid)
}
RestIntegTestTask integTestMinio = project.tasks.create('integTestMinio', RestIntegTestTask.class) {
description = "Runs REST tests using the Minio repository."
}
// The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
project.afterEvaluate {
ClusterConfiguration cluster = project.extensions.getByName('integTestMinioCluster') as ClusterConfiguration
cluster.dependsOn(project.bundlePlugin)
cluster.dependsOn(startMinio) // otherwise we don't know the Minio port
cluster.keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
cluster.keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
Closure<String> minioAddressAndPort = {
assert startMinio.minioPort > 0
return 'http://' + minioAddress + ':' + startMinio.minioPort
}
cluster.setting 's3.client.integration_test_permanent.endpoint', "${ -> minioAddressAndPort.call()}"
Task restIntegTestTask = project.tasks.getByName('integTestMinio')
restIntegTestTask.clusterConfig.plugin(project.path)
// Default jvm arguments for all test clusters
String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
" " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
" " + System.getProperty('tests.jvm.argline', '')
restIntegTestTask.clusterConfig.jvmArgs = jvmArgs
}
integTestMinioRunner.dependsOn(startMinio)
integTestMinioRunner.finalizedBy(stopMinio)
// Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
integTestMinioRunner.systemProperty 'tests.rest.blacklist', [
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
].join(",")
project.check.dependsOn(integTestMinio)
}
File parentFixtures = new File(project.buildDir, "fixtures")
File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties')
task s3FixtureProperties {
outputs.file(s3FixtureFile)
def s3FixtureOptions = [
"tests.seed" : project.testSeed,
"s3Fixture.permanent_bucket_name" : s3PermanentBucket,
"s3Fixture.permanent_key" : s3PermanentAccessKey,
"s3Fixture.temporary_bucket_name" : s3TemporaryBucket,
"s3Fixture.temporary_key" : s3TemporaryAccessKey,
"s3Fixture.temporary_session_token": s3TemporarySessionToken,
"s3Fixture.ec2_bucket_name" : s3EC2Bucket,
"s3Fixture.ecs_bucket_name" : s3ECSBucket
]
doLast {
file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n")
}
}
/** A task to start the AmazonS3Fixture which emulates an S3 service **/
task s3Fixture(type: AntFixture) {
dependsOn testClasses
dependsOn s3FixtureProperties
inputs.file(s3FixtureFile)
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
executable = new File(project.runtimeJavaHome, 'bin/java')
args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3FixtureFile.getAbsolutePath()
}
Map<String, Object> expansions = [
'permanent_bucket': s3PermanentBucket,
'permanent_base_path': s3PermanentBasePath,
'temporary_bucket': s3TemporaryBucket,
'temporary_base_path': s3TemporaryBasePath,
'ec2_bucket': s3EC2Bucket,
'ec2_base_path': s3EC2BasePath,
'ecs_bucket': s3ECSBucket,
'ecs_base_path': s3ECSBasePath
]
processTestResources {
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
project.afterEvaluate {
if (useFixture == false) {
// temporary_credentials, ec2_credentials and ecs_credentials are not ready for third-party-tests yet
integTestRunner.systemProperty 'tests.rest.blacklist',
[
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
].join(",")
}
}
integTestCluster {
keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
keystoreSetting 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
keystoreSetting 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
keystoreSetting 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
if (useFixture) {
dependsOn s3Fixture
/* Use a closure on the string to delay evaluation until tests are executed */
setting 's3.client.integration_test_permanent.endpoint', "http://${-> s3Fixture.addressAndPort}"
setting 's3.client.integration_test_temporary.endpoint', "http://${-> s3Fixture.addressAndPort}"
setting 's3.client.integration_test_ec2.endpoint', "http://${-> s3Fixture.addressAndPort}"
// to redirect InstanceProfileCredentialsProvider to custom auth point
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", "http://${-> s3Fixture.addressAndPort}"
} else {
println "Using an external service to test the repository-s3 plugin"
}
}
integTestRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
if (useFixture) {
RestIntegTestTask integTestECS = project.tasks.create('integTestECS', RestIntegTestTask.class) {
description = "Runs tests using the ECS repository."
}
// The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
project.afterEvaluate {
ClusterConfiguration cluster = project.extensions.getByName('integTestECSCluster') as ClusterConfiguration
cluster.dependsOn(project.s3Fixture)
cluster.setting 's3.client.integration_test_ecs.endpoint', "http://${-> s3Fixture.addressAndPort}"
Task integTestECSTask = project.tasks.getByName('integTestECS')
integTestECSTask.clusterConfig.plugin(project.path)
integTestECSTask.clusterConfig.environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI',
"http://${-> s3Fixture.addressAndPort}/ecs_credentials_endpoint"
integTestECSRunner.systemProperty 'tests.rest.blacklist', [
'repository_s3/10_basic/*',
'repository_s3/20_repository_permanent_credentials/*',
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*'
].join(",")
}
project.check.dependsOn(integTestECS)
}
thirdPartyAudit.excludes = [
// classes are missing
'javax.servlet.ServletContextEvent',
'javax.servlet.ServletContextListener',
'org.apache.avalon.framework.logger.Logger',
'org.apache.log.Hierarchy',
'org.apache.log.Logger',
'software.amazon.ion.IonReader',
'software.amazon.ion.IonSystem',
'software.amazon.ion.IonType',
'software.amazon.ion.IonWriter',
'software.amazon.ion.Timestamp',
'software.amazon.ion.system.IonBinaryWriterBuilder',
'software.amazon.ion.system.IonSystemBuilder',
'software.amazon.ion.system.IonTextWriterBuilder',
'software.amazon.ion.system.IonWriterBuilder',
]
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += [
'javax.xml.bind.Binder',
'javax.xml.bind.ContextFinder$1',
'javax.xml.bind.ContextFinder',
'javax.xml.bind.DataBindingException',
'javax.xml.bind.DatatypeConverter',
'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
'javax.xml.bind.DatatypeConverterImpl',
'javax.xml.bind.DatatypeConverterInterface',
'javax.xml.bind.Element',
'javax.xml.bind.GetPropertyAction',
'javax.xml.bind.JAXB$Cache',
'javax.xml.bind.JAXB',
'javax.xml.bind.JAXBContext',
'javax.xml.bind.JAXBElement$GlobalScope',
'javax.xml.bind.JAXBElement',
'javax.xml.bind.JAXBException',
'javax.xml.bind.JAXBIntrospector',
'javax.xml.bind.JAXBPermission',
'javax.xml.bind.MarshalException',
'javax.xml.bind.Marshaller$Listener',
'javax.xml.bind.Marshaller',
'javax.xml.bind.Messages',
'javax.xml.bind.NotIdentifiableEvent',
'javax.xml.bind.ParseConversionEvent',
'javax.xml.bind.PrintConversionEvent',
'javax.xml.bind.PropertyException',
'javax.xml.bind.SchemaOutputResolver',
'javax.xml.bind.TypeConstraintException',
'javax.xml.bind.UnmarshalException',
'javax.xml.bind.Unmarshaller$Listener',
'javax.xml.bind.Unmarshaller',
'javax.xml.bind.UnmarshallerHandler',
'javax.xml.bind.ValidationEvent',
'javax.xml.bind.ValidationEventHandler',
'javax.xml.bind.ValidationEventLocator',
'javax.xml.bind.ValidationException',
'javax.xml.bind.Validator',
'javax.xml.bind.WhiteSpaceProcessor',
'javax.xml.bind.annotation.DomHandler',
'javax.xml.bind.annotation.W3CDomHandler',
'javax.xml.bind.annotation.XmlAccessOrder',
'javax.xml.bind.annotation.XmlAccessType',
'javax.xml.bind.annotation.XmlAccessorOrder',
'javax.xml.bind.annotation.XmlAccessorType',
'javax.xml.bind.annotation.XmlAnyAttribute',
'javax.xml.bind.annotation.XmlAnyElement',
'javax.xml.bind.annotation.XmlAttachmentRef',
'javax.xml.bind.annotation.XmlAttribute',
'javax.xml.bind.annotation.XmlElement$DEFAULT',
'javax.xml.bind.annotation.XmlElement',
'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
'javax.xml.bind.annotation.XmlElementDecl',
'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
'javax.xml.bind.annotation.XmlElementRef',
'javax.xml.bind.annotation.XmlElementRefs',
'javax.xml.bind.annotation.XmlElementWrapper',
'javax.xml.bind.annotation.XmlElements',
'javax.xml.bind.annotation.XmlEnum',
'javax.xml.bind.annotation.XmlEnumValue',
'javax.xml.bind.annotation.XmlID',
'javax.xml.bind.annotation.XmlIDREF',
'javax.xml.bind.annotation.XmlInlineBinaryData',
'javax.xml.bind.annotation.XmlList',
'javax.xml.bind.annotation.XmlMimeType',
'javax.xml.bind.annotation.XmlMixed',
'javax.xml.bind.annotation.XmlNs',
'javax.xml.bind.annotation.XmlNsForm',
'javax.xml.bind.annotation.XmlRegistry',
'javax.xml.bind.annotation.XmlRootElement',
'javax.xml.bind.annotation.XmlSchema',
'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
'javax.xml.bind.annotation.XmlSchemaType',
'javax.xml.bind.annotation.XmlSchemaTypes',
'javax.xml.bind.annotation.XmlSeeAlso',
'javax.xml.bind.annotation.XmlTransient',
'javax.xml.bind.annotation.XmlType$DEFAULT',
'javax.xml.bind.annotation.XmlType',
'javax.xml.bind.annotation.XmlValue',
'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
'javax.xml.bind.annotation.adapters.XmlAdapter',
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
'javax.xml.bind.attachment.AttachmentMarshaller',
'javax.xml.bind.attachment.AttachmentUnmarshaller',
'javax.xml.bind.helpers.AbstractMarshallerImpl',
'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
'javax.xml.bind.helpers.DefaultValidationEventHandler',
'javax.xml.bind.helpers.Messages',
'javax.xml.bind.helpers.NotIdentifiableEventImpl',
'javax.xml.bind.helpers.ParseConversionEventImpl',
'javax.xml.bind.helpers.PrintConversionEventImpl',
'javax.xml.bind.helpers.ValidationEventImpl',
'javax.xml.bind.helpers.ValidationEventLocatorImpl',
'javax.xml.bind.util.JAXBResult',
'javax.xml.bind.util.JAXBSource$1',
'javax.xml.bind.util.JAXBSource',
'javax.xml.bind.util.Messages',
'javax.xml.bind.util.ValidationEventCollector'
]
} else {
thirdPartyAudit.excludes += ['javax.activation.DataHandler']
}
// AWS SDK is exposing some deprecated methods which we call using a delegate:
// * setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation)
// * changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass)
compileTestJava.options.compilerArgs << "-Xlint:-deprecation"