2018-07-04 02:04:07 -04:00
|
|
|
import org.apache.tools.ant.taskdefs.condition.Os
|
|
|
|
import org.elasticsearch.gradle.LoggedExec
|
|
|
|
import org.elasticsearch.gradle.MavenFilteringHack
|
|
|
|
import org.elasticsearch.gradle.test.AntFixture
|
|
|
|
import org.elasticsearch.gradle.test.ClusterConfiguration
|
|
|
|
import org.elasticsearch.gradle.test.RestIntegTestTask
|
2018-10-24 09:05:50 -04:00
|
|
|
import com.carrotsearch.gradle.junit4.RandomizedTestingTask
|
2018-07-04 02:04:07 -04:00
|
|
|
|
|
|
|
import java.lang.reflect.Field
|
|
|
|
|
2015-10-29 14:40:19 -04:00
|
|
|
/*
|
|
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
|
|
* license agreements. See the NOTICE file distributed with
|
|
|
|
* this work for additional information regarding copyright
|
|
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
|
|
* not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing,
|
|
|
|
* software distributed under the License is distributed on an
|
|
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
* KIND, either express or implied. See the License for the
|
|
|
|
* specific language governing permissions and limitations
|
|
|
|
* under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
esplugin {
|
2016-05-20 02:56:06 -04:00
|
|
|
description 'The S3 repository plugin adds S3 repositories'
|
2017-04-04 18:15:25 -04:00
|
|
|
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
|
2015-10-29 14:40:19 -04:00
|
|
|
}
|
|
|
|
|
2015-11-10 17:58:46 -05:00
|
|
|
versions << [
|
2018-09-12 09:27:57 -04:00
|
|
|
'aws': '1.11.406'
|
2015-11-10 17:58:46 -05:00
|
|
|
]
|
|
|
|
|
2015-10-29 14:40:19 -04:00
|
|
|
dependencies {
|
2015-11-10 17:58:46 -05:00
|
|
|
compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
|
|
|
|
compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}"
|
|
|
|
compile "com.amazonaws:aws-java-sdk-core:${versions.aws}"
|
2018-09-12 09:27:57 -04:00
|
|
|
compile "com.amazonaws:jmespath-java:${versions.aws}"
|
2015-10-29 14:40:19 -04:00
|
|
|
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
|
2015-11-10 17:58:46 -05:00
|
|
|
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
|
|
|
|
compile "commons-logging:commons-logging:${versions.commonslogging}"
|
|
|
|
compile "commons-codec:commons-codec:${versions.commonscodec}"
|
2018-09-12 09:27:57 -04:00
|
|
|
compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
|
2017-11-13 12:05:14 -05:00
|
|
|
compile 'com.fasterxml.jackson.core:jackson-databind:2.6.7.1'
|
|
|
|
compile 'com.fasterxml.jackson.core:jackson-annotations:2.6.0'
|
2018-09-12 09:27:57 -04:00
|
|
|
compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
|
2018-11-12 04:02:41 -05:00
|
|
|
compile "joda-time:joda-time:${versions.joda}"
|
2016-05-20 02:56:06 -04:00
|
|
|
|
|
|
|
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
|
|
|
|
// and whitelist this hack in JarHell
|
|
|
|
compile 'javax.xml.bind:jaxb-api:2.2.2'
|
2015-10-29 14:40:19 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
dependencyLicenses {
|
|
|
|
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
|
2018-09-12 09:27:57 -04:00
|
|
|
mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
|
2015-10-29 14:40:19 -04:00
|
|
|
mapping from: /jackson-.*/, to: 'jackson'
|
2016-05-20 02:56:06 -04:00
|
|
|
mapping from: /jaxb-.*/, to: 'jaxb'
|
2015-10-29 14:40:19 -04:00
|
|
|
}
|
|
|
|
|
2016-09-03 06:41:07 -04:00
|
|
|
bundlePlugin {
|
|
|
|
from('config/repository-s3') {
|
|
|
|
into 'config'
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-24 09:05:50 -04:00
|
|
|
task testRepositoryCreds(type: RandomizedTestingTask) {
|
2018-06-18 02:42:11 -04:00
|
|
|
include '**/RepositoryCredentialsTests.class'
|
2018-09-19 16:18:54 -04:00
|
|
|
include '**/S3BlobStoreRepositoryTests.class'
|
2017-05-11 15:14:23 -04:00
|
|
|
systemProperty 'es.allow_insecure_settings', 'true'
|
2018-10-24 09:05:50 -04:00
|
|
|
classpath = tasks.test.classpath
|
|
|
|
testClassesDirs = tasks.test.testClassesDirs
|
2017-05-11 15:14:23 -04:00
|
|
|
}
|
2018-10-24 09:05:50 -04:00
|
|
|
project.check.dependsOn(testRepositoryCreds)
|
2017-05-11 15:14:23 -04:00
|
|
|
|
|
|
|
test {
|
|
|
|
// these are tested explicitly in separate test tasks
|
|
|
|
exclude '**/*CredentialsTests.class'
|
2018-09-19 16:18:54 -04:00
|
|
|
exclude '**/S3BlobStoreRepositoryTests.class'
|
2017-05-11 15:14:23 -04:00
|
|
|
}
|
|
|
|
|
2018-07-04 02:04:07 -04:00
|
|
|
boolean useFixture = false
|
|
|
|
|
|
|
|
// We test against two repositories, one which uses the usual two-part "permanent" credentials and
|
|
|
|
// the other which uses three-part "temporary" or "session" credentials.
|
|
|
|
|
|
|
|
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
|
|
|
|
String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
|
|
|
|
String s3PermanentBucket = System.getenv("amazon_s3_bucket")
|
|
|
|
String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
|
|
|
|
|
|
|
|
String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
|
|
|
|
String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
|
|
|
|
String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
|
|
|
|
String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
|
|
|
|
String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
|
|
|
|
|
2018-07-18 06:18:00 -04:00
|
|
|
String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
|
|
|
|
String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
|
|
|
|
|
2018-07-19 06:54:38 -04:00
|
|
|
String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
|
|
|
|
String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
|
|
|
|
|
2018-07-04 02:04:07 -04:00
|
|
|
// If all these variables are missing then we are testing against the internal fixture instead, which has the following
|
|
|
|
// credentials hard-coded in.
|
|
|
|
|
2018-07-18 06:18:00 -04:00
|
|
|
if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath
|
2018-07-19 06:54:38 -04:00
|
|
|
&& !s3EC2Bucket && !s3EC2BasePath
|
|
|
|
&& !s3ECSBucket && !s3ECSBasePath) {
|
2018-07-04 02:04:07 -04:00
|
|
|
s3PermanentAccessKey = 's3_integration_test_permanent_access_key'
|
|
|
|
s3PermanentSecretKey = 's3_integration_test_permanent_secret_key'
|
|
|
|
s3PermanentBucket = 'permanent-bucket-test'
|
|
|
|
s3PermanentBasePath = 'integration_test'
|
|
|
|
|
2018-07-18 06:18:00 -04:00
|
|
|
s3EC2Bucket = 'ec2-bucket-test'
|
|
|
|
s3EC2BasePath = 'integration_test'
|
|
|
|
|
2018-07-19 06:54:38 -04:00
|
|
|
s3ECSBucket = 'ecs-bucket-test'
|
|
|
|
s3ECSBasePath = 'integration_test'
|
|
|
|
|
2018-07-12 03:21:10 -04:00
|
|
|
useFixture = true
|
|
|
|
|
2018-07-24 08:09:41 -04:00
|
|
|
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
|
2018-07-25 10:34:30 -04:00
|
|
|
throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
|
2018-07-12 03:21:10 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
|
2018-07-04 02:04:07 -04:00
|
|
|
s3TemporaryAccessKey = 's3_integration_test_temporary_access_key'
|
|
|
|
s3TemporarySecretKey = 's3_integration_test_temporary_secret_key'
|
|
|
|
s3TemporaryBucket = 'temporary-bucket-test'
|
|
|
|
s3TemporaryBasePath = 'integration_test'
|
|
|
|
s3TemporarySessionToken = 's3_integration_test_temporary_session_token'
|
|
|
|
|
2018-07-12 03:21:10 -04:00
|
|
|
} else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
|
2018-07-25 10:34:30 -04:00
|
|
|
throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
|
2018-07-04 02:04:07 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
final String minioVersion = 'RELEASE.2018-06-22T23-48-46Z'
|
|
|
|
final String minioBinDir = "${buildDir}/minio/bin"
|
|
|
|
final String minioDataDir = "${buildDir}/minio/data"
|
2018-08-14 10:36:37 -04:00
|
|
|
final String minioAddress = "127.0.0.1"
|
2018-07-04 02:04:07 -04:00
|
|
|
|
|
|
|
final String minioDistribution
|
|
|
|
final String minioCheckSum
|
|
|
|
if (Os.isFamily(Os.FAMILY_MAC)) {
|
|
|
|
minioDistribution = 'darwin-amd64'
|
|
|
|
minioCheckSum = '96b0bcb2f590e8e65fb83d5c3e221f9bd1106b49fa6f22c6b726b80b845d7c60'
|
|
|
|
} else if (Os.isFamily(Os.FAMILY_UNIX)) {
|
|
|
|
minioDistribution = 'linux-amd64'
|
|
|
|
minioCheckSum = '713dac7c105285eab3b92649be92b5e793b29d3525c7929fa7aaed99374fad99'
|
|
|
|
} else {
|
|
|
|
minioDistribution = null
|
|
|
|
minioCheckSum = null
|
|
|
|
}
|
|
|
|
|
|
|
|
buildscript {
|
|
|
|
repositories {
|
|
|
|
maven {
|
|
|
|
url 'https://plugins.gradle.org/m2/'
|
|
|
|
}
|
|
|
|
}
|
|
|
|
dependencies {
|
|
|
|
classpath 'de.undercouch:gradle-download-task:3.4.3'
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (useFixture && minioDistribution) {
|
|
|
|
apply plugin: 'de.undercouch.download'
|
|
|
|
|
|
|
|
final String minioFileName = "minio.${minioVersion}"
|
|
|
|
final String minioDownloadURL = "https://dl.minio.io/server/minio/release/${minioDistribution}/archive/${minioFileName}"
|
|
|
|
final String minioFilePath = "${gradle.gradleUserHomeDir}/downloads/minio/${minioDistribution}/${minioFileName}"
|
|
|
|
|
|
|
|
task downloadMinio(type: Download) {
|
|
|
|
src minioDownloadURL
|
|
|
|
dest minioFilePath
|
|
|
|
onlyIfModified true
|
|
|
|
}
|
|
|
|
|
|
|
|
task verifyMinioChecksum(type: Verify, dependsOn: downloadMinio) {
|
|
|
|
src minioFilePath
|
|
|
|
algorithm 'SHA-256'
|
|
|
|
checksum minioCheckSum
|
|
|
|
}
|
|
|
|
|
|
|
|
task installMinio(type: Sync, dependsOn: verifyMinioChecksum) {
|
|
|
|
from minioFilePath
|
|
|
|
into minioBinDir
|
|
|
|
fileMode 0755
|
|
|
|
}
|
|
|
|
|
|
|
|
task startMinio {
|
|
|
|
dependsOn installMinio
|
|
|
|
|
|
|
|
ext.minioPid = 0L
|
2018-08-14 10:36:37 -04:00
|
|
|
ext.minioPort = 0
|
2018-07-04 02:04:07 -04:00
|
|
|
|
|
|
|
doLast {
|
2018-08-14 10:36:37 -04:00
|
|
|
// get free port
|
2018-10-26 06:55:21 -04:00
|
|
|
ServerSocket serverSocket = new ServerSocket(0, 1, InetAddress.getByName(minioAddress))
|
|
|
|
try {
|
|
|
|
minioPort = serverSocket.localPort
|
|
|
|
} finally {
|
|
|
|
serverSocket.close()
|
2018-08-14 10:36:37 -04:00
|
|
|
}
|
|
|
|
if (minioPort == 0) {
|
|
|
|
throw new GradleException("Could not find a free port for Minio")
|
|
|
|
}
|
|
|
|
|
2018-07-04 02:04:07 -04:00
|
|
|
new File("${minioDataDir}/${s3PermanentBucket}").mkdirs()
|
|
|
|
// we skip these tests on Windows so we do no need to worry about compatibility here
|
|
|
|
final ProcessBuilder minio = new ProcessBuilder(
|
|
|
|
"${minioBinDir}/${minioFileName}",
|
|
|
|
"server",
|
|
|
|
"--address",
|
2018-08-14 10:36:37 -04:00
|
|
|
minioAddress + ":" + minioPort,
|
2018-07-04 02:04:07 -04:00
|
|
|
minioDataDir)
|
|
|
|
minio.environment().put('MINIO_ACCESS_KEY', s3PermanentAccessKey)
|
|
|
|
minio.environment().put('MINIO_SECRET_KEY', s3PermanentSecretKey)
|
|
|
|
final Process process = minio.start()
|
|
|
|
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
|
|
|
|
try {
|
|
|
|
Class<?> cProcessImpl = process.getClass()
|
|
|
|
Field fPid = cProcessImpl.getDeclaredField("pid")
|
|
|
|
if (!fPid.isAccessible()) {
|
|
|
|
fPid.setAccessible(true)
|
|
|
|
}
|
|
|
|
minioPid = fPid.getInt(process)
|
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error("failed to read pid from minio process", e)
|
|
|
|
process.destroyForcibly()
|
|
|
|
throw e
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
minioPid = process.pid()
|
|
|
|
}
|
|
|
|
|
|
|
|
new BufferedReader(new InputStreamReader(process.getInputStream())).withReader { br ->
|
|
|
|
String line
|
|
|
|
int httpPort = 0
|
|
|
|
while ((line = br.readLine()) != null) {
|
|
|
|
logger.info(line)
|
|
|
|
if (line.matches('.*Endpoint.*:\\d+$')) {
|
|
|
|
assert httpPort == 0
|
|
|
|
final int index = line.lastIndexOf(":")
|
|
|
|
assert index >= 0
|
|
|
|
httpPort = Integer.parseInt(line.substring(index + 1))
|
2018-08-14 10:36:37 -04:00
|
|
|
assert httpPort == minioPort : "Port mismatch, expected ${minioPort} but was ${httpPort}"
|
2018-07-04 02:04:07 -04:00
|
|
|
|
|
|
|
final File script = new File(project.buildDir, "minio/minio.killer.sh")
|
|
|
|
script.setText(
|
|
|
|
["function shutdown {",
|
|
|
|
" kill ${minioPid}",
|
|
|
|
"}",
|
|
|
|
"trap shutdown EXIT",
|
|
|
|
// will wait indefinitely for input, but we never pass input, and the pipe is only closed when the build dies
|
|
|
|
"read line\n"].join('\n'), 'UTF-8')
|
|
|
|
final ProcessBuilder killer = new ProcessBuilder("bash", script.absolutePath)
|
|
|
|
killer.start()
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
assert httpPort > 0
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
task stopMinio(type: LoggedExec) {
|
|
|
|
onlyIf { startMinio.minioPid > 0 }
|
|
|
|
|
|
|
|
doFirst {
|
|
|
|
logger.info("Shutting down minio with pid ${startMinio.minioPid}")
|
|
|
|
}
|
|
|
|
|
|
|
|
final Object pid = "${ -> startMinio.minioPid }"
|
|
|
|
|
|
|
|
// we skip these tests on Windows so we do no need to worry about compatibility here
|
|
|
|
executable = 'kill'
|
|
|
|
args('-9', pid)
|
|
|
|
}
|
|
|
|
|
|
|
|
RestIntegTestTask integTestMinio = project.tasks.create('integTestMinio', RestIntegTestTask.class) {
|
|
|
|
description = "Runs REST tests using the Minio repository."
|
|
|
|
}
|
|
|
|
|
|
|
|
// The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
|
|
|
|
project.afterEvaluate {
|
|
|
|
ClusterConfiguration cluster = project.extensions.getByName('integTestMinioCluster') as ClusterConfiguration
|
|
|
|
cluster.dependsOn(project.bundlePlugin)
|
2018-08-14 10:36:37 -04:00
|
|
|
cluster.dependsOn(startMinio) // otherwise we don't know the Minio port
|
2018-07-04 02:04:07 -04:00
|
|
|
cluster.keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
|
|
|
|
cluster.keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
|
|
|
|
|
2018-08-14 10:36:37 -04:00
|
|
|
Closure<String> minioAddressAndPort = {
|
|
|
|
assert startMinio.minioPort > 0
|
|
|
|
return 'http://' + minioAddress + ':' + startMinio.minioPort
|
|
|
|
}
|
|
|
|
cluster.setting 's3.client.integration_test_permanent.endpoint', "${ -> minioAddressAndPort.call()}"
|
2018-07-04 02:04:07 -04:00
|
|
|
|
|
|
|
Task restIntegTestTask = project.tasks.getByName('integTestMinio')
|
|
|
|
restIntegTestTask.clusterConfig.plugin(project.path)
|
|
|
|
|
|
|
|
// Default jvm arguments for all test clusters
|
|
|
|
String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
|
|
|
|
" " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
|
|
|
|
" " + System.getProperty('tests.jvm.argline', '')
|
|
|
|
|
|
|
|
restIntegTestTask.clusterConfig.jvmArgs = jvmArgs
|
|
|
|
}
|
|
|
|
|
|
|
|
integTestMinioRunner.dependsOn(startMinio)
|
|
|
|
integTestMinioRunner.finalizedBy(stopMinio)
|
|
|
|
// Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
|
2018-07-18 06:18:00 -04:00
|
|
|
integTestMinioRunner.systemProperty 'tests.rest.blacklist', [
|
|
|
|
'repository_s3/30_repository_temporary_credentials/*',
|
2018-07-19 06:54:38 -04:00
|
|
|
'repository_s3/40_repository_ec2_credentials/*',
|
|
|
|
'repository_s3/50_repository_ecs_credentials/*'
|
2018-07-18 06:18:00 -04:00
|
|
|
].join(",")
|
2018-07-04 02:04:07 -04:00
|
|
|
|
|
|
|
project.check.dependsOn(integTestMinio)
|
|
|
|
}
|
|
|
|
|
2018-07-18 06:18:00 -04:00
|
|
|
File parentFixtures = new File(project.buildDir, "fixtures")
|
|
|
|
File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties')
|
|
|
|
|
|
|
|
task s3FixtureProperties {
|
|
|
|
outputs.file(s3FixtureFile)
|
|
|
|
def s3FixtureOptions = [
|
|
|
|
"tests.seed" : project.testSeed,
|
|
|
|
"s3Fixture.permanent_bucket_name" : s3PermanentBucket,
|
|
|
|
"s3Fixture.permanent_key" : s3PermanentAccessKey,
|
|
|
|
"s3Fixture.temporary_bucket_name" : s3TemporaryBucket,
|
|
|
|
"s3Fixture.temporary_key" : s3TemporaryAccessKey,
|
|
|
|
"s3Fixture.temporary_session_token": s3TemporarySessionToken,
|
2018-07-19 06:54:38 -04:00
|
|
|
"s3Fixture.ec2_bucket_name" : s3EC2Bucket,
|
|
|
|
"s3Fixture.ecs_bucket_name" : s3ECSBucket
|
2018-07-18 06:18:00 -04:00
|
|
|
]
|
|
|
|
|
|
|
|
doLast {
|
|
|
|
file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-07-04 02:04:07 -04:00
|
|
|
/** A task to start the AmazonS3Fixture which emulates an S3 service **/
|
|
|
|
task s3Fixture(type: AntFixture) {
|
|
|
|
dependsOn testClasses
|
2018-07-18 06:18:00 -04:00
|
|
|
dependsOn s3FixtureProperties
|
|
|
|
inputs.file(s3FixtureFile)
|
|
|
|
|
2018-07-04 02:04:07 -04:00
|
|
|
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
|
|
|
|
executable = new File(project.runtimeJavaHome, 'bin/java')
|
2018-07-18 06:18:00 -04:00
|
|
|
args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3FixtureFile.getAbsolutePath()
|
2018-07-04 02:04:07 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
Map<String, Object> expansions = [
|
|
|
|
'permanent_bucket': s3PermanentBucket,
|
|
|
|
'permanent_base_path': s3PermanentBasePath,
|
|
|
|
'temporary_bucket': s3TemporaryBucket,
|
2018-07-18 06:18:00 -04:00
|
|
|
'temporary_base_path': s3TemporaryBasePath,
|
|
|
|
'ec2_bucket': s3EC2Bucket,
|
2018-07-19 06:54:38 -04:00
|
|
|
'ec2_base_path': s3EC2BasePath,
|
|
|
|
'ecs_bucket': s3ECSBucket,
|
|
|
|
'ecs_base_path': s3ECSBasePath
|
2018-07-04 02:04:07 -04:00
|
|
|
]
|
|
|
|
|
|
|
|
processTestResources {
|
|
|
|
inputs.properties(expansions)
|
|
|
|
MavenFilteringHack.filter(it, expansions)
|
2018-04-03 05:30:43 -04:00
|
|
|
}
|
|
|
|
|
2018-07-12 03:21:10 -04:00
|
|
|
project.afterEvaluate {
|
|
|
|
if (useFixture == false) {
|
2018-07-24 08:09:41 -04:00
|
|
|
// temporary_credentials, ec2_credentials and ecs_credentials are not ready for third-party-tests yet
|
|
|
|
integTestRunner.systemProperty 'tests.rest.blacklist',
|
|
|
|
[
|
|
|
|
'repository_s3/30_repository_temporary_credentials/*',
|
|
|
|
'repository_s3/40_repository_ec2_credentials/*',
|
|
|
|
'repository_s3/50_repository_ecs_credentials/*'
|
|
|
|
].join(",")
|
2018-07-12 03:21:10 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-11 18:42:56 -04:00
|
|
|
integTestCluster {
|
2018-07-04 02:04:07 -04:00
|
|
|
keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
|
|
|
|
keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
|
|
|
|
|
|
|
|
keystoreSetting 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
|
|
|
|
keystoreSetting 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
|
|
|
|
keystoreSetting 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
|
|
|
|
|
|
|
|
if (useFixture) {
|
|
|
|
dependsOn s3Fixture
|
|
|
|
/* Use a closure on the string to delay evaluation until tests are executed */
|
|
|
|
setting 's3.client.integration_test_permanent.endpoint', "http://${-> s3Fixture.addressAndPort}"
|
|
|
|
setting 's3.client.integration_test_temporary.endpoint', "http://${-> s3Fixture.addressAndPort}"
|
2018-07-18 06:18:00 -04:00
|
|
|
setting 's3.client.integration_test_ec2.endpoint', "http://${-> s3Fixture.addressAndPort}"
|
|
|
|
|
|
|
|
// to redirect InstanceProfileCredentialsProvider to custom auth point
|
|
|
|
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", "http://${-> s3Fixture.addressAndPort}"
|
2018-07-04 02:04:07 -04:00
|
|
|
} else {
|
|
|
|
println "Using an external service to test the repository-s3 plugin"
|
|
|
|
}
|
2017-04-11 18:42:56 -04:00
|
|
|
}
|
|
|
|
|
2018-07-19 06:54:38 -04:00
|
|
|
integTestRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
|
|
|
|
|
2018-07-25 10:34:30 -04:00
|
|
|
if (useFixture) {
|
|
|
|
RestIntegTestTask integTestECS = project.tasks.create('integTestECS', RestIntegTestTask.class) {
|
|
|
|
description = "Runs tests using the ECS repository."
|
|
|
|
}
|
2018-07-19 06:54:38 -04:00
|
|
|
|
|
|
|
// The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
|
2018-07-25 10:34:30 -04:00
|
|
|
project.afterEvaluate {
|
|
|
|
ClusterConfiguration cluster = project.extensions.getByName('integTestECSCluster') as ClusterConfiguration
|
|
|
|
cluster.dependsOn(project.s3Fixture)
|
|
|
|
|
|
|
|
cluster.setting 's3.client.integration_test_ecs.endpoint', "http://${-> s3Fixture.addressAndPort}"
|
|
|
|
|
|
|
|
Task integTestECSTask = project.tasks.getByName('integTestECS')
|
|
|
|
integTestECSTask.clusterConfig.plugin(project.path)
|
|
|
|
integTestECSTask.clusterConfig.environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI',
|
|
|
|
"http://${-> s3Fixture.addressAndPort}/ecs_credentials_endpoint"
|
|
|
|
integTestECSRunner.systemProperty 'tests.rest.blacklist', [
|
|
|
|
'repository_s3/10_basic/*',
|
|
|
|
'repository_s3/20_repository_permanent_credentials/*',
|
|
|
|
'repository_s3/30_repository_temporary_credentials/*',
|
|
|
|
'repository_s3/40_repository_ec2_credentials/*'
|
|
|
|
].join(",")
|
|
|
|
}
|
|
|
|
project.check.dependsOn(integTestECS)
|
2018-07-19 06:54:38 -04:00
|
|
|
}
|
|
|
|
|
2015-12-16 16:38:16 -05:00
|
|
|
thirdPartyAudit.excludes = [
|
2015-12-28 22:38:55 -05:00
|
|
|
// classes are missing
|
|
|
|
'javax.servlet.ServletContextEvent',
|
|
|
|
'javax.servlet.ServletContextListener',
|
|
|
|
'org.apache.avalon.framework.logger.Logger',
|
|
|
|
'org.apache.log.Hierarchy',
|
|
|
|
'org.apache.log.Logger',
|
2017-11-09 07:25:51 -05:00
|
|
|
'software.amazon.ion.IonReader',
|
|
|
|
'software.amazon.ion.IonSystem',
|
|
|
|
'software.amazon.ion.IonType',
|
|
|
|
'software.amazon.ion.IonWriter',
|
|
|
|
'software.amazon.ion.Timestamp',
|
|
|
|
'software.amazon.ion.system.IonBinaryWriterBuilder',
|
|
|
|
'software.amazon.ion.system.IonSystemBuilder',
|
|
|
|
'software.amazon.ion.system.IonTextWriterBuilder',
|
|
|
|
'software.amazon.ion.system.IonWriterBuilder',
|
2015-12-16 16:38:16 -05:00
|
|
|
]
|
Upgrade AWS SDK to 1.10.69
* Moving from JSON.org to Jackson for request marshallers.
* The Java SDK now supports retry throttling to limit the rate of retries during periods of reduced availability. This throttling behavior can be enabled via ClientConfiguration or via the system property "-Dcom.amazonaws.sdk.enableThrottledRetry".
* Fixed String case conversion issues when running with non English locales.
* AWS SDK for Java introduces a new dynamic endpoint system that can compute endpoints for services in new regions.
* Introducing a new AWS region, ap-northeast-2.
* Added a new metric, HttpSocketReadTime, that records socket read latency. You can enable this metric by adding enableHttpSocketReadMetric to the system property com.amazonaws.sdk.enableDefaultMetrics. For more information, see [Enabling Metrics with the AWS SDK for Java](https://java.awsblog.com/post/Tx3C0RV4NRRBKTG/Enabling-Metrics-with-the-AWS-SDK-for-Java).
* New Client Execution timeout feature to set a limit spent across retries, backoffs, ummarshalling, etc. This new timeout can be specified at the client level or per request.
Also included in this release is the ability to specify the existing HTTP Request timeout per request rather than just per client.
* Added support for RequesterPays for all operations.
* Ignore the 'Connection' header when generating S3 responses.
* Allow users to generate an AmazonS3URI from a string without using URL encoding.
* Fixed issue that prevented creating buckets when using a client configured for the s3-external-1 endpoint.
* Amazon S3 bucket lifecycle configuration supports two new features: the removal of expired object delete markers and an action to abort incomplete multipart uploads.
* Allow TransferManagerConfiguration to accept integer values for multipart upload threshold.
* Copy the list of ETags before sorting https://github.com/aws/aws-sdk-java/pull/589.
* Option to disable chunked encoding https://github.com/aws/aws-sdk-java/pull/586.
* Adding retry on InternalErrors in CompleteMultipartUpload operation. https://github.com/aws/aws-sdk-java/issues/538
* Deprecated two APIs : AmazonS3#changeObjectStorageClass and AmazonS3#setObjectRedirectLocation.
* Added support for the aws-exec-read canned ACL. Owner gets FULL_CONTROL. Amazon EC2 gets READ access to GET an Amazon Machine Image (AMI) bundle from Amazon S3.
* Added support for referencing security groups in peered Virtual Private Clouds (VPCs). For more information see the service announcement at https://aws.amazon.com/about-aws/whats-new/2016/03/announcing-support-for-security-group-references-in-a-peered-vpc/ .
* Fixed a bug in AWS SDK for Java - Amazon EC2 module that returns NPE for dry run requests.
* Regenerated client with new implementation of code generator.
* This feature enables support for DNS resolution of public hostnames to private IP addresses when queried over ClassicLink. Additionally, you can now access private hosted zones associated with your VPC from a linked EC2-Classic instance. ClassicLink DNS support makes it easier for EC2-Classic instances to communicate with VPC resources using public DNS hostnames.
* You can now use Network Address Translation (NAT) Gateway, a highly available AWS managed service that makes it easy to connect to the Internet from instances within a private subnet in an AWS Virtual Private Cloud (VPC). Previously, you needed to launch a NAT instance to enable NAT for instances in a private subnet. Amazon VPC NAT Gateway is available in the US East (N. Virginia), US West (Oregon), US West (N. California), EU (Ireland), Asia Pacific (Tokyo), Asia Pacific (Singapore), and Asia Pacific (Sydney) regions. To learn more about Amazon VPC NAT, see [New - Managed NAT (Network Address Translation) Gateway for AWS](https://aws.amazon.com/blogs/aws/new-managed-nat-network-address-translation-gateway-for-aws/)
* A default read timeout is now applied when querying data from EC2 metadata service.
2016-04-15 05:51:47 -04:00
|
|
|
|
2017-07-27 10:14:04 -04:00
|
|
|
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
|
2018-08-28 03:03:30 -04:00
|
|
|
if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
|
2017-07-27 10:14:04 -04:00
|
|
|
thirdPartyAudit.excludes += [
|
|
|
|
'javax.xml.bind.Binder',
|
|
|
|
'javax.xml.bind.ContextFinder$1',
|
|
|
|
'javax.xml.bind.ContextFinder',
|
|
|
|
'javax.xml.bind.DataBindingException',
|
|
|
|
'javax.xml.bind.DatatypeConverter',
|
|
|
|
'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
|
|
|
|
'javax.xml.bind.DatatypeConverterImpl',
|
|
|
|
'javax.xml.bind.DatatypeConverterInterface',
|
|
|
|
'javax.xml.bind.Element',
|
|
|
|
'javax.xml.bind.GetPropertyAction',
|
|
|
|
'javax.xml.bind.JAXB$Cache',
|
|
|
|
'javax.xml.bind.JAXB',
|
|
|
|
'javax.xml.bind.JAXBContext',
|
|
|
|
'javax.xml.bind.JAXBElement$GlobalScope',
|
|
|
|
'javax.xml.bind.JAXBElement',
|
|
|
|
'javax.xml.bind.JAXBException',
|
|
|
|
'javax.xml.bind.JAXBIntrospector',
|
|
|
|
'javax.xml.bind.JAXBPermission',
|
|
|
|
'javax.xml.bind.MarshalException',
|
|
|
|
'javax.xml.bind.Marshaller$Listener',
|
|
|
|
'javax.xml.bind.Marshaller',
|
|
|
|
'javax.xml.bind.Messages',
|
|
|
|
'javax.xml.bind.NotIdentifiableEvent',
|
|
|
|
'javax.xml.bind.ParseConversionEvent',
|
|
|
|
'javax.xml.bind.PrintConversionEvent',
|
|
|
|
'javax.xml.bind.PropertyException',
|
|
|
|
'javax.xml.bind.SchemaOutputResolver',
|
|
|
|
'javax.xml.bind.TypeConstraintException',
|
|
|
|
'javax.xml.bind.UnmarshalException',
|
|
|
|
'javax.xml.bind.Unmarshaller$Listener',
|
|
|
|
'javax.xml.bind.Unmarshaller',
|
|
|
|
'javax.xml.bind.UnmarshallerHandler',
|
|
|
|
'javax.xml.bind.ValidationEvent',
|
|
|
|
'javax.xml.bind.ValidationEventHandler',
|
|
|
|
'javax.xml.bind.ValidationEventLocator',
|
|
|
|
'javax.xml.bind.ValidationException',
|
|
|
|
'javax.xml.bind.Validator',
|
|
|
|
'javax.xml.bind.WhiteSpaceProcessor',
|
|
|
|
'javax.xml.bind.annotation.DomHandler',
|
|
|
|
'javax.xml.bind.annotation.W3CDomHandler',
|
|
|
|
'javax.xml.bind.annotation.XmlAccessOrder',
|
|
|
|
'javax.xml.bind.annotation.XmlAccessType',
|
|
|
|
'javax.xml.bind.annotation.XmlAccessorOrder',
|
|
|
|
'javax.xml.bind.annotation.XmlAccessorType',
|
|
|
|
'javax.xml.bind.annotation.XmlAnyAttribute',
|
|
|
|
'javax.xml.bind.annotation.XmlAnyElement',
|
|
|
|
'javax.xml.bind.annotation.XmlAttachmentRef',
|
|
|
|
'javax.xml.bind.annotation.XmlAttribute',
|
|
|
|
'javax.xml.bind.annotation.XmlElement$DEFAULT',
|
|
|
|
'javax.xml.bind.annotation.XmlElement',
|
|
|
|
'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
|
|
|
|
'javax.xml.bind.annotation.XmlElementDecl',
|
|
|
|
'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
|
|
|
|
'javax.xml.bind.annotation.XmlElementRef',
|
|
|
|
'javax.xml.bind.annotation.XmlElementRefs',
|
|
|
|
'javax.xml.bind.annotation.XmlElementWrapper',
|
|
|
|
'javax.xml.bind.annotation.XmlElements',
|
|
|
|
'javax.xml.bind.annotation.XmlEnum',
|
|
|
|
'javax.xml.bind.annotation.XmlEnumValue',
|
|
|
|
'javax.xml.bind.annotation.XmlID',
|
|
|
|
'javax.xml.bind.annotation.XmlIDREF',
|
|
|
|
'javax.xml.bind.annotation.XmlInlineBinaryData',
|
|
|
|
'javax.xml.bind.annotation.XmlList',
|
|
|
|
'javax.xml.bind.annotation.XmlMimeType',
|
|
|
|
'javax.xml.bind.annotation.XmlMixed',
|
|
|
|
'javax.xml.bind.annotation.XmlNs',
|
|
|
|
'javax.xml.bind.annotation.XmlNsForm',
|
|
|
|
'javax.xml.bind.annotation.XmlRegistry',
|
|
|
|
'javax.xml.bind.annotation.XmlRootElement',
|
|
|
|
'javax.xml.bind.annotation.XmlSchema',
|
|
|
|
'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
|
|
|
|
'javax.xml.bind.annotation.XmlSchemaType',
|
|
|
|
'javax.xml.bind.annotation.XmlSchemaTypes',
|
|
|
|
'javax.xml.bind.annotation.XmlSeeAlso',
|
|
|
|
'javax.xml.bind.annotation.XmlTransient',
|
|
|
|
'javax.xml.bind.annotation.XmlType$DEFAULT',
|
|
|
|
'javax.xml.bind.annotation.XmlType',
|
|
|
|
'javax.xml.bind.annotation.XmlValue',
|
|
|
|
'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
|
|
|
|
'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
|
|
|
|
'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
|
|
|
|
'javax.xml.bind.annotation.adapters.XmlAdapter',
|
|
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
|
|
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
|
|
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
|
|
|
|
'javax.xml.bind.attachment.AttachmentMarshaller',
|
|
|
|
'javax.xml.bind.attachment.AttachmentUnmarshaller',
|
|
|
|
'javax.xml.bind.helpers.AbstractMarshallerImpl',
|
|
|
|
'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
|
|
|
|
'javax.xml.bind.helpers.DefaultValidationEventHandler',
|
|
|
|
'javax.xml.bind.helpers.Messages',
|
|
|
|
'javax.xml.bind.helpers.NotIdentifiableEventImpl',
|
|
|
|
'javax.xml.bind.helpers.ParseConversionEventImpl',
|
|
|
|
'javax.xml.bind.helpers.PrintConversionEventImpl',
|
|
|
|
'javax.xml.bind.helpers.ValidationEventImpl',
|
|
|
|
'javax.xml.bind.helpers.ValidationEventLocatorImpl',
|
|
|
|
'javax.xml.bind.util.JAXBResult',
|
|
|
|
'javax.xml.bind.util.JAXBSource$1',
|
|
|
|
'javax.xml.bind.util.JAXBSource',
|
|
|
|
'javax.xml.bind.util.Messages',
|
|
|
|
'javax.xml.bind.util.ValidationEventCollector'
|
|
|
|
]
|
|
|
|
} else {
|
|
|
|
thirdPartyAudit.excludes += ['javax.activation.DataHandler']
|
|
|
|
}
|
|
|
|
|
2016-05-27 04:13:16 -04:00
|
|
|
// AWS SDK is exposing some deprecated methods which we call using a delegate:
|
|
|
|
// * setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation)
|
|
|
|
// * changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass)
|
Upgrade AWS SDK to 1.10.69
* Moving from JSON.org to Jackson for request marshallers.
* The Java SDK now supports retry throttling to limit the rate of retries during periods of reduced availability. This throttling behavior can be enabled via ClientConfiguration or via the system property "-Dcom.amazonaws.sdk.enableThrottledRetry".
* Fixed String case conversion issues when running with non English locales.
* AWS SDK for Java introduces a new dynamic endpoint system that can compute endpoints for services in new regions.
* Introducing a new AWS region, ap-northeast-2.
* Added a new metric, HttpSocketReadTime, that records socket read latency. You can enable this metric by adding enableHttpSocketReadMetric to the system property com.amazonaws.sdk.enableDefaultMetrics. For more information, see [Enabling Metrics with the AWS SDK for Java](https://java.awsblog.com/post/Tx3C0RV4NRRBKTG/Enabling-Metrics-with-the-AWS-SDK-for-Java).
* New Client Execution timeout feature to set a limit spent across retries, backoffs, ummarshalling, etc. This new timeout can be specified at the client level or per request.
Also included in this release is the ability to specify the existing HTTP Request timeout per request rather than just per client.
* Added support for RequesterPays for all operations.
* Ignore the 'Connection' header when generating S3 responses.
* Allow users to generate an AmazonS3URI from a string without using URL encoding.
* Fixed issue that prevented creating buckets when using a client configured for the s3-external-1 endpoint.
* Amazon S3 bucket lifecycle configuration supports two new features: the removal of expired object delete markers and an action to abort incomplete multipart uploads.
* Allow TransferManagerConfiguration to accept integer values for multipart upload threshold.
* Copy the list of ETags before sorting https://github.com/aws/aws-sdk-java/pull/589.
* Option to disable chunked encoding https://github.com/aws/aws-sdk-java/pull/586.
* Adding retry on InternalErrors in CompleteMultipartUpload operation. https://github.com/aws/aws-sdk-java/issues/538
* Deprecated two APIs : AmazonS3#changeObjectStorageClass and AmazonS3#setObjectRedirectLocation.
* Added support for the aws-exec-read canned ACL. Owner gets FULL_CONTROL. Amazon EC2 gets READ access to GET an Amazon Machine Image (AMI) bundle from Amazon S3.
* Added support for referencing security groups in peered Virtual Private Clouds (VPCs). For more information see the service announcement at https://aws.amazon.com/about-aws/whats-new/2016/03/announcing-support-for-security-group-references-in-a-peered-vpc/ .
* Fixed a bug in AWS SDK for Java - Amazon EC2 module that returns NPE for dry run requests.
* Regenerated client with new implementation of code generator.
* This feature enables support for DNS resolution of public hostnames to private IP addresses when queried over ClassicLink. Additionally, you can now access private hosted zones associated with your VPC from a linked EC2-Classic instance. ClassicLink DNS support makes it easier for EC2-Classic instances to communicate with VPC resources using public DNS hostnames.
* You can now use Network Address Translation (NAT) Gateway, a highly available AWS managed service that makes it easy to connect to the Internet from instances within a private subnet in an AWS Virtual Private Cloud (VPC). Previously, you needed to launch a NAT instance to enable NAT for instances in a private subnet. Amazon VPC NAT Gateway is available in the US East (N. Virginia), US West (Oregon), US West (N. California), EU (Ireland), Asia Pacific (Tokyo), Asia Pacific (Singapore), and Asia Pacific (Sydney) regions. To learn more about Amazon VPC NAT, see [New - Managed NAT (Network Address Translation) Gateway for AWS](https://aws.amazon.com/blogs/aws/new-managed-nat-network-address-translation-gateway-for-aws/)
* A default read timeout is now applied when querying data from EC2 metadata service.
2016-04-15 05:51:47 -04:00
|
|
|
compileTestJava.options.compilerArgs << "-Xlint:-deprecation"
|