From 7e195c2912b9ed9fad3099761562dd0a7731635c Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 12 Sep 2018 15:27:57 +0200 Subject: [PATCH] Update AWS SDK to 1.11.406 in repository-s3 (#30723) --- plugins/repository-s3/build.gradle | 7 +++- .../aws-java-sdk-core-1.11.223.jar.sha1 | 1 - .../aws-java-sdk-core-1.11.406.jar.sha1 | 1 + .../aws-java-sdk-kms-1.11.223.jar.sha1 | 1 - .../aws-java-sdk-kms-1.11.406.jar.sha1 | 1 + .../aws-java-sdk-s3-1.11.223.jar.sha1 | 1 - .../aws-java-sdk-s3-1.11.406.jar.sha1 | 1 + .../licenses/jmespath-java-1.11.406.jar.sha1 | 1 + .../repositories/s3/S3Service.java | 35 ++++++++++++------- .../s3/RepositoryCredentialsTests.java | 7 ++-- 10 files changed, 35 insertions(+), 21 deletions(-) delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.406.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.406.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.406.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.406.jar.sha1 diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 5d248b22caf..c56a9a8259a 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -32,19 +32,23 @@ esplugin { } versions << [ - 'aws': '1.11.223' + 'aws': '1.11.406' ] dependencies { compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}" compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}" compile "com.amazonaws:aws-java-sdk-core:${versions.aws}" + compile "com.amazonaws:jmespath-java:${versions.aws}" compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" compile "commons-logging:commons-logging:${versions.commonslogging}" compile "commons-codec:commons-codec:${versions.commonscodec}" + compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" compile 'com.fasterxml.jackson.core:jackson-databind:2.6.7.1' compile 'com.fasterxml.jackson.core:jackson-annotations:2.6.0' + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + compile 'joda-time:joda-time:2.10' // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here, // and whitelist this hack in JarHell @@ -53,6 +57,7 @@ dependencies { dependencyLicenses { mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk' + mapping from: /jmespath-java.*/, to: 'aws-java-sdk' mapping from: /jackson-.*/, to: 'jackson' mapping from: /jaxb-.*/, to: 'jaxb' } diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 deleted file mode 100644 index 9890dd8d600..00000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c3993cb44f5856fa721b7b7ccfc266377c0bf9c0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.406.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.406.jar.sha1 new file mode 100644 index 00000000000..415373b275e --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.406.jar.sha1 @@ -0,0 +1 @@ +43f3b7332d4d527bbf34d4ac6be094f3dabec6de \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 deleted file mode 100644 index d5bc9d30308..00000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c24e6ebe108c60a08098aeaad5ae0b6a5a77b618 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.406.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.406.jar.sha1 new file mode 100644 index 00000000000..f0eb9b71752 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.406.jar.sha1 @@ -0,0 +1 @@ +e29854e58dc20f5453c1da7e580a5921b1e9714a \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 deleted file mode 100644 index fe12b2d4847..00000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c2ef96732e22d97952fbcd0a94f1dc376d157eda \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.406.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.406.jar.sha1 new file mode 100644 index 00000000000..e57fd11c829 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.406.jar.sha1 @@ -0,0 +1 @@ +5c3c2c57b076602b3aeef841c63e5848ec52b00d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.406.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.406.jar.sha1 new file mode 100644 index 00000000000..bbb9b562a2f --- /dev/null +++ b/plugins/repository-s3/licenses/jmespath-java-1.11.406.jar.sha1 @@ -0,0 +1 @@ +06c291d1029943d4968a36fadffa3b71a6d8b4e4 \ No newline at end of file diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index b177686bd71..a431f4da1fd 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -23,10 +23,12 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; +import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.internal.Constants; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; @@ -93,19 +95,26 @@ class S3Service extends AbstractComponent implements Closeable { } } - private AmazonS3 buildClient(S3ClientSettings clientSettings) { - final AWSCredentialsProvider credentials = buildCredentials(logger, clientSettings); - final ClientConfiguration configuration = buildConfiguration(clientSettings); - final AmazonS3 client = buildClient(credentials, configuration); - if (Strings.hasText(clientSettings.endpoint)) { - client.setEndpoint(clientSettings.endpoint); - } - return client; - } - // proxy for testing - AmazonS3 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) { - return new AmazonS3Client(credentials, configuration); + AmazonS3 buildClient(final S3ClientSettings clientSettings) { + final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); + builder.withCredentials(buildCredentials(logger, clientSettings)); + builder.withClientConfiguration(buildConfiguration(clientSettings)); + + final String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME; + logger.debug("using endpoint [{}]", endpoint); + + // If the endpoint configuration isn't set on the builder then the default behaviour is to try + // and work out what region we are in and use an appropriate endpoint - see AwsClientBuilder#setRegion. + // In contrast, directly-constructed clients use s3.amazonaws.com unless otherwise instructed. We currently + // use a directly-constructed client, and need to keep the existing behaviour to avoid a breaking change, + // so to move to using the builder we must set it explicitly to keep the existing behaviour. + // + // We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too) + // so this change removes that usage of a deprecated API. + builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null)); + + return builder.build(); } // pkg private for tests diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java index 7eb603b4b78..17797a57583 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.repositories.s3; -import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; @@ -70,9 +69,9 @@ public class RepositoryCredentialsTests extends ESTestCase { } @Override - AmazonS3 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) { - final AmazonS3 client = super.buildClient(credentials, configuration); - return new ClientAndCredentials(client, credentials); + AmazonS3 buildClient(final S3ClientSettings clientSettings) { + final AmazonS3 client = super.buildClient(clientSettings); + return new ClientAndCredentials(client, buildCredentials(logger, clientSettings)); } }