Update AWS SDK to 1.11.406 in repository-s3 (#30723)

plugins/repository-s3/build.gradle

@@ -32,19 +32,23 @@ esplugin {
 }
 
 versions << [
-  'aws': '1.11.223'
+  'aws': '1.11.406'
 ]
 
 dependencies {
   compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
   compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}"
   compile "com.amazonaws:aws-java-sdk-core:${versions.aws}"
+  compile "com.amazonaws:jmespath-java:${versions.aws}"
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
+  compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
   compile 'com.fasterxml.jackson.core:jackson-databind:2.6.7.1'
   compile 'com.fasterxml.jackson.core:jackson-annotations:2.6.0'
+  compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
+  compile 'joda-time:joda-time:2.10'
 
   // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
   // and whitelist this hack in JarHell 
@@ -53,6 +57,7 @@ dependencies {
 
 dependencyLicenses {
   mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
+  mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
   mapping from: /jackson-.*/, to: 'jackson'
   mapping from: /jaxb-.*/, to: 'jaxb'
 }

plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1

@@ -1 +0,0 @@
-c3993cb44f5856fa721b7b7ccfc266377c0bf9c0

plugins/repository-s3/licenses/aws-java-sdk-core-1.11.406.jar.sha1

@@ -0,0 +1 @@
+43f3b7332d4d527bbf34d4ac6be094f3dabec6de

plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1

@@ -1 +0,0 @@
-c24e6ebe108c60a08098aeaad5ae0b6a5a77b618

plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.406.jar.sha1

@@ -0,0 +1 @@
+e29854e58dc20f5453c1da7e580a5921b1e9714a

plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1

@@ -1 +0,0 @@
-c2ef96732e22d97952fbcd0a94f1dc376d157eda

plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.406.jar.sha1

@@ -0,0 +1 @@
+5c3c2c57b076602b3aeef841c63e5848ec52b00d

plugins/repository-s3/licenses/jmespath-java-1.11.406.jar.sha1

@@ -0,0 +1 @@
+06c291d1029943d4968a36fadffa3b71a6d8b4e4

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -23,10 +23,12 @@ import com.amazonaws.ClientConfiguration;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
+import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.http.IdleConnectionReaper;
 import com.amazonaws.internal.StaticCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.internal.Constants;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
@@ -93,19 +95,26 @@ class S3Service extends AbstractComponent implements Closeable {
         }
     }
 
-    private AmazonS3 buildClient(S3ClientSettings clientSettings) {
-        final AWSCredentialsProvider credentials = buildCredentials(logger, clientSettings);
-        final ClientConfiguration configuration = buildConfiguration(clientSettings);
-        final AmazonS3 client = buildClient(credentials, configuration);
-        if (Strings.hasText(clientSettings.endpoint)) {
-            client.setEndpoint(clientSettings.endpoint);
-        }
-        return client;
-    }
-
     // proxy for testing
-    AmazonS3 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) {
+    AmazonS3 buildClient(final S3ClientSettings clientSettings) {
-        return new AmazonS3Client(credentials, configuration);
+        final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
+        builder.withCredentials(buildCredentials(logger, clientSettings));
+        builder.withClientConfiguration(buildConfiguration(clientSettings));
+
+        final String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME;
+        logger.debug("using endpoint [{}]", endpoint);
+
+        // If the endpoint configuration isn't set on the builder then the default behaviour is to try
+        // and work out what region we are in and use an appropriate endpoint - see AwsClientBuilder#setRegion.
+        // In contrast, directly-constructed clients use s3.amazonaws.com unless otherwise instructed. We currently
+        // use a directly-constructed client, and need to keep the existing behaviour to avoid a breaking change,
+        // so to move to using the builder we must set it explicitly to keep the existing behaviour.
+        //
+        // We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too)
+        // so this change removes that usage of a deprecated API.
+        builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null));
+
+        return builder.build();
     }
 
     // pkg private for tests

plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java

@@ -19,7 +19,6 @@
 
 package org.elasticsearch.repositories.s3;
 
-import com.amazonaws.ClientConfiguration;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
@@ -70,9 +69,9 @@ public class RepositoryCredentialsTests extends ESTestCase {
             }
 
             @Override
-            AmazonS3 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) {
+            AmazonS3 buildClient(final S3ClientSettings clientSettings) {
-                final AmazonS3 client = super.buildClient(credentials, configuration);
+                final AmazonS3 client = super.buildClient(clientSettings);
-                return new ClientAndCredentials(client, credentials);
+                return new ClientAndCredentials(client, buildCredentials(logger, clientSettings));
             }
 
         }