Merge branch 'pr/update-aws-sdk'

# Conflicts:
#	plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java

docs/plugins/repository-s3.asciidoc

@@ -39,7 +39,8 @@ http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
 credentials for authentication. These can be overridden by, in increasing
 order of precedence, system properties `aws.accessKeyId` and `aws.secretKey`,
 environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_KEY`, or the
-elasticsearch config using `cloud.aws.access_key` and `cloud.aws.secret_key`:
+elasticsearch config using `cloud.aws.access_key` and `cloud.aws.secret_key` or
+if you wish to set credentials specifically for s3 `cloud.aws.s3.access_key` and `cloud.aws.s3.secret_key`:
 
 [source,yaml]
 ----
@@ -106,6 +107,7 @@ cloud:
 ===== Region
 
 The `cloud.aws.region` can be set to a region and will automatically use the relevant settings for both `ec2` and `s3`.
+You can specifically set it for s3 only using `cloud.aws.s3.region`.
 The available values are:
 
 * `us-east` (`us-east-1`)
@@ -217,9 +219,13 @@ The following settings are supported:
 
     Number of retries in case of S3 errors. Defaults to `3`.
 
+`use_throttle_retries`::
+
+    Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
+
 `read_only`::
 
-    Makes repository read-only. coming[2.1.0]  Defaults to `false`.
+    Makes repository read-only. Defaults to `false`.
 
 `canned_acl`::
 
@@ -237,6 +243,10 @@ The following settings are supported:
     currently supported by the plugin. For more information about the
     different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide]
 
+Note that you can define S3 repository settings for all S3 repositories in `elasticsearch.yml` configuration file.
+They are all prefixed with `repositories.s3.`. For example, you can define compression for all S3 repositories
+by setting `repositories.s3.compress: true` in `elasticsearch.yml`.
+
 The S3 repositories use the same credentials as the rest of the AWS services
 provided by this plugin (`discovery`). See <<repository-s3-usage>> for details.
 

plugins/discovery-ec2/build.gradle

@@ -23,7 +23,7 @@ esplugin {
 }
 
 versions << [
-  'aws': '1.10.33'
+  'aws': '1.10.69'
 ]
 
 dependencies {
@@ -48,11 +48,6 @@ test {
 }
 
 thirdPartyAudit.excludes = [
-  // uses internal java api: com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl
-  // uses internal java api: com.sun.org.apache.xml.internal.dtm.ref.DTMManagerDefault
-  // uses internal java api: com.sun.org.apache.xpath.internal.XPathContext
-  'com.amazonaws.util.XpathUtils',
-
   // classes are missing
   'javax.servlet.ServletContextEvent', 
   'javax.servlet.ServletContextListener', 

plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1

@@ -1 +0,0 @@
-fabedbbe2b834b1add150b6a38395c5ef7380168

plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.69.jar.sha1

@@ -0,0 +1 @@
+a1f02d5f26ba1d8c37e2bf9c847db3c6729dda00

plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1

@@ -1 +0,0 @@
-202f6b5dbc196e355d50c131b0fd34969bfd89e6

plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.69.jar.sha1

@@ -0,0 +1 @@
+afbff1ece8365859eb4cfe0d3ba543d68b154d26

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java

@@ -29,6 +29,8 @@ import com.amazonaws.services.ec2.model.AcceptVpcPeeringConnectionRequest;
 import com.amazonaws.services.ec2.model.AcceptVpcPeeringConnectionResult;
 import com.amazonaws.services.ec2.model.AllocateAddressRequest;
 import com.amazonaws.services.ec2.model.AllocateAddressResult;
+import com.amazonaws.services.ec2.model.AllocateHostsRequest;
+import com.amazonaws.services.ec2.model.AllocateHostsResult;
 import com.amazonaws.services.ec2.model.AssignPrivateIpAddressesRequest;
 import com.amazonaws.services.ec2.model.AssociateAddressRequest;
 import com.amazonaws.services.ec2.model.AssociateAddressResult;
@@ -80,6 +82,8 @@ import com.amazonaws.services.ec2.model.CreateInternetGatewayRequest;
 import com.amazonaws.services.ec2.model.CreateInternetGatewayResult;
 import com.amazonaws.services.ec2.model.CreateKeyPairRequest;
 import com.amazonaws.services.ec2.model.CreateKeyPairResult;
+import com.amazonaws.services.ec2.model.CreateNatGatewayRequest;
+import com.amazonaws.services.ec2.model.CreateNatGatewayResult;
 import com.amazonaws.services.ec2.model.CreateNetworkAclEntryRequest;
 import com.amazonaws.services.ec2.model.CreateNetworkAclRequest;
 import com.amazonaws.services.ec2.model.CreateNetworkAclResult;
@@ -120,6 +124,8 @@ import com.amazonaws.services.ec2.model.DeleteFlowLogsRequest;
 import com.amazonaws.services.ec2.model.DeleteFlowLogsResult;
 import com.amazonaws.services.ec2.model.DeleteInternetGatewayRequest;
 import com.amazonaws.services.ec2.model.DeleteKeyPairRequest;
+import com.amazonaws.services.ec2.model.DeleteNatGatewayRequest;
+import com.amazonaws.services.ec2.model.DeleteNatGatewayResult;
 import com.amazonaws.services.ec2.model.DeleteNetworkAclEntryRequest;
 import com.amazonaws.services.ec2.model.DeleteNetworkAclRequest;
 import com.amazonaws.services.ec2.model.DeleteNetworkInterfaceRequest;
@@ -161,6 +167,10 @@ import com.amazonaws.services.ec2.model.DescribeExportTasksRequest;
 import com.amazonaws.services.ec2.model.DescribeExportTasksResult;
 import com.amazonaws.services.ec2.model.DescribeFlowLogsRequest;
 import com.amazonaws.services.ec2.model.DescribeFlowLogsResult;
+import com.amazonaws.services.ec2.model.DescribeHostsRequest;
+import com.amazonaws.services.ec2.model.DescribeHostsResult;
+import com.amazonaws.services.ec2.model.DescribeIdFormatRequest;
+import com.amazonaws.services.ec2.model.DescribeIdFormatResult;
 import com.amazonaws.services.ec2.model.DescribeImageAttributeRequest;
 import com.amazonaws.services.ec2.model.DescribeImageAttributeResult;
 import com.amazonaws.services.ec2.model.DescribeImagesRequest;
@@ -181,6 +191,8 @@ import com.amazonaws.services.ec2.model.DescribeKeyPairsRequest;
 import com.amazonaws.services.ec2.model.DescribeKeyPairsResult;
 import com.amazonaws.services.ec2.model.DescribeMovingAddressesRequest;
 import com.amazonaws.services.ec2.model.DescribeMovingAddressesResult;
+import com.amazonaws.services.ec2.model.DescribeNatGatewaysRequest;
+import com.amazonaws.services.ec2.model.DescribeNatGatewaysResult;
 import com.amazonaws.services.ec2.model.DescribeNetworkAclsRequest;
 import com.amazonaws.services.ec2.model.DescribeNetworkAclsResult;
 import com.amazonaws.services.ec2.model.DescribeNetworkInterfaceAttributeRequest;
@@ -203,6 +215,10 @@ import com.amazonaws.services.ec2.model.DescribeReservedInstancesRequest;
 import com.amazonaws.services.ec2.model.DescribeReservedInstancesResult;
 import com.amazonaws.services.ec2.model.DescribeRouteTablesRequest;
 import com.amazonaws.services.ec2.model.DescribeRouteTablesResult;
+import com.amazonaws.services.ec2.model.DescribeScheduledInstanceAvailabilityRequest;
+import com.amazonaws.services.ec2.model.DescribeScheduledInstanceAvailabilityResult;
+import com.amazonaws.services.ec2.model.DescribeScheduledInstancesRequest;
+import com.amazonaws.services.ec2.model.DescribeScheduledInstancesResult;
 import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest;
 import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult;
 import com.amazonaws.services.ec2.model.DescribeSnapshotAttributeRequest;
@@ -233,6 +249,8 @@ import com.amazonaws.services.ec2.model.DescribeVolumesRequest;
 import com.amazonaws.services.ec2.model.DescribeVolumesResult;
 import com.amazonaws.services.ec2.model.DescribeVpcAttributeRequest;
 import com.amazonaws.services.ec2.model.DescribeVpcAttributeResult;
+import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkDnsSupportRequest;
+import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkDnsSupportResult;
 import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkRequest;
 import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkResult;
 import com.amazonaws.services.ec2.model.DescribeVpcEndpointServicesRequest;
@@ -255,6 +273,8 @@ import com.amazonaws.services.ec2.model.DetachVolumeRequest;
 import com.amazonaws.services.ec2.model.DetachVolumeResult;
 import com.amazonaws.services.ec2.model.DetachVpnGatewayRequest;
 import com.amazonaws.services.ec2.model.DisableVgwRoutePropagationRequest;
+import com.amazonaws.services.ec2.model.DisableVpcClassicLinkDnsSupportRequest;
+import com.amazonaws.services.ec2.model.DisableVpcClassicLinkDnsSupportResult;
 import com.amazonaws.services.ec2.model.DisableVpcClassicLinkRequest;
 import com.amazonaws.services.ec2.model.DisableVpcClassicLinkResult;
 import com.amazonaws.services.ec2.model.DisassociateAddressRequest;
@@ -263,6 +283,8 @@ import com.amazonaws.services.ec2.model.DryRunResult;
 import com.amazonaws.services.ec2.model.DryRunSupportedRequest;
 import com.amazonaws.services.ec2.model.EnableVgwRoutePropagationRequest;
 import com.amazonaws.services.ec2.model.EnableVolumeIORequest;
+import com.amazonaws.services.ec2.model.EnableVpcClassicLinkDnsSupportRequest;
+import com.amazonaws.services.ec2.model.EnableVpcClassicLinkDnsSupportResult;
 import com.amazonaws.services.ec2.model.EnableVpcClassicLinkRequest;
 import com.amazonaws.services.ec2.model.EnableVpcClassicLinkResult;
 import com.amazonaws.services.ec2.model.Filter;
@@ -283,8 +305,13 @@ import com.amazonaws.services.ec2.model.ImportVolumeResult;
 import com.amazonaws.services.ec2.model.Instance;
 import com.amazonaws.services.ec2.model.InstanceState;
 import com.amazonaws.services.ec2.model.InstanceStateName;
+import com.amazonaws.services.ec2.model.ModifyHostsRequest;
+import com.amazonaws.services.ec2.model.ModifyHostsResult;
+import com.amazonaws.services.ec2.model.ModifyIdFormatRequest;
 import com.amazonaws.services.ec2.model.ModifyImageAttributeRequest;
 import com.amazonaws.services.ec2.model.ModifyInstanceAttributeRequest;
+import com.amazonaws.services.ec2.model.ModifyInstancePlacementRequest;
+import com.amazonaws.services.ec2.model.ModifyInstancePlacementResult;
 import com.amazonaws.services.ec2.model.ModifyNetworkInterfaceAttributeRequest;
 import com.amazonaws.services.ec2.model.ModifyReservedInstancesRequest;
 import com.amazonaws.services.ec2.model.ModifyReservedInstancesResult;
@@ -302,12 +329,16 @@ import com.amazonaws.services.ec2.model.MoveAddressToVpcRequest;
 import com.amazonaws.services.ec2.model.MoveAddressToVpcResult;
 import com.amazonaws.services.ec2.model.PurchaseReservedInstancesOfferingRequest;
 import com.amazonaws.services.ec2.model.PurchaseReservedInstancesOfferingResult;
+import com.amazonaws.services.ec2.model.PurchaseScheduledInstancesRequest;
+import com.amazonaws.services.ec2.model.PurchaseScheduledInstancesResult;
 import com.amazonaws.services.ec2.model.RebootInstancesRequest;
 import com.amazonaws.services.ec2.model.RegisterImageRequest;
 import com.amazonaws.services.ec2.model.RegisterImageResult;
 import com.amazonaws.services.ec2.model.RejectVpcPeeringConnectionRequest;
 import com.amazonaws.services.ec2.model.RejectVpcPeeringConnectionResult;
 import com.amazonaws.services.ec2.model.ReleaseAddressRequest;
+import com.amazonaws.services.ec2.model.ReleaseHostsRequest;
+import com.amazonaws.services.ec2.model.ReleaseHostsResult;
 import com.amazonaws.services.ec2.model.ReplaceNetworkAclAssociationRequest;
 import com.amazonaws.services.ec2.model.ReplaceNetworkAclAssociationResult;
 import com.amazonaws.services.ec2.model.ReplaceNetworkAclEntryRequest;
@@ -330,6 +361,8 @@ import com.amazonaws.services.ec2.model.RevokeSecurityGroupEgressRequest;
 import com.amazonaws.services.ec2.model.RevokeSecurityGroupIngressRequest;
 import com.amazonaws.services.ec2.model.RunInstancesRequest;
 import com.amazonaws.services.ec2.model.RunInstancesResult;
+import com.amazonaws.services.ec2.model.RunScheduledInstancesRequest;
+import com.amazonaws.services.ec2.model.RunScheduledInstancesResult;
 import com.amazonaws.services.ec2.model.StartInstancesRequest;
 import com.amazonaws.services.ec2.model.StartInstancesResult;
 import com.amazonaws.services.ec2.model.StopInstancesRequest;
@@ -519,6 +552,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public DeleteNatGatewayResult deleteNatGateway(DeleteNatGatewayRequest deleteNatGatewayRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public UnmonitorInstancesResult unmonitorInstances(UnmonitorInstancesRequest unmonitorInstancesRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -639,6 +677,16 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public ModifyHostsResult modifyHosts(ModifyHostsRequest modifyHostsRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
+    @Override
+    public void modifyIdFormat(ModifyIdFormatRequest modifyIdFormatRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public DescribeSecurityGroupsResult describeSecurityGroups(DescribeSecurityGroupsRequest describeSecurityGroupsRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -964,6 +1012,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public DisableVpcClassicLinkDnsSupportResult disableVpcClassicLinkDnsSupport(DisableVpcClassicLinkDnsSupportRequest disableVpcClassicLinkDnsSupportRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public DescribeInstanceAttributeResult describeInstanceAttribute(DescribeInstanceAttributeRequest describeInstanceAttributeRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -989,6 +1042,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public RunScheduledInstancesResult runScheduledInstances(RunScheduledInstancesRequest runScheduledInstancesRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public DescribeSubnetsResult describeSubnets(DescribeSubnetsRequest describeSubnetsRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1024,6 +1082,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public ModifyInstancePlacementResult modifyInstancePlacement(ModifyInstancePlacementRequest modifyInstancePlacementRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public CancelReservedInstancesListingResult cancelReservedInstancesListing(CancelReservedInstancesListingRequest cancelReservedInstancesListingRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1159,6 +1222,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public PurchaseScheduledInstancesResult purchaseScheduledInstances(PurchaseScheduledInstancesRequest purchaseScheduledInstancesRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public void modifySnapshotAttribute(ModifySnapshotAttributeRequest modifySnapshotAttributeRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1334,6 +1402,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public ReleaseHostsResult releaseHosts(ReleaseHostsRequest releaseHostsRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public void resetInstanceAttribute(ResetInstanceAttributeRequest resetInstanceAttributeRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1344,6 +1417,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public CreateNatGatewayResult createNatGateway(CreateNatGatewayRequest createNatGatewayRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public void replaceNetworkAclEntry(ReplaceNetworkAclEntryRequest replaceNetworkAclEntryRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1374,6 +1452,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public EnableVpcClassicLinkDnsSupportResult enableVpcClassicLinkDnsSupport(EnableVpcClassicLinkDnsSupportRequest enableVpcClassicLinkDnsSupportRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public void createVpnConnectionRoute(CreateVpnConnectionRouteRequest createVpnConnectionRouteRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1489,6 +1572,16 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public DescribeScheduledInstanceAvailabilityResult describeScheduledInstanceAvailability(DescribeScheduledInstanceAvailabilityRequest describeScheduledInstanceAvailabilityRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
+    @Override
+    public DescribeScheduledInstancesResult describeScheduledInstances(DescribeScheduledInstancesRequest describeScheduledInstancesRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public DescribeDhcpOptionsResult describeDhcpOptions() throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1529,6 +1622,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public DescribeNatGatewaysResult describeNatGateways(DescribeNatGatewaysRequest describeNatGatewaysRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public DescribeConversionTasksResult describeConversionTasks() throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1544,6 +1642,26 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public DescribeHostsResult describeHosts(DescribeHostsRequest describeHostsRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
+    @Override
+    public DescribeHostsResult describeHosts() {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
+    @Override
+    public DescribeIdFormatResult describeIdFormat(DescribeIdFormatRequest describeIdFormatRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
+    @Override
+    public DescribeIdFormatResult describeIdFormat() {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public DescribeVpcPeeringConnectionsResult describeVpcPeeringConnections() throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1584,6 +1702,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public DescribeVpcClassicLinkDnsSupportResult describeVpcClassicLinkDnsSupport(DescribeVpcClassicLinkDnsSupportRequest describeVpcClassicLinkDnsSupportRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public DescribeClassicLinkInstancesResult describeClassicLinkInstances() throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1649,6 +1772,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public AllocateHostsResult allocateHosts(AllocateHostsRequest allocateHostsRequest) {
+        throw new UnsupportedOperationException("Not supported in mock");
+    }
+
     @Override
     public DescribeSnapshotsResult describeSnapshots() throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");

plugins/repository-s3/build.gradle

@@ -23,7 +23,7 @@ esplugin {
 }
 
 versions << [
-  'aws': '1.10.33'
+  'aws': '1.10.69'
 ]
 
 dependencies {
@@ -54,11 +54,6 @@ test {
 }
 
 thirdPartyAudit.excludes = [
-  // uses internal java api: com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl
-  // uses internal java api: com.sun.org.apache.xml.internal.dtm.ref.DTMManagerDefault
-  // uses internal java api: com.sun.org.apache.xpath.internal.XPathContext
-  'com.amazonaws.util.XpathUtils',
-
   // classes are missing
   'javax.servlet.ServletContextEvent', 
   'javax.servlet.ServletContextListener', 
@@ -169,3 +164,8 @@ thirdPartyAudit.excludes = [
   'javax.xml.bind.util.Messages',
   'javax.xml.bind.util.ValidationEventCollector'
 ]
+
+// AWS SDK is exposing some deprecated methods which we call using a delegate:
+// * setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation)
+// * changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass)
+compileTestJava.options.compilerArgs << "-Xlint:-deprecation"

plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1

@@ -1 +0,0 @@
-fabedbbe2b834b1add150b6a38395c5ef7380168

plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1

@@ -0,0 +1 @@
+a1f02d5f26ba1d8c37e2bf9c847db3c6729dda00

plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1

@@ -1 +0,0 @@
-35881245894ecc4d893c074eacdf2e6b56820fda

plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1

@@ -0,0 +1 @@
+ed74ff3872193b4704a751f0b72ab2cf0db0651b

plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1

@@ -1 +0,0 @@
-5665cf77102a932a16e99ebf41d197e03ddbf25c

plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1

@@ -0,0 +1 @@
+6fa48bf0bff43f26436956b88d8d3764b6cf109e

plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java

@@ -154,5 +154,6 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
         Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
     }
 
-    AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries);
+    AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
+                    boolean useThrottleRetries);
 }

plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java

@@ -57,7 +57,8 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
     }
 
     @Override
-    public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
+    public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
+                                        boolean useThrottleRetries) {
         if (Strings.isNullOrEmpty(endpoint)) {
             // We need to set the endpoint based on the region
             if (region != null) {
@@ -69,10 +70,11 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
             }
         }
 
-        return getClient(endpoint, protocol, account, key, maxRetries);
+        return getClient(endpoint, protocol, account, key, maxRetries, useThrottleRetries);
     }
 
-    private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries) {
+    private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries,
+                                            boolean useThrottleRetries) {
         Tuple<String, String> clientDescriptor = new Tuple<>(endpoint, account);
         AmazonS3Client client = clients.get(clientDescriptor);
         if (client != null) {
@@ -102,6 +104,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
             // If not explicitly set, default to 3 with exponential backoff policy
             clientConfiguration.setMaxErrorRetry(maxRetries);
         }
+        clientConfiguration.setUseThrottleRetries(useThrottleRetries);
 
         // #155: we might have 3rd party users using older S3 API version
         String awsSigner = CLOUD_S3.SIGNER_SETTING.get(settings);

plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java

@@ -121,6 +121,7 @@ public class S3RepositoryPlugin extends Plugin {
         S3Repository.Repositories.STORAGE_CLASS_SETTING,
         S3Repository.Repositories.CANNED_ACL_SETTING,
         S3Repository.Repositories.BASE_PATH_SETTING,
+        S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING,
 
         // Register S3 single repository settings
         S3Repository.Repository.KEY_SETTING,

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.repositories.s3;
 
+import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
 import org.elasticsearch.cloud.aws.AwsS3Service;
 import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3;
@@ -115,6 +116,11 @@ public class S3Repository extends BlobStoreRepository {
          * repositories.s3.max_retries: Number of retries in case of S3 errors. Defaults to 3.
          */
         Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope);
+        /**
+         * repositories.s3.use_throttle_retries: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
+         */
+        Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("repositories.s3.use_throttle_retries",
+            ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
         /**
          * repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g.
          */
@@ -196,6 +202,12 @@ public class S3Repository extends BlobStoreRepository {
          * @see  Repositories#MAX_RETRIES_SETTING
          */
         Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3, Property.NodeScope);
+        /**
+         * use_throttle_retries
+         * @see  Repositories#USE_THROTTLE_RETRIES_SETTING
+         */
+        Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("use_throttle_retries",
+            ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
         /**
          * chunk_size
          * @see  Repositories#CHUNK_SIZE_SETTING
@@ -262,6 +274,7 @@ public class S3Repository extends BlobStoreRepository {
         boolean serverSideEncryption = getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
         ByteSizeValue bufferSize = getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING);
         Integer maxRetries = getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING);
+        boolean useThrottleRetries = getValue(repositorySettings, Repository.USE_THROTTLE_RETRIES_SETTING, Repositories.USE_THROTTLE_RETRIES_SETTING);
         this.chunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING);
         this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING);
 
@@ -275,13 +288,15 @@ public class S3Repository extends BlobStoreRepository {
         String storageClass = getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING);
         String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING);
 
-        logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]",
+        logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " +
-                bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
+            "buffer_size [{}], max_retries [{}], use_throttle_retries [{}], cannedACL [{}], storageClass [{}]",
+            bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
+            storageClass);
 
         String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
         String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING);
 
-        blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries),
+        blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries, useThrottleRetries),
                 bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
 
         String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);

plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java

@@ -50,6 +50,7 @@ import com.amazonaws.services.s3.model.CreateBucketRequest;
 import com.amazonaws.services.s3.model.DeleteBucketCrossOriginConfigurationRequest;
 import com.amazonaws.services.s3.model.DeleteBucketLifecycleConfigurationRequest;
 import com.amazonaws.services.s3.model.DeleteBucketPolicyRequest;
+import com.amazonaws.services.s3.model.DeleteBucketReplicationConfigurationRequest;
 import com.amazonaws.services.s3.model.DeleteBucketRequest;
 import com.amazonaws.services.s3.model.DeleteBucketTaggingConfigurationRequest;
 import com.amazonaws.services.s3.model.DeleteBucketWebsiteConfigurationRequest;
@@ -69,6 +70,7 @@ import com.amazonaws.services.s3.model.GetBucketReplicationConfigurationRequest;
 import com.amazonaws.services.s3.model.GetBucketTaggingConfigurationRequest;
 import com.amazonaws.services.s3.model.GetBucketVersioningConfigurationRequest;
 import com.amazonaws.services.s3.model.GetBucketWebsiteConfigurationRequest;
+import com.amazonaws.services.s3.model.GetObjectAclRequest;
 import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.GetS3AccountOwnerRequest;
@@ -253,6 +255,11 @@ public class AmazonS3Wrapper implements AmazonS3 {
         return delegate.getObjectAcl(bucketName, key, versionId);
     }
 
+    @Override
+    public AccessControlList getObjectAcl(GetObjectAclRequest getObjectAclRequest) throws AmazonClientException, AmazonServiceException {
+        return delegate.getObjectAcl(getObjectAclRequest);
+    }
+
     @Override
     public void setObjectAcl(String bucketName, String key, AccessControlList acl) throws AmazonClientException, AmazonServiceException {
         delegate.setObjectAcl(bucketName, key, acl);
@@ -358,6 +365,17 @@ public class AmazonS3Wrapper implements AmazonS3 {
         delegate.deleteBucketReplicationConfiguration(bucketName);
     }
 
+    @Override
+    public void deleteBucketReplicationConfiguration(DeleteBucketReplicationConfigurationRequest request) throws AmazonServiceException,
+        AmazonClientException {
+        delegate.deleteBucketReplicationConfiguration(request);
+    }
+
+    @Override
+    public boolean doesObjectExist(String bucketName, String objectName) throws AmazonServiceException, AmazonClientException {
+        return delegate.doesObjectExist(bucketName, objectName);
+    }
+
     @Override
     public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException {
         return delegate.putObject(putObjectRequest);

plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java

@@ -43,8 +43,9 @@ public class TestAwsS3Service extends InternalAwsS3Service {
 
 
     @Override
-    public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
+    public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
-        return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries));
+                                        boolean useThrottleRetries) {
+        return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, useThrottleRetries));
     }
 
     private AmazonS3 cachedWrapper(AmazonS3 client) {

plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java

@@ -200,7 +200,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
             S3Repository.Repositories.REGION_SETTING.get(settings),
             S3Repository.Repositories.KEY_SETTING.get(settings),
             S3Repository.Repositories.SECRET_SETTING.get(settings),
-            null);
+            null, randomBoolean());
 
         String bucketName = bucket.get("bucket");
         logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
@@ -475,7 +475,8 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
             // We check that settings has been set in elasticsearch.yml integration test file
             // as described in README
             assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
-            AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null);
+            AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey,
+                null, randomBoolean());
             try {
                 ObjectListing prevListing = null;
                 //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html

plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml

@@ -14,7 +14,7 @@
               canned_acl: "public-read"
               storage_class: "standard"
 
-    # Get repositry
+    # Get repository
     - do:
         snapshot.get_repository:
           repository: test_repo_s3_1