diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index f086158954c..63e3ad311b2 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -39,7 +39,8 @@ http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html credentials for authentication. These can be overridden by, in increasing order of precedence, system properties `aws.accessKeyId` and `aws.secretKey`, environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_KEY`, or the -elasticsearch config using `cloud.aws.access_key` and `cloud.aws.secret_key`: +elasticsearch config using `cloud.aws.access_key` and `cloud.aws.secret_key` or +if you wish to set credentials specifically for s3 `cloud.aws.s3.access_key` and `cloud.aws.s3.secret_key`: [source,yaml] ---- @@ -106,6 +107,7 @@ cloud: ===== Region The `cloud.aws.region` can be set to a region and will automatically use the relevant settings for both `ec2` and `s3`. +You can specifically set it for s3 only using `cloud.aws.s3.region`. The available values are: * `us-east` (`us-east-1`) @@ -217,9 +219,13 @@ The following settings are supported: Number of retries in case of S3 errors. Defaults to `3`. +`use_throttle_retries`:: + + Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`). + `read_only`:: - Makes repository read-only. coming[2.1.0] Defaults to `false`. + Makes repository read-only. Defaults to `false`. `canned_acl`:: @@ -237,6 +243,10 @@ The following settings are supported: currently supported by the plugin. For more information about the different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide] +Note that you can define S3 repository settings for all S3 repositories in `elasticsearch.yml` configuration file. +They are all prefixed with `repositories.s3.`. For example, you can define compression for all S3 repositories +by setting `repositories.s3.compress: true` in `elasticsearch.yml`. + The S3 repositories use the same credentials as the rest of the AWS services provided by this plugin (`discovery`). See <> for details. diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index e8baa1ec502..15cce1aa32b 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.10.33' + 'aws': '1.10.69' ] dependencies { @@ -48,11 +48,6 @@ test { } thirdPartyAudit.excludes = [ - // uses internal java api: com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl - // uses internal java api: com.sun.org.apache.xml.internal.dtm.ref.DTMManagerDefault - // uses internal java api: com.sun.org.apache.xpath.internal.XPathContext - 'com.amazonaws.util.XpathUtils', - // classes are missing 'javax.servlet.ServletContextEvent', 'javax.servlet.ServletContextListener', diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1 deleted file mode 100644 index 332a8f01035..00000000000 --- a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fabedbbe2b834b1add150b6a38395c5ef7380168 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.69.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.69.jar.sha1 new file mode 100644 index 00000000000..2971a33d7d9 --- /dev/null +++ b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.69.jar.sha1 @@ -0,0 +1 @@ +a1f02d5f26ba1d8c37e2bf9c847db3c6729dda00 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1 deleted file mode 100644 index 4737b80b3f2..00000000000 --- a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -202f6b5dbc196e355d50c131b0fd34969bfd89e6 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.69.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.69.jar.sha1 new file mode 100644 index 00000000000..a1a493d3b8f --- /dev/null +++ b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.69.jar.sha1 @@ -0,0 +1 @@ +afbff1ece8365859eb4cfe0d3ba543d68b154d26 \ No newline at end of file diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java index 88d87a2d8c2..49f4f885419 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java @@ -29,6 +29,8 @@ import com.amazonaws.services.ec2.model.AcceptVpcPeeringConnectionRequest; import com.amazonaws.services.ec2.model.AcceptVpcPeeringConnectionResult; import com.amazonaws.services.ec2.model.AllocateAddressRequest; import com.amazonaws.services.ec2.model.AllocateAddressResult; +import com.amazonaws.services.ec2.model.AllocateHostsRequest; +import com.amazonaws.services.ec2.model.AllocateHostsResult; import com.amazonaws.services.ec2.model.AssignPrivateIpAddressesRequest; import com.amazonaws.services.ec2.model.AssociateAddressRequest; import com.amazonaws.services.ec2.model.AssociateAddressResult; @@ -80,6 +82,8 @@ import com.amazonaws.services.ec2.model.CreateInternetGatewayRequest; import com.amazonaws.services.ec2.model.CreateInternetGatewayResult; import com.amazonaws.services.ec2.model.CreateKeyPairRequest; import com.amazonaws.services.ec2.model.CreateKeyPairResult; +import com.amazonaws.services.ec2.model.CreateNatGatewayRequest; +import com.amazonaws.services.ec2.model.CreateNatGatewayResult; import com.amazonaws.services.ec2.model.CreateNetworkAclEntryRequest; import com.amazonaws.services.ec2.model.CreateNetworkAclRequest; import com.amazonaws.services.ec2.model.CreateNetworkAclResult; @@ -120,6 +124,8 @@ import com.amazonaws.services.ec2.model.DeleteFlowLogsRequest; import com.amazonaws.services.ec2.model.DeleteFlowLogsResult; import com.amazonaws.services.ec2.model.DeleteInternetGatewayRequest; import com.amazonaws.services.ec2.model.DeleteKeyPairRequest; +import com.amazonaws.services.ec2.model.DeleteNatGatewayRequest; +import com.amazonaws.services.ec2.model.DeleteNatGatewayResult; import com.amazonaws.services.ec2.model.DeleteNetworkAclEntryRequest; import com.amazonaws.services.ec2.model.DeleteNetworkAclRequest; import com.amazonaws.services.ec2.model.DeleteNetworkInterfaceRequest; @@ -161,6 +167,10 @@ import com.amazonaws.services.ec2.model.DescribeExportTasksRequest; import com.amazonaws.services.ec2.model.DescribeExportTasksResult; import com.amazonaws.services.ec2.model.DescribeFlowLogsRequest; import com.amazonaws.services.ec2.model.DescribeFlowLogsResult; +import com.amazonaws.services.ec2.model.DescribeHostsRequest; +import com.amazonaws.services.ec2.model.DescribeHostsResult; +import com.amazonaws.services.ec2.model.DescribeIdFormatRequest; +import com.amazonaws.services.ec2.model.DescribeIdFormatResult; import com.amazonaws.services.ec2.model.DescribeImageAttributeRequest; import com.amazonaws.services.ec2.model.DescribeImageAttributeResult; import com.amazonaws.services.ec2.model.DescribeImagesRequest; @@ -181,6 +191,8 @@ import com.amazonaws.services.ec2.model.DescribeKeyPairsRequest; import com.amazonaws.services.ec2.model.DescribeKeyPairsResult; import com.amazonaws.services.ec2.model.DescribeMovingAddressesRequest; import com.amazonaws.services.ec2.model.DescribeMovingAddressesResult; +import com.amazonaws.services.ec2.model.DescribeNatGatewaysRequest; +import com.amazonaws.services.ec2.model.DescribeNatGatewaysResult; import com.amazonaws.services.ec2.model.DescribeNetworkAclsRequest; import com.amazonaws.services.ec2.model.DescribeNetworkAclsResult; import com.amazonaws.services.ec2.model.DescribeNetworkInterfaceAttributeRequest; @@ -203,6 +215,10 @@ import com.amazonaws.services.ec2.model.DescribeReservedInstancesRequest; import com.amazonaws.services.ec2.model.DescribeReservedInstancesResult; import com.amazonaws.services.ec2.model.DescribeRouteTablesRequest; import com.amazonaws.services.ec2.model.DescribeRouteTablesResult; +import com.amazonaws.services.ec2.model.DescribeScheduledInstanceAvailabilityRequest; +import com.amazonaws.services.ec2.model.DescribeScheduledInstanceAvailabilityResult; +import com.amazonaws.services.ec2.model.DescribeScheduledInstancesRequest; +import com.amazonaws.services.ec2.model.DescribeScheduledInstancesResult; import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest; import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult; import com.amazonaws.services.ec2.model.DescribeSnapshotAttributeRequest; @@ -233,6 +249,8 @@ import com.amazonaws.services.ec2.model.DescribeVolumesRequest; import com.amazonaws.services.ec2.model.DescribeVolumesResult; import com.amazonaws.services.ec2.model.DescribeVpcAttributeRequest; import com.amazonaws.services.ec2.model.DescribeVpcAttributeResult; +import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkDnsSupportRequest; +import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkDnsSupportResult; import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkRequest; import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkResult; import com.amazonaws.services.ec2.model.DescribeVpcEndpointServicesRequest; @@ -255,6 +273,8 @@ import com.amazonaws.services.ec2.model.DetachVolumeRequest; import com.amazonaws.services.ec2.model.DetachVolumeResult; import com.amazonaws.services.ec2.model.DetachVpnGatewayRequest; import com.amazonaws.services.ec2.model.DisableVgwRoutePropagationRequest; +import com.amazonaws.services.ec2.model.DisableVpcClassicLinkDnsSupportRequest; +import com.amazonaws.services.ec2.model.DisableVpcClassicLinkDnsSupportResult; import com.amazonaws.services.ec2.model.DisableVpcClassicLinkRequest; import com.amazonaws.services.ec2.model.DisableVpcClassicLinkResult; import com.amazonaws.services.ec2.model.DisassociateAddressRequest; @@ -263,6 +283,8 @@ import com.amazonaws.services.ec2.model.DryRunResult; import com.amazonaws.services.ec2.model.DryRunSupportedRequest; import com.amazonaws.services.ec2.model.EnableVgwRoutePropagationRequest; import com.amazonaws.services.ec2.model.EnableVolumeIORequest; +import com.amazonaws.services.ec2.model.EnableVpcClassicLinkDnsSupportRequest; +import com.amazonaws.services.ec2.model.EnableVpcClassicLinkDnsSupportResult; import com.amazonaws.services.ec2.model.EnableVpcClassicLinkRequest; import com.amazonaws.services.ec2.model.EnableVpcClassicLinkResult; import com.amazonaws.services.ec2.model.Filter; @@ -283,8 +305,13 @@ import com.amazonaws.services.ec2.model.ImportVolumeResult; import com.amazonaws.services.ec2.model.Instance; import com.amazonaws.services.ec2.model.InstanceState; import com.amazonaws.services.ec2.model.InstanceStateName; +import com.amazonaws.services.ec2.model.ModifyHostsRequest; +import com.amazonaws.services.ec2.model.ModifyHostsResult; +import com.amazonaws.services.ec2.model.ModifyIdFormatRequest; import com.amazonaws.services.ec2.model.ModifyImageAttributeRequest; import com.amazonaws.services.ec2.model.ModifyInstanceAttributeRequest; +import com.amazonaws.services.ec2.model.ModifyInstancePlacementRequest; +import com.amazonaws.services.ec2.model.ModifyInstancePlacementResult; import com.amazonaws.services.ec2.model.ModifyNetworkInterfaceAttributeRequest; import com.amazonaws.services.ec2.model.ModifyReservedInstancesRequest; import com.amazonaws.services.ec2.model.ModifyReservedInstancesResult; @@ -302,12 +329,16 @@ import com.amazonaws.services.ec2.model.MoveAddressToVpcRequest; import com.amazonaws.services.ec2.model.MoveAddressToVpcResult; import com.amazonaws.services.ec2.model.PurchaseReservedInstancesOfferingRequest; import com.amazonaws.services.ec2.model.PurchaseReservedInstancesOfferingResult; +import com.amazonaws.services.ec2.model.PurchaseScheduledInstancesRequest; +import com.amazonaws.services.ec2.model.PurchaseScheduledInstancesResult; import com.amazonaws.services.ec2.model.RebootInstancesRequest; import com.amazonaws.services.ec2.model.RegisterImageRequest; import com.amazonaws.services.ec2.model.RegisterImageResult; import com.amazonaws.services.ec2.model.RejectVpcPeeringConnectionRequest; import com.amazonaws.services.ec2.model.RejectVpcPeeringConnectionResult; import com.amazonaws.services.ec2.model.ReleaseAddressRequest; +import com.amazonaws.services.ec2.model.ReleaseHostsRequest; +import com.amazonaws.services.ec2.model.ReleaseHostsResult; import com.amazonaws.services.ec2.model.ReplaceNetworkAclAssociationRequest; import com.amazonaws.services.ec2.model.ReplaceNetworkAclAssociationResult; import com.amazonaws.services.ec2.model.ReplaceNetworkAclEntryRequest; @@ -330,6 +361,8 @@ import com.amazonaws.services.ec2.model.RevokeSecurityGroupEgressRequest; import com.amazonaws.services.ec2.model.RevokeSecurityGroupIngressRequest; import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.RunInstancesResult; +import com.amazonaws.services.ec2.model.RunScheduledInstancesRequest; +import com.amazonaws.services.ec2.model.RunScheduledInstancesResult; import com.amazonaws.services.ec2.model.StartInstancesRequest; import com.amazonaws.services.ec2.model.StartInstancesResult; import com.amazonaws.services.ec2.model.StopInstancesRequest; @@ -519,6 +552,11 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public DeleteNatGatewayResult deleteNatGateway(DeleteNatGatewayRequest deleteNatGatewayRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public UnmonitorInstancesResult unmonitorInstances(UnmonitorInstancesRequest unmonitorInstancesRequest) throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -639,6 +677,16 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public ModifyHostsResult modifyHosts(ModifyHostsRequest modifyHostsRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + + @Override + public void modifyIdFormat(ModifyIdFormatRequest modifyIdFormatRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public DescribeSecurityGroupsResult describeSecurityGroups(DescribeSecurityGroupsRequest describeSecurityGroupsRequest) throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -964,6 +1012,11 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public DisableVpcClassicLinkDnsSupportResult disableVpcClassicLinkDnsSupport(DisableVpcClassicLinkDnsSupportRequest disableVpcClassicLinkDnsSupportRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public DescribeInstanceAttributeResult describeInstanceAttribute(DescribeInstanceAttributeRequest describeInstanceAttributeRequest) throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -989,6 +1042,11 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public RunScheduledInstancesResult runScheduledInstances(RunScheduledInstancesRequest runScheduledInstancesRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public DescribeSubnetsResult describeSubnets(DescribeSubnetsRequest describeSubnetsRequest) throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -1024,6 +1082,11 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public ModifyInstancePlacementResult modifyInstancePlacement(ModifyInstancePlacementRequest modifyInstancePlacementRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public CancelReservedInstancesListingResult cancelReservedInstancesListing(CancelReservedInstancesListingRequest cancelReservedInstancesListingRequest) throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -1159,6 +1222,11 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public PurchaseScheduledInstancesResult purchaseScheduledInstances(PurchaseScheduledInstancesRequest purchaseScheduledInstancesRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public void modifySnapshotAttribute(ModifySnapshotAttributeRequest modifySnapshotAttributeRequest) throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -1334,6 +1402,11 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public ReleaseHostsResult releaseHosts(ReleaseHostsRequest releaseHostsRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public void resetInstanceAttribute(ResetInstanceAttributeRequest resetInstanceAttributeRequest) throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -1344,6 +1417,11 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public CreateNatGatewayResult createNatGateway(CreateNatGatewayRequest createNatGatewayRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public void replaceNetworkAclEntry(ReplaceNetworkAclEntryRequest replaceNetworkAclEntryRequest) throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -1374,6 +1452,11 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public EnableVpcClassicLinkDnsSupportResult enableVpcClassicLinkDnsSupport(EnableVpcClassicLinkDnsSupportRequest enableVpcClassicLinkDnsSupportRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public void createVpnConnectionRoute(CreateVpnConnectionRouteRequest createVpnConnectionRouteRequest) throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -1489,6 +1572,16 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public DescribeScheduledInstanceAvailabilityResult describeScheduledInstanceAvailability(DescribeScheduledInstanceAvailabilityRequest describeScheduledInstanceAvailabilityRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + + @Override + public DescribeScheduledInstancesResult describeScheduledInstances(DescribeScheduledInstancesRequest describeScheduledInstancesRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public DescribeDhcpOptionsResult describeDhcpOptions() throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -1529,6 +1622,11 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public DescribeNatGatewaysResult describeNatGateways(DescribeNatGatewaysRequest describeNatGatewaysRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public DescribeConversionTasksResult describeConversionTasks() throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -1544,6 +1642,26 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public DescribeHostsResult describeHosts(DescribeHostsRequest describeHostsRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + + @Override + public DescribeHostsResult describeHosts() { + throw new UnsupportedOperationException("Not supported in mock"); + } + + @Override + public DescribeIdFormatResult describeIdFormat(DescribeIdFormatRequest describeIdFormatRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + + @Override + public DescribeIdFormatResult describeIdFormat() { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public DescribeVpcPeeringConnectionsResult describeVpcPeeringConnections() throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -1584,6 +1702,11 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public DescribeVpcClassicLinkDnsSupportResult describeVpcClassicLinkDnsSupport(DescribeVpcClassicLinkDnsSupportRequest describeVpcClassicLinkDnsSupportRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public DescribeClassicLinkInstancesResult describeClassicLinkInstances() throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); @@ -1649,6 +1772,11 @@ public class AmazonEC2Mock implements AmazonEC2 { throw new UnsupportedOperationException("Not supported in mock"); } + @Override + public AllocateHostsResult allocateHosts(AllocateHostsRequest allocateHostsRequest) { + throw new UnsupportedOperationException("Not supported in mock"); + } + @Override public DescribeSnapshotsResult describeSnapshots() throws AmazonServiceException, AmazonClientException { throw new UnsupportedOperationException("Not supported in mock"); diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index a7b8d8233eb..f713d8eadc5 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.10.33' + 'aws': '1.10.69' ] dependencies { @@ -54,11 +54,6 @@ test { } thirdPartyAudit.excludes = [ - // uses internal java api: com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl - // uses internal java api: com.sun.org.apache.xml.internal.dtm.ref.DTMManagerDefault - // uses internal java api: com.sun.org.apache.xpath.internal.XPathContext - 'com.amazonaws.util.XpathUtils', - // classes are missing 'javax.servlet.ServletContextEvent', 'javax.servlet.ServletContextListener', @@ -169,3 +164,8 @@ thirdPartyAudit.excludes = [ 'javax.xml.bind.util.Messages', 'javax.xml.bind.util.ValidationEventCollector' ] + +// AWS SDK is exposing some deprecated methods which we call using a delegate: +// * setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation) +// * changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass) +compileTestJava.options.compilerArgs << "-Xlint:-deprecation" diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1 deleted file mode 100644 index 332a8f01035..00000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fabedbbe2b834b1add150b6a38395c5ef7380168 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1 new file mode 100644 index 00000000000..2971a33d7d9 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1 @@ -0,0 +1 @@ +a1f02d5f26ba1d8c37e2bf9c847db3c6729dda00 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1 deleted file mode 100644 index 0d7ab9f8381..00000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -35881245894ecc4d893c074eacdf2e6b56820fda \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1 new file mode 100644 index 00000000000..22e1f924664 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1 @@ -0,0 +1 @@ +ed74ff3872193b4704a751f0b72ab2cf0db0651b \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1 deleted file mode 100644 index 3328f01c658..00000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5665cf77102a932a16e99ebf41d197e03ddbf25c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1 new file mode 100644 index 00000000000..64e7336c2bc --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1 @@ -0,0 +1 @@ +6fa48bf0bff43f26436956b88d8d3764b6cf109e \ No newline at end of file diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index 87c37b52e72..15cd55c823d 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -154,5 +154,6 @@ public interface AwsS3Service extends LifecycleComponent { Setting ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope); } - AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries); + AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, + boolean useThrottleRetries); } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 81b6463a746..352aa196b06 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -57,7 +57,8 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent clientDescriptor = new Tuple<>(endpoint, account); AmazonS3Client client = clients.get(clientDescriptor); if (client != null) { @@ -102,6 +104,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope); + /** + * repositories.s3.use_throttle_retries: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`). + */ + Setting USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("repositories.s3.use_throttle_retries", + ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope); /** * repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g. */ @@ -196,6 +202,12 @@ public class S3Repository extends BlobStoreRepository { * @see Repositories#MAX_RETRIES_SETTING */ Setting MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3, Property.NodeScope); + /** + * use_throttle_retries + * @see Repositories#USE_THROTTLE_RETRIES_SETTING + */ + Setting USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("use_throttle_retries", + ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope); /** * chunk_size * @see Repositories#CHUNK_SIZE_SETTING @@ -262,6 +274,7 @@ public class S3Repository extends BlobStoreRepository { boolean serverSideEncryption = getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING); ByteSizeValue bufferSize = getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING); Integer maxRetries = getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING); + boolean useThrottleRetries = getValue(repositorySettings, Repository.USE_THROTTLE_RETRIES_SETTING, Repositories.USE_THROTTLE_RETRIES_SETTING); this.chunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING); this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING); @@ -275,13 +288,15 @@ public class S3Repository extends BlobStoreRepository { String storageClass = getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING); String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING); - logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]", - bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass); + logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " + + "buffer_size [{}], max_retries [{}], use_throttle_retries [{}], cannedACL [{}], storageClass [{}]", + bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL, + storageClass); String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING); String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING); - blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries), + blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries, useThrottleRetries), bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass); String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java index 97829f9d689..7cda9ee0947 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java @@ -50,6 +50,7 @@ import com.amazonaws.services.s3.model.CreateBucketRequest; import com.amazonaws.services.s3.model.DeleteBucketCrossOriginConfigurationRequest; import com.amazonaws.services.s3.model.DeleteBucketLifecycleConfigurationRequest; import com.amazonaws.services.s3.model.DeleteBucketPolicyRequest; +import com.amazonaws.services.s3.model.DeleteBucketReplicationConfigurationRequest; import com.amazonaws.services.s3.model.DeleteBucketRequest; import com.amazonaws.services.s3.model.DeleteBucketTaggingConfigurationRequest; import com.amazonaws.services.s3.model.DeleteBucketWebsiteConfigurationRequest; @@ -69,6 +70,7 @@ import com.amazonaws.services.s3.model.GetBucketReplicationConfigurationRequest; import com.amazonaws.services.s3.model.GetBucketTaggingConfigurationRequest; import com.amazonaws.services.s3.model.GetBucketVersioningConfigurationRequest; import com.amazonaws.services.s3.model.GetBucketWebsiteConfigurationRequest; +import com.amazonaws.services.s3.model.GetObjectAclRequest; import com.amazonaws.services.s3.model.GetObjectMetadataRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.GetS3AccountOwnerRequest; @@ -253,6 +255,11 @@ public class AmazonS3Wrapper implements AmazonS3 { return delegate.getObjectAcl(bucketName, key, versionId); } + @Override + public AccessControlList getObjectAcl(GetObjectAclRequest getObjectAclRequest) throws AmazonClientException, AmazonServiceException { + return delegate.getObjectAcl(getObjectAclRequest); + } + @Override public void setObjectAcl(String bucketName, String key, AccessControlList acl) throws AmazonClientException, AmazonServiceException { delegate.setObjectAcl(bucketName, key, acl); @@ -358,6 +365,17 @@ public class AmazonS3Wrapper implements AmazonS3 { delegate.deleteBucketReplicationConfiguration(bucketName); } + @Override + public void deleteBucketReplicationConfiguration(DeleteBucketReplicationConfigurationRequest request) throws AmazonServiceException, + AmazonClientException { + delegate.deleteBucketReplicationConfiguration(request); + } + + @Override + public boolean doesObjectExist(String bucketName, String objectName) throws AmazonServiceException, AmazonClientException { + return delegate.doesObjectExist(bucketName, objectName); + } + @Override public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException { return delegate.putObject(putObjectRequest); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java index f83ad222b70..36608c8b172 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java @@ -43,8 +43,9 @@ public class TestAwsS3Service extends InternalAwsS3Service { @Override - public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) { - return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries)); + public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, + boolean useThrottleRetries) { + return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, useThrottleRetries)); } private AmazonS3 cachedWrapper(AmazonS3 client) { diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index 743e2645ab2..2e196610c1a 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -200,7 +200,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase S3Repository.Repositories.REGION_SETTING.get(settings), S3Repository.Repositories.KEY_SETTING.get(settings), S3Repository.Repositories.SECRET_SETTING.get(settings), - null); + null, randomBoolean()); String bucketName = bucket.get("bucket"); logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath); @@ -475,7 +475,8 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase // We check that settings has been set in elasticsearch.yml integration test file // as described in README assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue()); - AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null); + AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, + null, randomBoolean()); try { ObjectListing prevListing = null; //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml index eb909f91f1d..17294986d2d 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml @@ -14,7 +14,7 @@ canned_acl: "public-read" storage_class: "standard" - # Get repositry + # Get repository - do: snapshot.get_repository: repository: test_repo_s3_1