honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'pr/update-aws-sdk' 

# Conflicts:
#	plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
This commit is contained in:
David Pilato 2016-06-17 17:23:12 +02:00
parent fb2a48d0f0 82fee9f7a7
commit bf7a6f5509
22 changed files with 204 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/plugins/repository-s3.asciidoc
View File

@ -39,7 +39,8 @@ http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
credentials for authentication. These can be overridden by, in increasing
order of precedence, system properties `aws.accessKeyId` and `aws.secretKey`,
environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_KEY`, or the
elasticsearch config using `cloud.aws.access_key` and `cloud.aws.secret_key`:
elasticsearch config using `cloud.aws.access_key` and `cloud.aws.secret_key` or
if you wish to set credentials specifically for s3 `cloud.aws.s3.access_key` and `cloud.aws.s3.secret_key`:
[source,yaml]
----
@ -106,6 +107,7 @@ cloud:
===== Region
The `cloud.aws.region` can be set to a region and will automatically use the relevant settings for both `ec2` and `s3`.
You can specifically set it for s3 only using `cloud.aws.s3.region`.
The available values are:
* `us-east` (`us-east-1`)
@ -217,9 +219,13 @@ The following settings are supported:
Number of retries in case of S3 errors. Defaults to `3`.
`use_throttle_retries`::
Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
`read_only`::
Makes repository read-only. coming[2.1.0] Defaults to `false`.
Makes repository read-only. Defaults to `false`.
`canned_acl`::
@ -237,6 +243,10 @@ The following settings are supported:
currently supported by the plugin. For more information about the
different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide]
Note that you can define S3 repository settings for all S3 repositories in `elasticsearch.yml` configuration file.
They are all prefixed with `repositories.s3.`. For example, you can define compression for all S3 repositories
by setting `repositories.s3.compress: true` in `elasticsearch.yml`.
The S3 repositories use the same credentials as the rest of the AWS services
provided by this plugin (`discovery`). See <<repository-s3-usage>> for details.

7
plugins/discovery-ec2/build.gradle
View File

@ -23,7 +23,7 @@ esplugin {
}
versions << [
'aws': '1.10.33'
'aws': '1.10.69'
]
dependencies {
@ -48,11 +48,6 @@ test {
}
thirdPartyAudit.excludes = [
// uses internal java api: com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl
// uses internal java api: com.sun.org.apache.xml.internal.dtm.ref.DTMManagerDefault
// uses internal java api: com.sun.org.apache.xpath.internal.XPathContext
'com.amazonaws.util.XpathUtils',
// classes are missing
'javax.servlet.ServletContextEvent',
'javax.servlet.ServletContextListener',

1
plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1
View File

@ -1 +0,0 @@
fabedbbe2b834b1add150b6a38395c5ef7380168

1
plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.69.jar.sha1 Normal file
View File

@ -0,0 +1 @@
a1f02d5f26ba1d8c37e2bf9c847db3c6729dda00

1
plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1
View File

@ -1 +0,0 @@
202f6b5dbc196e355d50c131b0fd34969bfd89e6

1
plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.69.jar.sha1 Normal file
View File

@ -0,0 +1 @@
afbff1ece8365859eb4cfe0d3ba543d68b154d26

128
plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java
View File

@ -29,6 +29,8 @@ import com.amazonaws.services.ec2.model.AcceptVpcPeeringConnectionRequest;
import com.amazonaws.services.ec2.model.AcceptVpcPeeringConnectionResult;
import com.amazonaws.services.ec2.model.AllocateAddressRequest;
import com.amazonaws.services.ec2.model.AllocateAddressResult;
import com.amazonaws.services.ec2.model.AllocateHostsRequest;
import com.amazonaws.services.ec2.model.AllocateHostsResult;
import com.amazonaws.services.ec2.model.AssignPrivateIpAddressesRequest;
import com.amazonaws.services.ec2.model.AssociateAddressRequest;
import com.amazonaws.services.ec2.model.AssociateAddressResult;
@ -80,6 +82,8 @@ import com.amazonaws.services.ec2.model.CreateInternetGatewayRequest;
import com.amazonaws.services.ec2.model.CreateInternetGatewayResult;
import com.amazonaws.services.ec2.model.CreateKeyPairRequest;
import com.amazonaws.services.ec2.model.CreateKeyPairResult;
import com.amazonaws.services.ec2.model.CreateNatGatewayRequest;
import com.amazonaws.services.ec2.model.CreateNatGatewayResult;
import com.amazonaws.services.ec2.model.CreateNetworkAclEntryRequest;
import com.amazonaws.services.ec2.model.CreateNetworkAclRequest;
import com.amazonaws.services.ec2.model.CreateNetworkAclResult;
@ -120,6 +124,8 @@ import com.amazonaws.services.ec2.model.DeleteFlowLogsRequest;
import com.amazonaws.services.ec2.model.DeleteFlowLogsResult;
import com.amazonaws.services.ec2.model.DeleteInternetGatewayRequest;
import com.amazonaws.services.ec2.model.DeleteKeyPairRequest;
import com.amazonaws.services.ec2.model.DeleteNatGatewayRequest;
import com.amazonaws.services.ec2.model.DeleteNatGatewayResult;
import com.amazonaws.services.ec2.model.DeleteNetworkAclEntryRequest;
import com.amazonaws.services.ec2.model.DeleteNetworkAclRequest;
import com.amazonaws.services.ec2.model.DeleteNetworkInterfaceRequest;
@ -161,6 +167,10 @@ import com.amazonaws.services.ec2.model.DescribeExportTasksRequest;
import com.amazonaws.services.ec2.model.DescribeExportTasksResult;
import com.amazonaws.services.ec2.model.DescribeFlowLogsRequest;
import com.amazonaws.services.ec2.model.DescribeFlowLogsResult;
import com.amazonaws.services.ec2.model.DescribeHostsRequest;
import com.amazonaws.services.ec2.model.DescribeHostsResult;
import com.amazonaws.services.ec2.model.DescribeIdFormatRequest;
import com.amazonaws.services.ec2.model.DescribeIdFormatResult;
import com.amazonaws.services.ec2.model.DescribeImageAttributeRequest;
import com.amazonaws.services.ec2.model.DescribeImageAttributeResult;
import com.amazonaws.services.ec2.model.DescribeImagesRequest;
@ -181,6 +191,8 @@ import com.amazonaws.services.ec2.model.DescribeKeyPairsRequest;
import com.amazonaws.services.ec2.model.DescribeKeyPairsResult;
import com.amazonaws.services.ec2.model.DescribeMovingAddressesRequest;
import com.amazonaws.services.ec2.model.DescribeMovingAddressesResult;
import com.amazonaws.services.ec2.model.DescribeNatGatewaysRequest;
import com.amazonaws.services.ec2.model.DescribeNatGatewaysResult;
import com.amazonaws.services.ec2.model.DescribeNetworkAclsRequest;
import com.amazonaws.services.ec2.model.DescribeNetworkAclsResult;
import com.amazonaws.services.ec2.model.DescribeNetworkInterfaceAttributeRequest;
@ -203,6 +215,10 @@ import com.amazonaws.services.ec2.model.DescribeReservedInstancesRequest;
import com.amazonaws.services.ec2.model.DescribeReservedInstancesResult;
import com.amazonaws.services.ec2.model.DescribeRouteTablesRequest;
import com.amazonaws.services.ec2.model.DescribeRouteTablesResult;
import com.amazonaws.services.ec2.model.DescribeScheduledInstanceAvailabilityRequest;
import com.amazonaws.services.ec2.model.DescribeScheduledInstanceAvailabilityResult;
import com.amazonaws.services.ec2.model.DescribeScheduledInstancesRequest;
import com.amazonaws.services.ec2.model.DescribeScheduledInstancesResult;
import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest;
import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult;
import com.amazonaws.services.ec2.model.DescribeSnapshotAttributeRequest;
@ -233,6 +249,8 @@ import com.amazonaws.services.ec2.model.DescribeVolumesRequest;
import com.amazonaws.services.ec2.model.DescribeVolumesResult;
import com.amazonaws.services.ec2.model.DescribeVpcAttributeRequest;
import com.amazonaws.services.ec2.model.DescribeVpcAttributeResult;
import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkDnsSupportRequest;
import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkDnsSupportResult;
import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkRequest;
import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkResult;
import com.amazonaws.services.ec2.model.DescribeVpcEndpointServicesRequest;
@ -255,6 +273,8 @@ import com.amazonaws.services.ec2.model.DetachVolumeRequest;
import com.amazonaws.services.ec2.model.DetachVolumeResult;
import com.amazonaws.services.ec2.model.DetachVpnGatewayRequest;
import com.amazonaws.services.ec2.model.DisableVgwRoutePropagationRequest;
import com.amazonaws.services.ec2.model.DisableVpcClassicLinkDnsSupportRequest;
import com.amazonaws.services.ec2.model.DisableVpcClassicLinkDnsSupportResult;
import com.amazonaws.services.ec2.model.DisableVpcClassicLinkRequest;
import com.amazonaws.services.ec2.model.DisableVpcClassicLinkResult;
import com.amazonaws.services.ec2.model.DisassociateAddressRequest;
@ -263,6 +283,8 @@ import com.amazonaws.services.ec2.model.DryRunResult;
import com.amazonaws.services.ec2.model.DryRunSupportedRequest;
import com.amazonaws.services.ec2.model.EnableVgwRoutePropagationRequest;
import com.amazonaws.services.ec2.model.EnableVolumeIORequest;
import com.amazonaws.services.ec2.model.EnableVpcClassicLinkDnsSupportRequest;
import com.amazonaws.services.ec2.model.EnableVpcClassicLinkDnsSupportResult;
import com.amazonaws.services.ec2.model.EnableVpcClassicLinkRequest;
import com.amazonaws.services.ec2.model.EnableVpcClassicLinkResult;
import com.amazonaws.services.ec2.model.Filter;
@ -283,8 +305,13 @@ import com.amazonaws.services.ec2.model.ImportVolumeResult;
import com.amazonaws.services.ec2.model.Instance;
import com.amazonaws.services.ec2.model.InstanceState;
import com.amazonaws.services.ec2.model.InstanceStateName;
import com.amazonaws.services.ec2.model.ModifyHostsRequest;
import com.amazonaws.services.ec2.model.ModifyHostsResult;
import com.amazonaws.services.ec2.model.ModifyIdFormatRequest;
import com.amazonaws.services.ec2.model.ModifyImageAttributeRequest;
import com.amazonaws.services.ec2.model.ModifyInstanceAttributeRequest;
import com.amazonaws.services.ec2.model.ModifyInstancePlacementRequest;
import com.amazonaws.services.ec2.model.ModifyInstancePlacementResult;
import com.amazonaws.services.ec2.model.ModifyNetworkInterfaceAttributeRequest;
import com.amazonaws.services.ec2.model.ModifyReservedInstancesRequest;
import com.amazonaws.services.ec2.model.ModifyReservedInstancesResult;
@ -302,12 +329,16 @@ import com.amazonaws.services.ec2.model.MoveAddressToVpcRequest;
import com.amazonaws.services.ec2.model.MoveAddressToVpcResult;
import com.amazonaws.services.ec2.model.PurchaseReservedInstancesOfferingRequest;
import com.amazonaws.services.ec2.model.PurchaseReservedInstancesOfferingResult;
import com.amazonaws.services.ec2.model.PurchaseScheduledInstancesRequest;
import com.amazonaws.services.ec2.model.PurchaseScheduledInstancesResult;
import com.amazonaws.services.ec2.model.RebootInstancesRequest;
import com.amazonaws.services.ec2.model.RegisterImageRequest;
import com.amazonaws.services.ec2.model.RegisterImageResult;
import com.amazonaws.services.ec2.model.RejectVpcPeeringConnectionRequest;
import com.amazonaws.services.ec2.model.RejectVpcPeeringConnectionResult;
import com.amazonaws.services.ec2.model.ReleaseAddressRequest;
import com.amazonaws.services.ec2.model.ReleaseHostsRequest;
import com.amazonaws.services.ec2.model.ReleaseHostsResult;
import com.amazonaws.services.ec2.model.ReplaceNetworkAclAssociationRequest;
import com.amazonaws.services.ec2.model.ReplaceNetworkAclAssociationResult;
import com.amazonaws.services.ec2.model.ReplaceNetworkAclEntryRequest;
@ -330,6 +361,8 @@ import com.amazonaws.services.ec2.model.RevokeSecurityGroupEgressRequest;
import com.amazonaws.services.ec2.model.RevokeSecurityGroupIngressRequest;
import com.amazonaws.services.ec2.model.RunInstancesRequest;
import com.amazonaws.services.ec2.model.RunInstancesResult;
import com.amazonaws.services.ec2.model.RunScheduledInstancesRequest;
import com.amazonaws.services.ec2.model.RunScheduledInstancesResult;
import com.amazonaws.services.ec2.model.StartInstancesRequest;
import com.amazonaws.services.ec2.model.StartInstancesResult;
import com.amazonaws.services.ec2.model.StopInstancesRequest;
@ -519,6 +552,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DeleteNatGatewayResult deleteNatGateway(DeleteNatGatewayRequest deleteNatGatewayRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public UnmonitorInstancesResult unmonitorInstances(UnmonitorInstancesRequest unmonitorInstancesRequest) throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -639,6 +677,16 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public ModifyHostsResult modifyHosts(ModifyHostsRequest modifyHostsRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public void modifyIdFormat(ModifyIdFormatRequest modifyIdFormatRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeSecurityGroupsResult describeSecurityGroups(DescribeSecurityGroupsRequest describeSecurityGroupsRequest) throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -964,6 +1012,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DisableVpcClassicLinkDnsSupportResult disableVpcClassicLinkDnsSupport(DisableVpcClassicLinkDnsSupportRequest disableVpcClassicLinkDnsSupportRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeInstanceAttributeResult describeInstanceAttribute(DescribeInstanceAttributeRequest describeInstanceAttributeRequest) throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -989,6 +1042,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public RunScheduledInstancesResult runScheduledInstances(RunScheduledInstancesRequest runScheduledInstancesRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeSubnetsResult describeSubnets(DescribeSubnetsRequest describeSubnetsRequest) throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -1024,6 +1082,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public ModifyInstancePlacementResult modifyInstancePlacement(ModifyInstancePlacementRequest modifyInstancePlacementRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public CancelReservedInstancesListingResult cancelReservedInstancesListing(CancelReservedInstancesListingRequest cancelReservedInstancesListingRequest) throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -1159,6 +1222,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public PurchaseScheduledInstancesResult purchaseScheduledInstances(PurchaseScheduledInstancesRequest purchaseScheduledInstancesRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public void modifySnapshotAttribute(ModifySnapshotAttributeRequest modifySnapshotAttributeRequest) throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -1334,6 +1402,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public ReleaseHostsResult releaseHosts(ReleaseHostsRequest releaseHostsRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public void resetInstanceAttribute(ResetInstanceAttributeRequest resetInstanceAttributeRequest) throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -1344,6 +1417,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public CreateNatGatewayResult createNatGateway(CreateNatGatewayRequest createNatGatewayRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public void replaceNetworkAclEntry(ReplaceNetworkAclEntryRequest replaceNetworkAclEntryRequest) throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -1374,6 +1452,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public EnableVpcClassicLinkDnsSupportResult enableVpcClassicLinkDnsSupport(EnableVpcClassicLinkDnsSupportRequest enableVpcClassicLinkDnsSupportRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public void createVpnConnectionRoute(CreateVpnConnectionRouteRequest createVpnConnectionRouteRequest) throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -1489,6 +1572,16 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeScheduledInstanceAvailabilityResult describeScheduledInstanceAvailability(DescribeScheduledInstanceAvailabilityRequest describeScheduledInstanceAvailabilityRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeScheduledInstancesResult describeScheduledInstances(DescribeScheduledInstancesRequest describeScheduledInstancesRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeDhcpOptionsResult describeDhcpOptions() throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -1529,6 +1622,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeNatGatewaysResult describeNatGateways(DescribeNatGatewaysRequest describeNatGatewaysRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeConversionTasksResult describeConversionTasks() throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -1544,6 +1642,26 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeHostsResult describeHosts(DescribeHostsRequest describeHostsRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeHostsResult describeHosts() {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeIdFormatResult describeIdFormat(DescribeIdFormatRequest describeIdFormatRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeIdFormatResult describeIdFormat() {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeVpcPeeringConnectionsResult describeVpcPeeringConnections() throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -1584,6 +1702,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeVpcClassicLinkDnsSupportResult describeVpcClassicLinkDnsSupport(DescribeVpcClassicLinkDnsSupportRequest describeVpcClassicLinkDnsSupportRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeClassicLinkInstancesResult describeClassicLinkInstances() throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");
@ -1649,6 +1772,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public AllocateHostsResult allocateHosts(AllocateHostsRequest allocateHostsRequest) {
throw new UnsupportedOperationException("Not supported in mock");
}
@Override
public DescribeSnapshotsResult describeSnapshots() throws AmazonServiceException, AmazonClientException {
throw new UnsupportedOperationException("Not supported in mock");

12
plugins/repository-s3/build.gradle
View File

@ -23,7 +23,7 @@ esplugin {
}
versions << [
'aws': '1.10.33'
'aws': '1.10.69'
]
dependencies {
@ -54,11 +54,6 @@ test {
}
thirdPartyAudit.excludes = [
// uses internal java api: com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl
// uses internal java api: com.sun.org.apache.xml.internal.dtm.ref.DTMManagerDefault
// uses internal java api: com.sun.org.apache.xpath.internal.XPathContext
'com.amazonaws.util.XpathUtils',
// classes are missing
'javax.servlet.ServletContextEvent',
'javax.servlet.ServletContextListener',
@ -169,3 +164,8 @@ thirdPartyAudit.excludes = [
'javax.xml.bind.util.Messages',
'javax.xml.bind.util.ValidationEventCollector'
]
// AWS SDK is exposing some deprecated methods which we call using a delegate:
// * setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation)
// * changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass)
compileTestJava.options.compilerArgs << "-Xlint:-deprecation"

1
plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1
View File

@ -1 +0,0 @@
fabedbbe2b834b1add150b6a38395c5ef7380168

1
plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1 Normal file
View File

@ -0,0 +1 @@
a1f02d5f26ba1d8c37e2bf9c847db3c6729dda00

1
plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1
View File

@ -1 +0,0 @@
35881245894ecc4d893c074eacdf2e6b56820fda

1
plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1 Normal file
View File

@ -0,0 +1 @@
ed74ff3872193b4704a751f0b72ab2cf0db0651b

1
plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1
View File

@ -1 +0,0 @@
5665cf77102a932a16e99ebf41d197e03ddbf25c

1
plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1 Normal file
View File

@ -0,0 +1 @@
6fa48bf0bff43f26436956b88d8d3764b6cf109e

3
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
View File

@ -154,5 +154,6 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
}
AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries);
AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
boolean useThrottleRetries);
}

9
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
View File

@ -57,7 +57,8 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
}
@Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
boolean useThrottleRetries) {
if (Strings.isNullOrEmpty(endpoint)) {
// We need to set the endpoint based on the region
if (region != null) {
@ -69,10 +70,11 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
}
}
return getClient(endpoint, protocol, account, key, maxRetries);
return getClient(endpoint, protocol, account, key, maxRetries, useThrottleRetries);
}
private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries) {
private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries,
boolean useThrottleRetries) {
Tuple<String, String> clientDescriptor = new Tuple<>(endpoint, account);
AmazonS3Client client = clients.get(clientDescriptor);
if (client != null) {
@ -102,6 +104,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
// If not explicitly set, default to 3 with exponential backoff policy
clientConfiguration.setMaxErrorRetry(maxRetries);
}
clientConfiguration.setUseThrottleRetries(useThrottleRetries);
// #155: we might have 3rd party users using older S3 API version
String awsSigner = CLOUD_S3.SIGNER_SETTING.get(settings);

1
plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
View File

@ -121,6 +121,7 @@ public class S3RepositoryPlugin extends Plugin {
S3Repository.Repositories.STORAGE_CLASS_SETTING,
S3Repository.Repositories.CANNED_ACL_SETTING,
S3Repository.Repositories.BASE_PATH_SETTING,
S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING,
// Register S3 single repository settings
S3Repository.Repository.KEY_SETTING,

21
plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.repositories.s3;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import org.elasticsearch.cloud.aws.AwsS3Service;
import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3;
@ -115,6 +116,11 @@ public class S3Repository extends BlobStoreRepository {
* repositories.s3.max_retries: Number of retries in case of S3 errors. Defaults to 3.
*/
Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope);
/**
* repositories.s3.use_throttle_retries: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
*/
Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("repositories.s3.use_throttle_retries",
ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
/**
* repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g.
*/
@ -196,6 +202,12 @@ public class S3Repository extends BlobStoreRepository {
* @see Repositories#MAX_RETRIES_SETTING
*/
Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3, Property.NodeScope);
/**
* use_throttle_retries
* @see Repositories#USE_THROTTLE_RETRIES_SETTING
*/
Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("use_throttle_retries",
ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
/**
* chunk_size
* @see Repositories#CHUNK_SIZE_SETTING
@ -262,6 +274,7 @@ public class S3Repository extends BlobStoreRepository {
boolean serverSideEncryption = getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
ByteSizeValue bufferSize = getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING);
Integer maxRetries = getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING);
boolean useThrottleRetries = getValue(repositorySettings, Repository.USE_THROTTLE_RETRIES_SETTING, Repositories.USE_THROTTLE_RETRIES_SETTING);
this.chunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING);
this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING);
@ -275,13 +288,15 @@ public class S3Repository extends BlobStoreRepository {
String storageClass = getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING);
String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING);
logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]",
bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " +
"buffer_size [{}], max_retries [{}], use_throttle_retries [{}], cannedACL [{}], storageClass [{}]",
bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
storageClass);
String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING);
blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries),
blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries, useThrottleRetries),
bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);

18
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java
View File

@ -50,6 +50,7 @@ import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.model.DeleteBucketCrossOriginConfigurationRequest;
import com.amazonaws.services.s3.model.DeleteBucketLifecycleConfigurationRequest;
import com.amazonaws.services.s3.model.DeleteBucketPolicyRequest;
import com.amazonaws.services.s3.model.DeleteBucketReplicationConfigurationRequest;
import com.amazonaws.services.s3.model.DeleteBucketRequest;
import com.amazonaws.services.s3.model.DeleteBucketTaggingConfigurationRequest;
import com.amazonaws.services.s3.model.DeleteBucketWebsiteConfigurationRequest;
@ -69,6 +70,7 @@ import com.amazonaws.services.s3.model.GetBucketReplicationConfigurationRequest;
import com.amazonaws.services.s3.model.GetBucketTaggingConfigurationRequest;
import com.amazonaws.services.s3.model.GetBucketVersioningConfigurationRequest;
import com.amazonaws.services.s3.model.GetBucketWebsiteConfigurationRequest;
import com.amazonaws.services.s3.model.GetObjectAclRequest;
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.GetS3AccountOwnerRequest;
@ -253,6 +255,11 @@ public class AmazonS3Wrapper implements AmazonS3 {
return delegate.getObjectAcl(bucketName, key, versionId);
}
@Override
public AccessControlList getObjectAcl(GetObjectAclRequest getObjectAclRequest) throws AmazonClientException, AmazonServiceException {
return delegate.getObjectAcl(getObjectAclRequest);
}
@Override
public void setObjectAcl(String bucketName, String key, AccessControlList acl) throws AmazonClientException, AmazonServiceException {
delegate.setObjectAcl(bucketName, key, acl);
@ -358,6 +365,17 @@ public class AmazonS3Wrapper implements AmazonS3 {
delegate.deleteBucketReplicationConfiguration(bucketName);
}
@Override
public void deleteBucketReplicationConfiguration(DeleteBucketReplicationConfigurationRequest request) throws AmazonServiceException,
AmazonClientException {
delegate.deleteBucketReplicationConfiguration(request);
}
@Override
public boolean doesObjectExist(String bucketName, String objectName) throws AmazonServiceException, AmazonClientException {
return delegate.doesObjectExist(bucketName, objectName);
}
@Override
public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException {
return delegate.putObject(putObjectRequest);

5
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
View File

@ -43,8 +43,9 @@ public class TestAwsS3Service extends InternalAwsS3Service {
@Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries));
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
boolean useThrottleRetries) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, useThrottleRetries));
}
private AmazonS3 cachedWrapper(AmazonS3 client) {

5
plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
View File

@ -200,7 +200,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
S3Repository.Repositories.REGION_SETTING.get(settings),
S3Repository.Repositories.KEY_SETTING.get(settings),
S3Repository.Repositories.SECRET_SETTING.get(settings),
null);
null, randomBoolean());
String bucketName = bucket.get("bucket");
logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
@ -475,7 +475,8 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
// We check that settings has been set in elasticsearch.yml integration test file
// as described in README
assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null);
AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey,
null, randomBoolean());
try {
ObjectListing prevListing = null;
//From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html

2
plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
View File

@ -14,7 +14,7 @@
canned_acl: "public-read"
storage_class: "standard"
# Get repositry
# Get repository
- do:
snapshot.get_repository:
repository: test_repo_s3_1