honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix after review 

* changes `throttle_retries` to `use_throttle_retries`
* removes registering of all individual repository settings when the plugin starts. Not needed
* adds more comment about deprecated method in AWS SDK we need to implement though in a Delegate class within our tests
This commit is contained in:
David Pilato 2016-05-27 10:13:16 +02:00
parent f4cd3bd348
commit a445654123
7 changed files with 22 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/plugins/repository-s3.asciidoc
View File

@ -218,7 +218,7 @@ The following settings are supported:
Number of retries in case of S3 errors. Defaults to `3`.
`throttle_retries`::
`use_throttle_retries`::
Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
@ -243,7 +243,8 @@ The following settings are supported:
different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide]
Note that you can define S3 repository settings for all S3 repositories in `elasticsearch.yml` configuration file.
They are all prefixed with `repositories.s3.`.
They are all prefixed with `repositories.s3.`. For example, you can define compression for all S3 repositories
by setting `repositories.s3.compress: true` in `elasticsearch.yml`.
The S3 repositories use the same credentials as the rest of the AWS services
provided by this plugin (`discovery`). See <<repository-s3-usage>> for details.

4
plugins/repository-s3/build.gradle
View File

@ -57,5 +57,7 @@ thirdPartyAudit.excludes = [
'org.apache.log.Logger',
]
// AWS SDK is exposing some deprecated methods which we call using a delegate
// AWS SDK is exposing some deprecated methods which we call using a delegate:
// * setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation)
// * changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass)
compileTestJava.options.compilerArgs << "-Xlint:-deprecation"

2
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
View File

@ -154,5 +154,5 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
}
AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
boolean throttleRetries);
boolean useThrottleRetries);
}

8
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
View File

@ -58,7 +58,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
@Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
boolean throttleRetries) {
boolean useThrottleRetries) {
if (Strings.isNullOrEmpty(endpoint)) {
// We need to set the endpoint based on the region
if (region != null) {
@ -70,11 +70,11 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
}
}
return getClient(endpoint, protocol, account, key, maxRetries, throttleRetries);
return getClient(endpoint, protocol, account, key, maxRetries, useThrottleRetries);
}
private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries,
boolean throttleRetries) {
boolean useThrottleRetries) {
Tuple<String, String> clientDescriptor = new Tuple<>(endpoint, account);
AmazonS3Client client = clients.get(clientDescriptor);
if (client != null) {
@ -104,7 +104,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
// If not explicitly set, default to 3 with exponential backoff policy
clientConfiguration.setMaxErrorRetry(maxRetries);
}
clientConfiguration.setUseThrottleRetries(throttleRetries);
clientConfiguration.setUseThrottleRetries(useThrottleRetries);
// #155: we might have 3rd party users using older S3 API version
String awsSigner = CLOUD_S3.SIGNER_SETTING.get(settings);

19
plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
View File

@ -128,24 +128,7 @@ public class S3RepositoryPlugin extends Plugin {
settingsModule.registerSetting(S3Repository.Repositories.STORAGE_CLASS_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.CANNED_ACL_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.BASE_PATH_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.THROTTLE_RETRIES_SETTING);
// Register S3 single repository settings
settingsModule.registerSetting(S3Repository.Repository.KEY_SETTING);
settingsModule.registerSetting(S3Repository.Repository.SECRET_SETTING);
settingsModule.registerSetting(S3Repository.Repository.BUCKET_SETTING);
settingsModule.registerSetting(S3Repository.Repository.ENDPOINT_SETTING);
settingsModule.registerSetting(S3Repository.Repository.PROTOCOL_SETTING);
settingsModule.registerSetting(S3Repository.Repository.REGION_SETTING);
settingsModule.registerSetting(S3Repository.Repository.SERVER_SIDE_ENCRYPTION_SETTING);
settingsModule.registerSetting(S3Repository.Repository.BUFFER_SIZE_SETTING);
settingsModule.registerSetting(S3Repository.Repository.MAX_RETRIES_SETTING);
settingsModule.registerSetting(S3Repository.Repository.CHUNK_SIZE_SETTING);
settingsModule.registerSetting(S3Repository.Repository.COMPRESS_SETTING);
settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING);
settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING);
settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING);
settingsModule.registerSetting(S3Repository.Repository.THROTTLE_RETRIES_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING);
}
/**

16
plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View File

@ -117,9 +117,9 @@ public class S3Repository extends BlobStoreRepository {
*/
Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope);
/**
* repositories.s3.throttle_retries: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
* repositories.s3.use_throttle_retries: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
*/
Setting<Boolean> THROTTLE_RETRIES_SETTING = Setting.boolSetting("repositories.s3.throttle_retries",
Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("repositories.s3.use_throttle_retries",
ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
/**
* repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g.
@ -203,10 +203,10 @@ public class S3Repository extends BlobStoreRepository {
*/
Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3, Property.NodeScope);
/**
* throttle_retries
* @see Repositories#THROTTLE_RETRIES_SETTING
* use_throttle_retries
* @see Repositories#USE_THROTTLE_RETRIES_SETTING
*/
Setting<Boolean> THROTTLE_RETRIES_SETTING = Setting.boolSetting("throttle_retries",
Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("use_throttle_retries",
ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
/**
* chunk_size
@ -274,7 +274,7 @@ public class S3Repository extends BlobStoreRepository {
boolean serverSideEncryption = getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
ByteSizeValue bufferSize = getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING);
Integer maxRetries = getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING);
boolean throttleRetries = getValue(repositorySettings, Repository.THROTTLE_RETRIES_SETTING, Repositories.THROTTLE_RETRIES_SETTING);
boolean useThrottleRetries = getValue(repositorySettings, Repository.USE_THROTTLE_RETRIES_SETTING, Repositories.USE_THROTTLE_RETRIES_SETTING);
this.chunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING);
this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING);
@ -290,13 +290,13 @@ public class S3Repository extends BlobStoreRepository {
logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " +
"buffer_size [{}], max_retries [{}], throttle_retries [{}], cannedACL [{}], storageClass [{}]",
bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, throttleRetries, cannedACL,
bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
storageClass);
String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING);
blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries, throttleRetries),
blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries, useThrottleRetries),
bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);

4
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
View File

@ -52,8 +52,8 @@ public class TestAwsS3Service extends InternalAwsS3Service {
@Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
boolean throttleRetries) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, throttleRetries));
boolean useThrottleRetries) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, useThrottleRetries));
}
private AmazonS3 cachedWrapper(AmazonS3 client) {