honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix after review 

* changes `throttle_retries` to `use_throttle_retries`
* removes registering of all individual repository settings when the plugin starts. Not needed
* adds more comment about deprecated method in AWS SDK we need to implement though in a Delegate class within our tests
This commit is contained in:
David Pilato 2016-05-27 10:13:16 +02:00
parent f4cd3bd348
commit a445654123
7 changed files with 22 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/plugins/repository-s3.asciidoc
View File

@ -218,7 +218,7 @@ The following settings are supported:
Number of retries in case of S3 errors. Defaults to `3`. Number of retries in case of S3 errors. Defaults to `3`.
`throttle_retries`:: `use_throttle_retries`::
Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`). Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
@ -243,7 +243,8 @@ The following settings are supported:
different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide] different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide]
Note that you can define S3 repository settings for all S3 repositories in `elasticsearch.yml` configuration file. Note that you can define S3 repository settings for all S3 repositories in `elasticsearch.yml` configuration file.
They are all prefixed with `repositories.s3.`. They are all prefixed with `repositories.s3.`. For example, you can define compression for all S3 repositories
by setting `repositories.s3.compress: true` in `elasticsearch.yml`.
The S3 repositories use the same credentials as the rest of the AWS services The S3 repositories use the same credentials as the rest of the AWS services
provided by this plugin (`discovery`). See <<repository-s3-usage>> for details. provided by this plugin (`discovery`). See <<repository-s3-usage>> for details.

4
plugins/repository-s3/build.gradle
View File

@ -57,5 +57,7 @@ thirdPartyAudit.excludes = [
'org.apache.log.Logger', 'org.apache.log.Logger',
] ]
// AWS SDK is exposing some deprecated methods which we call using a delegate // AWS SDK is exposing some deprecated methods which we call using a delegate:
// * setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation)
// * changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass)
compileTestJava.options.compilerArgs << "-Xlint:-deprecation" compileTestJava.options.compilerArgs << "-Xlint:-deprecation"

2
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
View File

@ -154,5 +154,5 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
} }
AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
boolean throttleRetries); boolean useThrottleRetries);
} }

8
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
View File

@ -58,7 +58,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
@Override @Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
boolean throttleRetries) { boolean useThrottleRetries) {
if (Strings.isNullOrEmpty(endpoint)) { if (Strings.isNullOrEmpty(endpoint)) {
// We need to set the endpoint based on the region // We need to set the endpoint based on the region
if (region != null) { if (region != null) {
@ -70,11 +70,11 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
} }
} }
return getClient(endpoint, protocol, account, key, maxRetries, throttleRetries); return getClient(endpoint, protocol, account, key, maxRetries, useThrottleRetries);
} }
private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries, private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries,
boolean throttleRetries) { boolean useThrottleRetries) {
Tuple<String, String> clientDescriptor = new Tuple<>(endpoint, account); Tuple<String, String> clientDescriptor = new Tuple<>(endpoint, account);
AmazonS3Client client = clients.get(clientDescriptor); AmazonS3Client client = clients.get(clientDescriptor);
if (client != null) { if (client != null) {
@ -104,7 +104,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
// If not explicitly set, default to 3 with exponential backoff policy // If not explicitly set, default to 3 with exponential backoff policy
clientConfiguration.setMaxErrorRetry(maxRetries); clientConfiguration.setMaxErrorRetry(maxRetries);
} }
clientConfiguration.setUseThrottleRetries(throttleRetries); clientConfiguration.setUseThrottleRetries(useThrottleRetries);
// #155: we might have 3rd party users using older S3 API version // #155: we might have 3rd party users using older S3 API version
String awsSigner = CLOUD_S3.SIGNER_SETTING.get(settings); String awsSigner = CLOUD_S3.SIGNER_SETTING.get(settings);

19
plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
View File

@ -128,24 +128,7 @@ public class S3RepositoryPlugin extends Plugin {
settingsModule.registerSetting(S3Repository.Repositories.STORAGE_CLASS_SETTING); settingsModule.registerSetting(S3Repository.Repositories.STORAGE_CLASS_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.CANNED_ACL_SETTING); settingsModule.registerSetting(S3Repository.Repositories.CANNED_ACL_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.BASE_PATH_SETTING); settingsModule.registerSetting(S3Repository.Repositories.BASE_PATH_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.THROTTLE_RETRIES_SETTING); settingsModule.registerSetting(S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING);
// Register S3 single repository settings
settingsModule.registerSetting(S3Repository.Repository.KEY_SETTING);
settingsModule.registerSetting(S3Repository.Repository.SECRET_SETTING);
settingsModule.registerSetting(S3Repository.Repository.BUCKET_SETTING);
settingsModule.registerSetting(S3Repository.Repository.ENDPOINT_SETTING);
settingsModule.registerSetting(S3Repository.Repository.PROTOCOL_SETTING);
settingsModule.registerSetting(S3Repository.Repository.REGION_SETTING);
settingsModule.registerSetting(S3Repository.Repository.SERVER_SIDE_ENCRYPTION_SETTING);
settingsModule.registerSetting(S3Repository.Repository.BUFFER_SIZE_SETTING);
settingsModule.registerSetting(S3Repository.Repository.MAX_RETRIES_SETTING);
settingsModule.registerSetting(S3Repository.Repository.CHUNK_SIZE_SETTING);
settingsModule.registerSetting(S3Repository.Repository.COMPRESS_SETTING);
settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING);
settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING);
settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING);
settingsModule.registerSetting(S3Repository.Repository.THROTTLE_RETRIES_SETTING);
} }
/** /**

16
plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View File

@ -117,9 +117,9 @@ public class S3Repository extends BlobStoreRepository {
*/ */
Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope); Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope);
/** /**
* repositories.s3.throttle_retries: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`). * repositories.s3.use_throttle_retries: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
*/ */
Setting<Boolean> THROTTLE_RETRIES_SETTING = Setting.boolSetting("repositories.s3.throttle_retries", Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("repositories.s3.use_throttle_retries",
ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope); ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
/** /**
* repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g. * repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g.
@ -203,10 +203,10 @@ public class S3Repository extends BlobStoreRepository {
*/ */
Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3, Property.NodeScope); Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3, Property.NodeScope);
/** /**
* throttle_retries * use_throttle_retries
* @see Repositories#THROTTLE_RETRIES_SETTING * @see Repositories#USE_THROTTLE_RETRIES_SETTING
*/ */
Setting<Boolean> THROTTLE_RETRIES_SETTING = Setting.boolSetting("throttle_retries", Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("use_throttle_retries",
ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope); ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
/** /**
* chunk_size * chunk_size
@ -274,7 +274,7 @@ public class S3Repository extends BlobStoreRepository {
boolean serverSideEncryption = getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING); boolean serverSideEncryption = getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
ByteSizeValue bufferSize = getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING); ByteSizeValue bufferSize = getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING);
Integer maxRetries = getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING); Integer maxRetries = getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING);
boolean throttleRetries = getValue(repositorySettings, Repository.THROTTLE_RETRIES_SETTING, Repositories.THROTTLE_RETRIES_SETTING); boolean useThrottleRetries = getValue(repositorySettings, Repository.USE_THROTTLE_RETRIES_SETTING, Repositories.USE_THROTTLE_RETRIES_SETTING);
this.chunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING); this.chunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING);
this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING); this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING);
@ -290,13 +290,13 @@ public class S3Repository extends BlobStoreRepository {
logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " + logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " +
"buffer_size [{}], max_retries [{}], throttle_retries [{}], cannedACL [{}], storageClass [{}]", "buffer_size [{}], max_retries [{}], throttle_retries [{}], cannedACL [{}], storageClass [{}]",
bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, throttleRetries, cannedACL, bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
storageClass); storageClass);
String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING); String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING); String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING);
blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries, throttleRetries), blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries, useThrottleRetries),
bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass); bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING); String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);

4
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
View File

@ -52,8 +52,8 @@ public class TestAwsS3Service extends InternalAwsS3Service {
@Override @Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
boolean throttleRetries) { boolean useThrottleRetries) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, throttleRetries)); return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, useThrottleRetries));
} }
private AmazonS3 cachedWrapper(AmazonS3 client) { private AmazonS3 cachedWrapper(AmazonS3 client) {