honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for path_style_access 

From https://github.com/elastic/elasticsearch-cloud-aws/pull/159

Add a new option `path_style_access` for S3 buckets. It adds support for path style access for [virtual hosting of buckets](http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html).
Defaults to `false`.

Closes https://github.com/elastic/elasticsearch-cloud-aws/issues/124.
This commit is contained in:
David Pilato 2015-11-30 15:55:09 +01:00
parent 025e9818e7
commit c1f7171e61
5 changed files with 27 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/plugins/repository-s3.asciidoc
View File

@ -235,6 +235,11 @@ The following settings are supported:
currently supported by the plugin. For more information about the
different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide]
`path_style_access`::
Activate path style access for [virtual hosting of buckets](http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html).
Defaults to `false`.
The S3 repositories use the same credentials as the rest of the AWS services
provided by this plugin (`discovery`). See <<repository-s3-usage>> for details.

3
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
View File

@ -72,6 +72,7 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
public static final String STORAGE_CLASS = "repositories.s3.storage_class";
public static final String CANNED_ACL = "repositories.s3.canned_acl";
public static final String BASE_PATH = "repositories.s3.base_path";
public static final String PATH_STYLE_ACCESS = "repositories.s3.path_style_access";
}
@ -80,5 +81,5 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
AmazonS3 client(String endpoint, String protocol, String region, String account, String key);
AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries);
AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries, Boolean pathStyleAccess);
}

16
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
View File

@ -26,6 +26,7 @@ import com.amazonaws.http.IdleConnectionReaper;
import com.amazonaws.internal.StaticCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.S3ClientOptions;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
@ -65,16 +66,16 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
String endpoint = getDefaultEndpoint();
String account = settings.get(CLOUD_S3.KEY, settings.get(CLOUD_AWS.KEY));
String key = settings.get(CLOUD_S3.SECRET, settings.get(CLOUD_AWS.SECRET));
return getClient(endpoint, null, account, key, null);
return getClient(endpoint, null, account, key, null, null);
}
@Override
public AmazonS3 client(String endpoint, String protocol, String region, String account, String key) {
return client(endpoint, protocol, region, account, key, null);
return client(endpoint, protocol, region, account, key, null, null);
}
@Override
public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries) {
public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries, Boolean pathStyleAccess) {
if (region != null && endpoint == null) {
endpoint = getEndpoint(region);
logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint);
@ -86,11 +87,11 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
key = settings.get(CLOUD_S3.SECRET, settings.get(CLOUD_AWS.SECRET));
}
return getClient(endpoint, protocol, account, key, maxRetries);
return getClient(endpoint, protocol, account, key, maxRetries, pathStyleAccess);
}
private synchronized AmazonS3 getClient(String endpoint, String protocol, String account, String key, Integer maxRetries) {
private synchronized AmazonS3 getClient(String endpoint, String protocol, String account, String key, Integer maxRetries, Boolean pathStyleAccess) {
Tuple<String, String> clientDescriptor = new Tuple<String, String>(endpoint, account);
AmazonS3Client client = clients.get(clientDescriptor);
if (client != null) {
@ -165,6 +166,11 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
if (endpoint != null) {
client.setEndpoint(endpoint);
}
if (pathStyleAccess != null) {
client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess));
}
clients.put(clientDescriptor, client);
return client;
}

11
plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.repositories.s3;
import com.amazonaws.services.s3.AmazonS3;
import org.elasticsearch.cloud.aws.AwsS3Service;
import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_AWS;
import org.elasticsearch.cloud.aws.AwsS3Service.REPOSITORY_S3;
@ -123,12 +124,14 @@ public class S3Repository extends BlobStoreRepository {
// Parse and validate the user's S3 Storage Class setting
String storageClass = repositorySettings.settings().get("storage_class", settings.get(REPOSITORY_S3.STORAGE_CLASS, null));
String cannedACL = repositorySettings.settings().get("canned_acl", settings.get(REPOSITORY_S3.CANNED_ACL, null));
Boolean pathStyleAccess = repositorySettings.settings().getAsBoolean("path_style_access", settings.getAsBoolean(REPOSITORY_S3.PATH_STYLE_ACCESS, null));
logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]",
bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], canned_acl [{}], storage_class [{}], path_style_access [{}]",
bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass, pathStyleAccess);
blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key"), maxRetries),
bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
AmazonS3 client = s3Service.client(endpoint, protocol, region, repositorySettings.settings().get("access_key"),
repositorySettings.settings().get("secret_key"), maxRetries, pathStyleAccess);
blobStore = new S3BlobStore(settings, client, bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
String basePath = repositorySettings.settings().get("base_path", settings.get(REPOSITORY_S3.BASE_PATH));
if (Strings.hasLength(basePath)) {

4
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
View File

@ -61,8 +61,8 @@ public class TestAwsS3Service extends InternalAwsS3Service {
}
@Override
public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries));
public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries, Boolean pathStyleAccess) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, pathStyleAccess));
}
private AmazonS3 cachedWrapper(AmazonS3 client) {