From 7dcb40bcacae2b44174f2f1cb645ee5f13e30f45 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 7 Dec 2015 23:06:11 +0100 Subject: [PATCH] Add support for proxy authentication for s3 and ec2 When using S3 or EC2, it was possible to use a proxy to access EC2 or S3 API but username and password were not possible to be set. This commit adds support for this. Also, to make all that consistent, proxy settings for both plugins have been renamed: * from `cloud.aws.proxy_host` to `cloud.aws.proxy.host` * from `cloud.aws.ec2.proxy_host` to `cloud.aws.ec2.proxy.host` * from `cloud.aws.s3.proxy_host` to `cloud.aws.s3.proxy.host` * from `cloud.aws.proxy_port` to `cloud.aws.proxy.port` * from `cloud.aws.ec2.proxy_port` to `cloud.aws.ec2.proxy.port` * from `cloud.aws.s3.proxy_port` to `cloud.aws.s3.proxy.port` New settings are `proxy.username` and `proxy.password`. ```yml cloud: aws: protocol: https proxy: host: proxy1.company.com port: 8083 username: myself password: theBestPasswordEver! ``` You can also set different proxies for `ec2` and `s3`: ```yml cloud: aws: s3: proxy: host: proxy1.company.com port: 8083 username: myself1 password: theBestPasswordEver1! ec2: proxy: host: proxy2.company.com port: 8083 username: myself2 password: theBestPasswordEver2! ``` Note that `password` is filtered with `SettingsFilter`. We also fix a potential issue in S3 repository. We were supposed to accept key/secret either set under `cloud.aws` or `cloud.aws.s3` but the actual code never implemented that. It was: ```java account = settings.get("cloud.aws.access_key"); key = settings.get("cloud.aws.secret_key"); ``` We replaced that by: ```java String account = settings.get(CLOUD_S3.KEY, settings.get(CLOUD_AWS.KEY)); String key = settings.get(CLOUD_S3.SECRET, settings.get(CLOUD_AWS.SECRET)); ``` Also, we extract all settings for S3 in `AwsS3Service` as it's already the case for `AwsEc2Service` class. Closes #15268. --- docs/plugins/discovery-ec2.asciidoc | 25 +++++++--- docs/plugins/repository-s3.asciidoc | 25 +++++++--- docs/reference/migration/migrate_3_0.asciidoc | 9 ++++ plugins/discovery-ec2/build.gradle | 2 +- .../cloud/aws/AwsEc2Service.java | 20 ++++++-- .../cloud/aws/AwsEc2ServiceImpl.java | 17 +++++-- .../elasticsearch/cloud/aws/AwsS3Service.java | 50 +++++++++++++++++++ .../cloud/aws/InternalAwsS3Service.java | 48 +++++++++++------- .../repositories/s3/S3Repository.java | 28 ++++++----- 9 files changed, 168 insertions(+), 56 deletions(-) diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index a2b80495003..bdd46fb72fd 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -64,16 +64,19 @@ cloud: protocol: https ---- -In addition, a proxy can be configured with the `proxy_host` and `proxy_port` settings (note that protocol can be -`http` or `https`): +In addition, a proxy can be configured with the `proxy.host`, `proxy.port`, `proxy.username` and `proxy.password` settings +(note that protocol can be `http` or `https`): [source,yaml] ---- cloud: aws: protocol: https - proxy_host: proxy1.company.com - proxy_port: 8083 + proxy: + host: proxy1.company.com + port: 8083 + username: myself + password: theBestPasswordEver! ---- You can also set different proxies for `ec2` and `s3`: @@ -83,11 +86,17 @@ You can also set different proxies for `ec2` and `s3`: cloud: aws: s3: - proxy_host: proxy1.company.com - proxy_port: 8083 + proxy: + host: proxy1.company.com + port: 8083 + username: myself1 + password: theBestPasswordEver1! ec2: - proxy_host: proxy2.company.com - proxy_port: 8083 + proxy: + host: proxy2.company.com + port: 8083 + username: myself2 + password: theBestPasswordEver2! ---- [[discovery-ec2-usage-region]] diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 16505885446..faaa87302ee 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -67,16 +67,19 @@ cloud: protocol: https ---- -In addition, a proxy can be configured with the `proxy_host` and `proxy_port` settings (note that protocol can be -`http` or `https`): +In addition, a proxy can be configured with the `proxy.host`, `proxy.port`, `proxy.username` and `proxy.password` settings +(note that protocol can be `http` or `https`): [source,yaml] ---- cloud: aws: protocol: https - proxy_host: proxy1.company.com - proxy_port: 8083 + proxy: + host: proxy1.company.com + port: 8083 + username: myself + password: theBestPasswordEver! ---- You can also set different proxies for `ec2` and `s3`: @@ -86,11 +89,17 @@ You can also set different proxies for `ec2` and `s3`: cloud: aws: s3: - proxy_host: proxy1.company.com - proxy_port: 8083 + proxy: + host: proxy1.company.com + port: 8083 + username: myself1 + password: theBestPasswordEver1! ec2: - proxy_host: proxy2.company.com - proxy_port: 8083 + proxy: + host: proxy2.company.com + port: 8083 + username: myself2 + password: theBestPasswordEver2! ---- [[repository-s3-usage-region]] diff --git a/docs/reference/migration/migrate_3_0.asciidoc b/docs/reference/migration/migrate_3_0.asciidoc index b8683bc6fd0..2b58303919c 100644 --- a/docs/reference/migration/migrate_3_0.asciidoc +++ b/docs/reference/migration/migrate_3_0.asciidoc @@ -237,6 +237,15 @@ Cloud AWS plugin has been split in two plugins: * {plugins}/discovery-ec2.html[Discovery EC2 plugin] * {plugins}/repository-s3.html[Repository S3 plugin] +Proxy settings for both plugins have been renamed: + +* from `cloud.aws.proxy_host` to `cloud.aws.proxy.host` +* from `cloud.aws.ec2.proxy_host` to `cloud.aws.ec2.proxy.host` +* from `cloud.aws.s3.proxy_host` to `cloud.aws.s3.proxy.host` +* from `cloud.aws.proxy_port` to `cloud.aws.proxy.port` +* from `cloud.aws.ec2.proxy_port` to `cloud.aws.ec2.proxy.port` +* from `cloud.aws.s3.proxy_port` to `cloud.aws.s3.proxy.port` + ==== Cloud Azure plugin changes Cloud Azure plugin has been split in three plugins: diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index 25706619c15..77cfd6626d5 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -42,7 +42,7 @@ dependencyLicenses { mapping from: /jackson-.*/, to: 'jackson' } -compileJava.options.compilerArgs << '-Xlint:-rawtypes' +compileJava.options.compilerArgs << '-Xlint:-rawtypes,-deprecation' test { // this is needed for insecure plugins, remove if possible! diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java index a427b4af4ab..d71d9dfb0af 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java @@ -27,20 +27,32 @@ public interface AwsEc2Service extends LifecycleComponent { public static final String KEY = "cloud.aws.access_key"; public static final String SECRET = "cloud.aws.secret_key"; public static final String PROTOCOL = "cloud.aws.protocol"; - public static final String PROXY_HOST = "cloud.aws.proxy_host"; - public static final String PROXY_PORT = "cloud.aws.proxy_port"; + public static final String PROXY_HOST = "cloud.aws.proxy.host"; + public static final String PROXY_PORT = "cloud.aws.proxy.port"; + public static final String PROXY_USERNAME = "cloud.aws.proxy.username"; + public static final String PROXY_PASSWORD = "cloud.aws.proxy.password"; public static final String SIGNER = "cloud.aws.signer"; public static final String REGION = "cloud.aws.region"; + @Deprecated + public static final String DEPRECATED_PROXY_HOST = "cloud.aws.proxy_host"; + @Deprecated + public static final String DEPRECATED_PROXY_PORT = "cloud.aws.proxy_port"; } final class CLOUD_EC2 { public static final String KEY = "cloud.aws.ec2.access_key"; public static final String SECRET = "cloud.aws.ec2.secret_key"; public static final String PROTOCOL = "cloud.aws.ec2.protocol"; - public static final String PROXY_HOST = "cloud.aws.ec2.proxy_host"; - public static final String PROXY_PORT = "cloud.aws.ec2.proxy_port"; + public static final String PROXY_HOST = "cloud.aws.ec2.proxy.host"; + public static final String PROXY_PORT = "cloud.aws.ec2.proxy.port"; + public static final String PROXY_USERNAME = "cloud.aws.ec2.proxy.username"; + public static final String PROXY_PASSWORD = "cloud.aws.ec2.proxy.password"; public static final String SIGNER = "cloud.aws.ec2.signer"; public static final String ENDPOINT = "cloud.aws.ec2.endpoint"; + @Deprecated + public static final String DEPRECATED_PROXY_HOST = "cloud.aws.ec2.proxy_host"; + @Deprecated + public static final String DEPRECATED_PROXY_PORT = "cloud.aws.ec2.proxy_port"; } final class DISCOVERY_EC2 { diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java index 76c3262db3f..ec1ffd54a77 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java @@ -56,8 +56,10 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent // Filter global settings settingsFilter.addFilter(CLOUD_AWS.KEY); settingsFilter.addFilter(CLOUD_AWS.SECRET); + settingsFilter.addFilter(CLOUD_AWS.PROXY_PASSWORD); settingsFilter.addFilter(CLOUD_EC2.KEY); settingsFilter.addFilter(CLOUD_EC2.SECRET); + settingsFilter.addFilter(CLOUD_EC2.PROXY_PASSWORD); // add specific ec2 name resolver networkService.addCustomNameResolver(new Ec2NameResolver(settings)); discoveryNodeService.addCustomAttributeProvider(new Ec2CustomNodeAttributes(settings)); @@ -83,16 +85,25 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent String account = settings.get(CLOUD_EC2.KEY, settings.get(CLOUD_AWS.KEY)); String key = settings.get(CLOUD_EC2.SECRET, settings.get(CLOUD_AWS.SECRET)); - String proxyHost = settings.get(CLOUD_EC2.PROXY_HOST, settings.get(CLOUD_AWS.PROXY_HOST)); + String proxyHost = settings.get(CLOUD_AWS.PROXY_HOST, settings.get(CLOUD_AWS.DEPRECATED_PROXY_HOST)); + proxyHost = settings.get(CLOUD_EC2.PROXY_HOST, settings.get(CLOUD_EC2.DEPRECATED_PROXY_HOST, proxyHost)); if (proxyHost != null) { - String portString = settings.get(CLOUD_EC2.PROXY_PORT, settings.get(CLOUD_AWS.PROXY_PORT, "80")); + String portString = settings.get(CLOUD_AWS.PROXY_PORT, settings.get(CLOUD_AWS.DEPRECATED_PROXY_PORT, "80")); + portString = settings.get(CLOUD_EC2.PROXY_PORT, settings.get(CLOUD_EC2.DEPRECATED_PROXY_PORT, portString)); Integer proxyPort; try { proxyPort = Integer.parseInt(portString, 10); } catch (NumberFormatException ex) { throw new IllegalArgumentException("The configured proxy port value [" + portString + "] is invalid", ex); } - clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort); + String proxyUsername = settings.get(CLOUD_EC2.PROXY_USERNAME, settings.get(CLOUD_AWS.PROXY_USERNAME)); + String proxyPassword = settings.get(CLOUD_EC2.PROXY_PASSWORD, settings.get(CLOUD_AWS.PROXY_PASSWORD)); + + clientConfiguration + .withProxyHost(proxyHost) + .withProxyPort(proxyPort) + .withProxyUsername(proxyUsername) + .withProxyPassword(proxyPassword); } // #155: we might have 3rd party users using older EC2 API version diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index e5db2ed7357..711b8db9374 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -26,6 +26,56 @@ import org.elasticsearch.common.component.LifecycleComponent; * */ public interface AwsS3Service extends LifecycleComponent { + + final class CLOUD_AWS { + public static final String KEY = "cloud.aws.access_key"; + public static final String SECRET = "cloud.aws.secret_key"; + public static final String PROTOCOL = "cloud.aws.protocol"; + public static final String PROXY_HOST = "cloud.aws.proxy.host"; + public static final String PROXY_PORT = "cloud.aws.proxy.port"; + public static final String PROXY_USERNAME = "cloud.aws.proxy.username"; + public static final String PROXY_PASSWORD = "cloud.aws.proxy.password"; + public static final String SIGNER = "cloud.aws.signer"; + public static final String REGION = "cloud.aws.region"; + @Deprecated + public static final String DEPRECATED_PROXY_HOST = "cloud.aws.proxy_host"; + @Deprecated + public static final String DEPRECATED_PROXY_PORT = "cloud.aws.proxy_port"; + } + + final class CLOUD_S3 { + public static final String KEY = "cloud.aws.s3.access_key"; + public static final String SECRET = "cloud.aws.s3.secret_key"; + public static final String PROTOCOL = "cloud.aws.s3.protocol"; + public static final String PROXY_HOST = "cloud.aws.s3.proxy.host"; + public static final String PROXY_PORT = "cloud.aws.s3.proxy.port"; + public static final String PROXY_USERNAME = "cloud.aws.s3.proxy.username"; + public static final String PROXY_PASSWORD = "cloud.aws.s3.proxy.password"; + public static final String SIGNER = "cloud.aws.s3.signer"; + public static final String ENDPOINT = "cloud.aws.s3.endpoint"; + @Deprecated + public static final String DEPRECATED_PROXY_HOST = "cloud.aws.s3.proxy_host"; + @Deprecated + public static final String DEPRECATED_PROXY_PORT = "cloud.aws.s3.proxy_port"; + } + + final class REPOSITORY_S3 { + public static final String BUCKET = "repositories.s3.bucket"; + public static final String ENDPOINT = "repositories.s3.endpoint"; + public static final String PROTOCOL = "repositories.s3.protocol"; + public static final String REGION = "repositories.s3.region"; + public static final String SERVER_SIDE_ENCRYPTION = "repositories.s3.server_side_encryption"; + public static final String BUFFER_SIZE = "repositories.s3.buffer_size"; + public static final String MAX_RETRIES = "repositories.s3.max_retries"; + public static final String CHUNK_SIZE = "repositories.s3.chunk_size"; + public static final String COMPRESS = "repositories.s3.compress"; + public static final String STORAGE_CLASS = "repositories.s3.storage_class"; + public static final String CANNED_ACL = "repositories.s3.canned_acl"; + public static final String BASE_PATH = "repositories.s3.base_path"; + } + + + AmazonS3 client(); AmazonS3 client(String endpoint, String protocol, String region, String account, String key); diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 4752a3f80b2..7d0b72cd63c 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -50,8 +50,12 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent