honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'fix/15268-proxy-auth'

This commit is contained in:
David Pilato 2015-12-09 23:21:00 +01:00
parent f675801b26 7dcb40bcac
commit 414fccb7d1
9 changed files with 168 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
docs/plugins/discovery-ec2.asciidoc
View File

@ -64,16 +64,19 @@ cloud:
protocol: https
----
In addition, a proxy can be configured with the `proxy_host` and `proxy_port` settings (note that protocol can be
`http` or `https`):
In addition, a proxy can be configured with the `proxy.host`, `proxy.port`, `proxy.username` and `proxy.password` settings
(note that protocol can be `http` or `https`):
[source,yaml]
----
cloud:
aws:
protocol: https
proxy_host: proxy1.company.com
proxy_port: 8083
proxy:
host: proxy1.company.com
port: 8083
username: myself
password: theBestPasswordEver!
----
You can also set different proxies for `ec2` and `s3`:
@ -83,11 +86,17 @@ You can also set different proxies for `ec2` and `s3`:
cloud:
aws:
s3:
proxy_host: proxy1.company.com
proxy_port: 8083
proxy:
host: proxy1.company.com
port: 8083
username: myself1
password: theBestPasswordEver1!
ec2:
proxy_host: proxy2.company.com
proxy_port: 8083
proxy:
host: proxy2.company.com
port: 8083
username: myself2
password: theBestPasswordEver2!
----
[[discovery-ec2-usage-region]]

25
docs/plugins/repository-s3.asciidoc
View File

@ -67,16 +67,19 @@ cloud:
protocol: https
----
In addition, a proxy can be configured with the `proxy_host` and `proxy_port` settings (note that protocol can be
`http` or `https`):
In addition, a proxy can be configured with the `proxy.host`, `proxy.port`, `proxy.username` and `proxy.password` settings
(note that protocol can be `http` or `https`):
[source,yaml]
----
cloud:
aws:
protocol: https
proxy_host: proxy1.company.com
proxy_port: 8083
proxy:
host: proxy1.company.com
port: 8083
username: myself
password: theBestPasswordEver!
----
You can also set different proxies for `ec2` and `s3`:
@ -86,11 +89,17 @@ You can also set different proxies for `ec2` and `s3`:
cloud:
aws:
s3:
proxy_host: proxy1.company.com
proxy_port: 8083
proxy:
host: proxy1.company.com
port: 8083
username: myself1
password: theBestPasswordEver1!
ec2:
proxy_host: proxy2.company.com
proxy_port: 8083
proxy:
host: proxy2.company.com
port: 8083
username: myself2
password: theBestPasswordEver2!
----
[[repository-s3-usage-region]]

9
docs/reference/migration/migrate_3_0.asciidoc
View File

@ -244,6 +244,15 @@ Cloud AWS plugin has been split in two plugins:
* {plugins}/discovery-ec2.html[Discovery EC2 plugin]
* {plugins}/repository-s3.html[Repository S3 plugin]
Proxy settings for both plugins have been renamed:
* from `cloud.aws.proxy_host` to `cloud.aws.proxy.host`
* from `cloud.aws.ec2.proxy_host` to `cloud.aws.ec2.proxy.host`
* from `cloud.aws.s3.proxy_host` to `cloud.aws.s3.proxy.host`
* from `cloud.aws.proxy_port` to `cloud.aws.proxy.port`
* from `cloud.aws.ec2.proxy_port` to `cloud.aws.ec2.proxy.port`
* from `cloud.aws.s3.proxy_port` to `cloud.aws.s3.proxy.port`
==== Cloud Azure plugin changes
Cloud Azure plugin has been split in three plugins:

2
plugins/discovery-ec2/build.gradle
View File

@ -42,7 +42,7 @@ dependencyLicenses {
mapping from: /jackson-.*/, to: 'jackson'
}
compileJava.options.compilerArgs << '-Xlint:-rawtypes'
compileJava.options.compilerArgs << '-Xlint:-rawtypes,-deprecation'
test {
// this is needed for insecure plugins, remove if possible!

20
plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
View File

@ -27,20 +27,32 @@ public interface AwsEc2Service extends LifecycleComponent<AwsEc2Service> {
public static final String KEY = "cloud.aws.access_key";
public static final String SECRET = "cloud.aws.secret_key";
public static final String PROTOCOL = "cloud.aws.protocol";
public static final String PROXY_HOST = "cloud.aws.proxy_host";
public static final String PROXY_PORT = "cloud.aws.proxy_port";
public static final String PROXY_HOST = "cloud.aws.proxy.host";
public static final String PROXY_PORT = "cloud.aws.proxy.port";
public static final String PROXY_USERNAME = "cloud.aws.proxy.username";
public static final String PROXY_PASSWORD = "cloud.aws.proxy.password";
public static final String SIGNER = "cloud.aws.signer";
public static final String REGION = "cloud.aws.region";
@Deprecated
public static final String DEPRECATED_PROXY_HOST = "cloud.aws.proxy_host";
@Deprecated
public static final String DEPRECATED_PROXY_PORT = "cloud.aws.proxy_port";
}
final class CLOUD_EC2 {
public static final String KEY = "cloud.aws.ec2.access_key";
public static final String SECRET = "cloud.aws.ec2.secret_key";
public static final String PROTOCOL = "cloud.aws.ec2.protocol";
public static final String PROXY_HOST = "cloud.aws.ec2.proxy_host";
public static final String PROXY_PORT = "cloud.aws.ec2.proxy_port";
public static final String PROXY_HOST = "cloud.aws.ec2.proxy.host";
public static final String PROXY_PORT = "cloud.aws.ec2.proxy.port";
public static final String PROXY_USERNAME = "cloud.aws.ec2.proxy.username";
public static final String PROXY_PASSWORD = "cloud.aws.ec2.proxy.password";
public static final String SIGNER = "cloud.aws.ec2.signer";
public static final String ENDPOINT = "cloud.aws.ec2.endpoint";
@Deprecated
public static final String DEPRECATED_PROXY_HOST = "cloud.aws.ec2.proxy_host";
@Deprecated
public static final String DEPRECATED_PROXY_PORT = "cloud.aws.ec2.proxy_port";
}
final class DISCOVERY_EC2 {

17
plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
View File

@ -56,8 +56,10 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent<AwsEc2Service>
// Filter global settings
settingsFilter.addFilter(CLOUD_AWS.KEY);
settingsFilter.addFilter(CLOUD_AWS.SECRET);
settingsFilter.addFilter(CLOUD_AWS.PROXY_PASSWORD);
settingsFilter.addFilter(CLOUD_EC2.KEY);
settingsFilter.addFilter(CLOUD_EC2.SECRET);
settingsFilter.addFilter(CLOUD_EC2.PROXY_PASSWORD);
// add specific ec2 name resolver
networkService.addCustomNameResolver(new Ec2NameResolver(settings));
discoveryNodeService.addCustomAttributeProvider(new Ec2CustomNodeAttributes(settings));
@ -83,16 +85,25 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent<AwsEc2Service>
String account = settings.get(CLOUD_EC2.KEY, settings.get(CLOUD_AWS.KEY));
String key = settings.get(CLOUD_EC2.SECRET, settings.get(CLOUD_AWS.SECRET));
String proxyHost = settings.get(CLOUD_EC2.PROXY_HOST, settings.get(CLOUD_AWS.PROXY_HOST));
String proxyHost = settings.get(CLOUD_AWS.PROXY_HOST, settings.get(CLOUD_AWS.DEPRECATED_PROXY_HOST));
proxyHost = settings.get(CLOUD_EC2.PROXY_HOST, settings.get(CLOUD_EC2.DEPRECATED_PROXY_HOST, proxyHost));
if (proxyHost != null) {
String portString = settings.get(CLOUD_EC2.PROXY_PORT, settings.get(CLOUD_AWS.PROXY_PORT, "80"));
String portString = settings.get(CLOUD_AWS.PROXY_PORT, settings.get(CLOUD_AWS.DEPRECATED_PROXY_PORT, "80"));
portString = settings.get(CLOUD_EC2.PROXY_PORT, settings.get(CLOUD_EC2.DEPRECATED_PROXY_PORT, portString));
Integer proxyPort;
try {
proxyPort = Integer.parseInt(portString, 10);
} catch (NumberFormatException ex) {
throw new IllegalArgumentException("The configured proxy port value [" + portString + "] is invalid", ex);
}
clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort);
String proxyUsername = settings.get(CLOUD_EC2.PROXY_USERNAME, settings.get(CLOUD_AWS.PROXY_USERNAME));
String proxyPassword = settings.get(CLOUD_EC2.PROXY_PASSWORD, settings.get(CLOUD_AWS.PROXY_PASSWORD));
clientConfiguration
.withProxyHost(proxyHost)
.withProxyPort(proxyPort)
.withProxyUsername(proxyUsername)
.withProxyPassword(proxyPassword);
}
// #155: we might have 3rd party users using older EC2 API version

50
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
View File

@ -26,6 +26,56 @@ import org.elasticsearch.common.component.LifecycleComponent;
*
*/
public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
final class CLOUD_AWS {
public static final String KEY = "cloud.aws.access_key";
public static final String SECRET = "cloud.aws.secret_key";
public static final String PROTOCOL = "cloud.aws.protocol";
public static final String PROXY_HOST = "cloud.aws.proxy.host";
public static final String PROXY_PORT = "cloud.aws.proxy.port";
public static final String PROXY_USERNAME = "cloud.aws.proxy.username";
public static final String PROXY_PASSWORD = "cloud.aws.proxy.password";
public static final String SIGNER = "cloud.aws.signer";
public static final String REGION = "cloud.aws.region";
@Deprecated
public static final String DEPRECATED_PROXY_HOST = "cloud.aws.proxy_host";
@Deprecated
public static final String DEPRECATED_PROXY_PORT = "cloud.aws.proxy_port";
}
final class CLOUD_S3 {
public static final String KEY = "cloud.aws.s3.access_key";
public static final String SECRET = "cloud.aws.s3.secret_key";
public static final String PROTOCOL = "cloud.aws.s3.protocol";
public static final String PROXY_HOST = "cloud.aws.s3.proxy.host";
public static final String PROXY_PORT = "cloud.aws.s3.proxy.port";
public static final String PROXY_USERNAME = "cloud.aws.s3.proxy.username";
public static final String PROXY_PASSWORD = "cloud.aws.s3.proxy.password";
public static final String SIGNER = "cloud.aws.s3.signer";
public static final String ENDPOINT = "cloud.aws.s3.endpoint";
@Deprecated
public static final String DEPRECATED_PROXY_HOST = "cloud.aws.s3.proxy_host";
@Deprecated
public static final String DEPRECATED_PROXY_PORT = "cloud.aws.s3.proxy_port";
}
final class REPOSITORY_S3 {
public static final String BUCKET = "repositories.s3.bucket";
public static final String ENDPOINT = "repositories.s3.endpoint";
public static final String PROTOCOL = "repositories.s3.protocol";
public static final String REGION = "repositories.s3.region";
public static final String SERVER_SIDE_ENCRYPTION = "repositories.s3.server_side_encryption";
public static final String BUFFER_SIZE = "repositories.s3.buffer_size";
public static final String MAX_RETRIES = "repositories.s3.max_retries";
public static final String CHUNK_SIZE = "repositories.s3.chunk_size";
public static final String COMPRESS = "repositories.s3.compress";
public static final String STORAGE_CLASS = "repositories.s3.storage_class";
public static final String CANNED_ACL = "repositories.s3.canned_acl";
public static final String BASE_PATH = "repositories.s3.base_path";
}
AmazonS3 client();
AmazonS3 client(String endpoint, String protocol, String region, String account, String key);

48
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
View File

@ -50,8 +50,12 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
@Inject
public InternalAwsS3Service(Settings settings, SettingsFilter settingsFilter) {
super(settings);
settingsFilter.addFilter("cloud.aws.access_key");
settingsFilter.addFilter("cloud.aws.secret_key");
settingsFilter.addFilter(CLOUD_AWS.KEY);
settingsFilter.addFilter(CLOUD_AWS.SECRET);
settingsFilter.addFilter(CLOUD_AWS.PROXY_PASSWORD);
settingsFilter.addFilter(CLOUD_S3.KEY);
settingsFilter.addFilter(CLOUD_S3.SECRET);
settingsFilter.addFilter(CLOUD_S3.PROXY_PASSWORD);
settingsFilter.addFilter("access_key");
settingsFilter.addFilter("secret_key");
}
@ -59,9 +63,8 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
@Override
public synchronized AmazonS3 client() {
String endpoint = getDefaultEndpoint();
String account = settings.get("cloud.aws.access_key");
String key = settings.get("cloud.aws.secret_key");
String account = settings.get(CLOUD_S3.KEY, settings.get(CLOUD_AWS.KEY));
String key = settings.get(CLOUD_S3.SECRET, settings.get(CLOUD_AWS.SECRET));
return getClient(endpoint, null, account, key, null);
}
@ -79,8 +82,8 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
endpoint = getDefaultEndpoint();
}
if (account == null || key == null) {
account = settings.get("cloud.aws.access_key");
key = settings.get("cloud.aws.secret_key");
account = settings.get(CLOUD_S3.KEY, settings.get(CLOUD_AWS.KEY));
key = settings.get(CLOUD_S3.SECRET, settings.get(CLOUD_AWS.SECRET));
}
return getClient(endpoint, protocol, account, key, maxRetries);
@ -99,8 +102,8 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
// but can force objects from every response to the old generation.
clientConfiguration.setResponseMetadataCacheSize(0);
if (protocol == null) {
protocol = settings.get("cloud.aws.protocol", "https").toLowerCase(Locale.ROOT);
protocol = settings.get("cloud.aws.s3.protocol", protocol).toLowerCase(Locale.ROOT);
protocol = settings.get(CLOUD_AWS.PROTOCOL, "https").toLowerCase(Locale.ROOT);
protocol = settings.get(CLOUD_S3.PROTOCOL, protocol).toLowerCase(Locale.ROOT);
}
if ("http".equals(protocol)) {
@ -111,18 +114,25 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
throw new IllegalArgumentException("No protocol supported [" + protocol + "], can either be [http] or [https]");
}
String proxyHost = settings.get("cloud.aws.proxy_host");
proxyHost = settings.get("cloud.aws.s3.proxy_host", proxyHost);
String proxyHost = settings.get(CLOUD_AWS.PROXY_HOST, settings.get(CLOUD_AWS.DEPRECATED_PROXY_HOST));
proxyHost = settings.get(CLOUD_S3.PROXY_HOST, settings.get(CLOUD_S3.DEPRECATED_PROXY_HOST, proxyHost));
if (proxyHost != null) {
String portString = settings.get("cloud.aws.proxy_port", "80");
portString = settings.get("cloud.aws.s3.proxy_port", portString);
String portString = settings.get(CLOUD_AWS.PROXY_PORT, settings.get(CLOUD_AWS.DEPRECATED_PROXY_PORT, "80"));
portString = settings.get(CLOUD_S3.PROXY_PORT, settings.get(CLOUD_S3.DEPRECATED_PROXY_PORT, portString));
Integer proxyPort;
try {
proxyPort = Integer.parseInt(portString, 10);
} catch (NumberFormatException ex) {
throw new IllegalArgumentException("The configured proxy port value [" + portString + "] is invalid", ex);
}
clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort);
String proxyUsername = settings.get(CLOUD_S3.PROXY_USERNAME, settings.get(CLOUD_AWS.PROXY_USERNAME));
String proxyPassword = settings.get(CLOUD_S3.PROXY_PASSWORD, settings.get(CLOUD_AWS.PROXY_PASSWORD));
clientConfiguration
.withProxyHost(proxyHost)
.withProxyPort(proxyPort)
.withProxyUsername(proxyUsername)
.withProxyPassword(proxyPassword);
}
if (maxRetries != null) {
@ -131,7 +141,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
}
// #155: we might have 3rd party users using older S3 API version
String awsSigner = settings.get("cloud.aws.s3.signer", settings.get("cloud.aws.signer"));
String awsSigner = settings.get(CLOUD_S3.SIGNER, settings.get(CLOUD_AWS.SIGNER));
if (awsSigner != null) {
logger.debug("using AWS API signer [{}]", awsSigner);
AwsSigner.configureSigner(awsSigner, clientConfiguration, endpoint);
@ -161,11 +171,11 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
private String getDefaultEndpoint() {
String endpoint = null;
if (settings.get("cloud.aws.s3.endpoint") != null) {
endpoint = settings.get("cloud.aws.s3.endpoint");
if (settings.get(CLOUD_S3.ENDPOINT) != null) {
endpoint = settings.get(CLOUD_S3.ENDPOINT);
logger.debug("using explicit s3 endpoint [{}]", endpoint);
} else if (settings.get("cloud.aws.region") != null) {
String region = settings.get("cloud.aws.region").toLowerCase(Locale.ROOT);
} else if (settings.get(CLOUD_AWS.REGION) != null) {
String region = settings.get(CLOUD_AWS.REGION).toLowerCase(Locale.ROOT);
endpoint = getEndpoint(region);
logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint);
}

28
plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View File

@ -20,6 +20,8 @@
package org.elasticsearch.repositories.s3;
import org.elasticsearch.cloud.aws.AwsS3Service;
import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_AWS;
import org.elasticsearch.cloud.aws.AwsS3Service.REPOSITORY_S3;
import org.elasticsearch.cloud.aws.blobstore.S3BlobStore;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.blobstore.BlobPath;
@ -73,18 +75,18 @@ public class S3Repository extends BlobStoreRepository {
public S3Repository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, AwsS3Service s3Service) throws IOException {
super(name.getName(), repositorySettings, indexShardRepository);
String bucket = repositorySettings.settings().get("bucket", settings.get("repositories.s3.bucket"));
String bucket = repositorySettings.settings().get("bucket", settings.get(REPOSITORY_S3.BUCKET));
if (bucket == null) {
throw new RepositoryException(name.name(), "No bucket defined for s3 gateway");
}
String endpoint = repositorySettings.settings().get("endpoint", settings.get("repositories.s3.endpoint"));
String protocol = repositorySettings.settings().get("protocol", settings.get("repositories.s3.protocol"));
String endpoint = repositorySettings.settings().get("endpoint", settings.get(REPOSITORY_S3.ENDPOINT));
String protocol = repositorySettings.settings().get("protocol", settings.get(REPOSITORY_S3.PROTOCOL));
String region = repositorySettings.settings().get("region", settings.get("repositories.s3.region"));
String region = repositorySettings.settings().get("region", settings.get(REPOSITORY_S3.REGION));
if (region == null) {
// Bucket setting is not set - use global region setting
String regionSetting = repositorySettings.settings().get("cloud.aws.region", settings.get("cloud.aws.region"));
String regionSetting = settings.get(CLOUD_AWS.REGION);
if (regionSetting != null) {
regionSetting = regionSetting.toLowerCase(Locale.ENGLISH);
if ("us-east".equals(regionSetting) || "us-east-1".equals(regionSetting)) {
@ -112,15 +114,15 @@ public class S3Repository extends BlobStoreRepository {
}
}
boolean serverSideEncryption = repositorySettings.settings().getAsBoolean("server_side_encryption", settings.getAsBoolean("repositories.s3.server_side_encryption", false));
ByteSizeValue bufferSize = repositorySettings.settings().getAsBytesSize("buffer_size", settings.getAsBytesSize("repositories.s3.buffer_size", null));
Integer maxRetries = repositorySettings.settings().getAsInt("max_retries", settings.getAsInt("repositories.s3.max_retries", 3));
this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", settings.getAsBytesSize("repositories.s3.chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB)));
this.compress = repositorySettings.settings().getAsBoolean("compress", settings.getAsBoolean("repositories.s3.compress", false));
boolean serverSideEncryption = repositorySettings.settings().getAsBoolean("server_side_encryption", settings.getAsBoolean(REPOSITORY_S3.SERVER_SIDE_ENCRYPTION, false));
ByteSizeValue bufferSize = repositorySettings.settings().getAsBytesSize("buffer_size", settings.getAsBytesSize(REPOSITORY_S3.BUFFER_SIZE, null));
Integer maxRetries = repositorySettings.settings().getAsInt("max_retries", settings.getAsInt(REPOSITORY_S3.MAX_RETRIES, 3));
this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", settings.getAsBytesSize(REPOSITORY_S3.CHUNK_SIZE, new ByteSizeValue(100, ByteSizeUnit.MB)));
this.compress = repositorySettings.settings().getAsBoolean("compress", settings.getAsBoolean(REPOSITORY_S3.COMPRESS, false));
// Parse and validate the user's S3 Storage Class setting
String storageClass = repositorySettings.settings().get("storage_class", settings.get("repositories.s3.storage_class", null));
String cannedACL = repositorySettings.settings().get("canned_acl", settings.get("repositories.s3.canned_acl", null));
String storageClass = repositorySettings.settings().get("storage_class", settings.get(REPOSITORY_S3.STORAGE_CLASS, null));
String cannedACL = repositorySettings.settings().get("canned_acl", settings.get(REPOSITORY_S3.CANNED_ACL, null));
logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]",
bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
@ -128,7 +130,7 @@ public class S3Repository extends BlobStoreRepository {
blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key"), maxRetries),
bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
String basePath = repositorySettings.settings().get("base_path", settings.get("repositories.s3.base_path"));
String basePath = repositorySettings.settings().get("base_path", settings.get(REPOSITORY_S3.BASE_PATH));
if (Strings.hasLength(basePath)) {
BlobPath path = new BlobPath();
for(String elem : Strings.splitStringToArray(basePath, '/')) {