honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix ec2 settings 

Follow up for #18662

We add some tests to check that settings are correctly applied.
Tests revealed that some checks were missing.

But we ignore `testAWSCredentialsWithSystemProviders` test for now.
This commit is contained in:
David Pilato 2016-06-01 22:44:39 +02:00
parent 0216cef7bd
commit 0578925423
3 changed files with 279 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
plugins/discovery-ec2/build.gradle
View File

@ -44,7 +44,11 @@ dependencyLicenses {
test {
// this is needed for insecure plugins, remove if possible!
systemProperty 'tests.artifact', project.name
systemProperty 'tests.artifact', project.name
// this could be needed by AwsEc2ServiceImplTests#testAWSCredentialsWithSystemProviders()
// As it's marked as Ignored for now, we can comment those
// systemProperty 'aws.accessKeyId', 'DUMMY_ACCESS_KEY'
// systemProperty 'aws.secretKey', 'DUMMY_SECRET_KEY'
}
thirdPartyAudit.excludes = [

171
plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
View File

@ -39,6 +39,7 @@ import org.elasticsearch.common.Randomness;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.network.NetworkService;
import org.elasticsearch.common.settings.Settings;
@ -66,15 +67,45 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent implements Aws
return client;
}
this.client = new AmazonEC2Client(buildCredentials(logger, settings), buildConfiguration(logger, settings));
String endpoint = findEndpoint(logger, settings);
if (endpoint != null) {
client.setEndpoint(endpoint);
}
return this.client;
}
protected static AWSCredentialsProvider buildCredentials(ESLogger logger, Settings settings) {
AWSCredentialsProvider credentials;
String key = CLOUD_EC2.KEY_SETTING.get(settings);
String secret = CLOUD_EC2.SECRET_SETTING.get(settings);
if (key.isEmpty() && secret.isEmpty()) {
logger.debug("Using either environment variables, system properties or instance profile credentials");
credentials = new AWSCredentialsProviderChain(
new EnvironmentVariableCredentialsProvider(),
new SystemPropertiesCredentialsProvider(),
new InstanceProfileCredentialsProvider()
);
} else {
logger.debug("Using basic key/secret credentials");
credentials = new AWSCredentialsProviderChain(
new StaticCredentialsProvider(new BasicAWSCredentials(key, secret))
);
}
return credentials;
}
protected static ClientConfiguration buildConfiguration(ESLogger logger, Settings settings) {
ClientConfiguration clientConfiguration = new ClientConfiguration();
// the response metadata cache is only there for diagnostics purposes,
// but can force objects from every response to the old generation.
clientConfiguration.setResponseMetadataCacheSize(0);
clientConfiguration.setProtocol(CLOUD_EC2.PROTOCOL_SETTING.get(settings));
String key = CLOUD_EC2.KEY_SETTING.get(settings);
String secret = CLOUD_EC2.SECRET_SETTING.get(settings);
if (CLOUD_EC2.PROXY_HOST_SETTING.exists(settings)) {
if (PROXY_HOST_SETTING.exists(settings) || CLOUD_EC2.PROXY_HOST_SETTING.exists(settings)) {
String proxyHost = CLOUD_EC2.PROXY_HOST_SETTING.get(settings);
Integer proxyPort = CLOUD_EC2.PROXY_PORT_SETTING.get(settings);
String proxyUsername = CLOUD_EC2.PROXY_USERNAME_SETTING.get(settings);
@ -97,78 +128,86 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent implements Aws
// Increase the number of retries in case of 5xx API responses
final Random rand = Randomness.get();
RetryPolicy retryPolicy = new RetryPolicy(
RetryPolicy.RetryCondition.NO_RETRY_CONDITION,
new RetryPolicy.BackoffStrategy() {
@Override
public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest,
AmazonClientException exception,
int retriesAttempted) {
// with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000)
logger.warn("EC2 API request failed, retry again. Reason was:", exception);
return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble()));
}
},
10,
false);
RetryPolicy.RetryCondition.NO_RETRY_CONDITION,
new RetryPolicy.BackoffStrategy() {
@Override
public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest,
AmazonClientException exception,
int retriesAttempted) {
// with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000)
logger.warn("EC2 API request failed, retry again. Reason was:", exception);
return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble()));
}
},
10,
false);
clientConfiguration.setRetryPolicy(retryPolicy);
AWSCredentialsProvider credentials;
if (key.isEmpty() && secret.isEmpty()) {
credentials = new AWSCredentialsProviderChain(
new EnvironmentVariableCredentialsProvider(),
new SystemPropertiesCredentialsProvider(),
new InstanceProfileCredentialsProvider()
);
} else {
credentials = new AWSCredentialsProviderChain(
new StaticCredentialsProvider(new BasicAWSCredentials(key, secret))
);
}
this.client = new AmazonEC2Client(credentials, clientConfiguration);
return clientConfiguration;
}
protected static String findEndpoint(ESLogger logger, Settings settings) {
String endpoint = null;
if (CLOUD_EC2.ENDPOINT_SETTING.exists(settings)) {
final String endpoint = CLOUD_EC2.ENDPOINT_SETTING.get(settings);
endpoint = CLOUD_EC2.ENDPOINT_SETTING.get(settings);
logger.debug("using explicit ec2 endpoint [{}]", endpoint);
client.setEndpoint(endpoint);
} else if (CLOUD_EC2.REGION_SETTING.exists(settings)) {
} else if (REGION_SETTING.exists(settings) || CLOUD_EC2.REGION_SETTING.exists(settings)) {
final String region = CLOUD_EC2.REGION_SETTING.get(settings);
final String endpoint;
if (region.equals("us-east-1") || region.equals("us-east")) {
endpoint = "ec2.us-east-1.amazonaws.com";
} else if (region.equals("us-west") || region.equals("us-west-1")) {
endpoint = "ec2.us-west-1.amazonaws.com";
} else if (region.equals("us-west-2")) {
endpoint = "ec2.us-west-2.amazonaws.com";
} else if (region.equals("ap-southeast") || region.equals("ap-southeast-1")) {
endpoint = "ec2.ap-southeast-1.amazonaws.com";
} else if (region.equals("us-gov-west") || region.equals("us-gov-west-1")) {
endpoint = "ec2.us-gov-west-1.amazonaws.com";
} else if (region.equals("ap-south-1")) {
endpoint = "ec2.ap-south-1.amazonaws.com";
} else if (region.equals("ap-southeast-2")) {
endpoint = "ec2.ap-southeast-2.amazonaws.com";
} else if (region.equals("ap-northeast") || region.equals("ap-northeast-1")) {
endpoint = "ec2.ap-northeast-1.amazonaws.com";
} else if (region.equals("ap-northeast-2")) {
endpoint = "ec2.ap-northeast-2.amazonaws.com";
} else if (region.equals("eu-west") || region.equals("eu-west-1")) {
endpoint = "ec2.eu-west-1.amazonaws.com";
} else if (region.equals("eu-central") || region.equals("eu-central-1")) {
endpoint = "ec2.eu-central-1.amazonaws.com";
} else if (region.equals("sa-east") || region.equals("sa-east-1")) {
endpoint = "ec2.sa-east-1.amazonaws.com";
} else if (region.equals("cn-north") || region.equals("cn-north-1")) {
endpoint = "ec2.cn-north-1.amazonaws.com.cn";
} else {
throw new IllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]");
switch (region) {
case "us-east-1":
case "us-east":
endpoint = "ec2.us-east-1.amazonaws.com";
break;
case "us-west":
case "us-west-1":
endpoint = "ec2.us-west-1.amazonaws.com";
break;
case "us-west-2":
endpoint = "ec2.us-west-2.amazonaws.com";
break;
case "ap-southeast":
case "ap-southeast-1":
endpoint = "ec2.ap-southeast-1.amazonaws.com";
break;
case "ap-south-1":
endpoint = "ec2.ap-south-1.amazonaws.com";
break;
case "us-gov-west":
case "us-gov-west-1":
endpoint = "ec2.us-gov-west-1.amazonaws.com";
break;
case "ap-southeast-2":
endpoint = "ec2.ap-southeast-2.amazonaws.com";
break;
case "ap-northeast":
case "ap-northeast-1":
endpoint = "ec2.ap-northeast-1.amazonaws.com";
break;
case "ap-northeast-2":
endpoint = "ec2.ap-northeast-2.amazonaws.com";
break;
case "eu-west":
case "eu-west-1":
endpoint = "ec2.eu-west-1.amazonaws.com";
break;
case "eu-central":
case "eu-central-1":
endpoint = "ec2.eu-central-1.amazonaws.com";
break;
case "sa-east":
case "sa-east-1":
endpoint = "ec2.sa-east-1.amazonaws.com";
break;
case "cn-north":
case "cn-north-1":
endpoint = "ec2.cn-north-1.amazonaws.com.cn";
break;
default:
throw new IllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]");
}
logger.debug("using ec2 region [{}], with endpoint [{}]", region, endpoint);
client.setEndpoint(endpoint);
}
return this.client;
return endpoint;
}
@Override

169
plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImplTests.java Normal file
View File

@ -0,0 +1,169 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cloud.aws;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
public class AwsEc2ServiceImplTests extends ESTestCase {
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/19556")
public void testAWSCredentialsWithSystemProviders() {
AWSCredentialsProvider credentialsProvider = AwsEc2ServiceImpl.buildCredentials(logger, Settings.EMPTY);
AWSCredentials credentials = credentialsProvider.getCredentials();
assertThat(credentials.getAWSAccessKeyId(), is("DUMMY_ACCESS_KEY"));
assertThat(credentials.getAWSSecretKey(), is("DUMMY_SECRET_KEY"));
}
public void testAWSCredentialsWithElasticsearchAwsSettings() {
Settings settings = Settings.builder()
.put(AwsEc2Service.KEY_SETTING.getKey(), "aws_key")
.put(AwsEc2Service.SECRET_SETTING.getKey(), "aws_secret")
.build();
launchAWSCredentialsWithElasticsearchSettingsTest(settings, "aws_key", "aws_secret");
}
public void testAWSCredentialsWithElasticsearchEc2Settings() {
Settings settings = Settings.builder()
.put(AwsEc2Service.CLOUD_EC2.KEY_SETTING.getKey(), "ec2_key")
.put(AwsEc2Service.CLOUD_EC2.SECRET_SETTING.getKey(), "ec2_secret")
.build();
launchAWSCredentialsWithElasticsearchSettingsTest(settings, "ec2_key", "ec2_secret");
}
public void testAWSCredentialsWithElasticsearchAwsAndEc2Settings() {
Settings settings = Settings.builder()
.put(AwsEc2Service.KEY_SETTING.getKey(), "aws_key")
.put(AwsEc2Service.SECRET_SETTING.getKey(), "aws_secret")
.put(AwsEc2Service.CLOUD_EC2.KEY_SETTING.getKey(), "ec2_key")
.put(AwsEc2Service.CLOUD_EC2.SECRET_SETTING.getKey(), "ec2_secret")
.build();
launchAWSCredentialsWithElasticsearchSettingsTest(settings, "ec2_key", "ec2_secret");
}
protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings settings, String expectedKey, String expectedSecret) {
AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, settings).getCredentials();
assertThat(credentials.getAWSAccessKeyId(), is(expectedKey));
assertThat(credentials.getAWSSecretKey(), is(expectedSecret));
}
public void testAWSDefaultConfiguration() {
launchAWSConfigurationTest(Settings.EMPTY, Protocol.HTTPS, null, -1, null, null, null);
}
public void testAWSConfigurationWithAwsSettings() {
Settings settings = Settings.builder()
.put(AwsEc2Service.PROTOCOL_SETTING.getKey(), "http")
.put(AwsEc2Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
.put(AwsEc2Service.PROXY_PORT_SETTING.getKey(), 8080)
.put(AwsEc2Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
.put(AwsEc2Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
.put(AwsEc2Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
.build();
launchAWSConfigurationTest(settings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", "aws_proxy_password",
"AWS3SignerType");
}
public void testAWSConfigurationWithAwsAndEc2Settings() {
Settings settings = Settings.builder()
.put(AwsEc2Service.PROTOCOL_SETTING.getKey(), "http")
.put(AwsEc2Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
.put(AwsEc2Service.PROXY_PORT_SETTING.getKey(), 8080)
.put(AwsEc2Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
.put(AwsEc2Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
.put(AwsEc2Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
.put(AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING.getKey(), "https")
.put(AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING.getKey(), "ec2_proxy_host")
.put(AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING.getKey(), 8081)
.put(AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING.getKey(), "ec2_proxy_username")
.put(AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING.getKey(), "ec2_proxy_password")
.put(AwsEc2Service.CLOUD_EC2.SIGNER_SETTING.getKey(), "NoOpSignerType")
.build();
launchAWSConfigurationTest(settings, Protocol.HTTPS, "ec2_proxy_host", 8081, "ec2_proxy_username", "ec2_proxy_password",
"NoOpSignerType");
}
protected void launchAWSConfigurationTest(Settings settings,
Protocol expectedProtocol,
String expectedProxyHost,
int expectedProxyPort,
String expectedProxyUsername,
String expectedProxyPassword,
String expectedSigner) {
ClientConfiguration configuration = AwsEc2ServiceImpl.buildConfiguration(logger, settings);
assertThat(configuration.getResponseMetadataCacheSize(), is(0));
assertThat(configuration.getProtocol(), is(expectedProtocol));
assertThat(configuration.getProxyHost(), is(expectedProxyHost));
assertThat(configuration.getProxyPort(), is(expectedProxyPort));
assertThat(configuration.getProxyUsername(), is(expectedProxyUsername));
assertThat(configuration.getProxyPassword(), is(expectedProxyPassword));
assertThat(configuration.getSignerOverride(), is(expectedSigner));
}
public void testDefaultEndpoint() {
String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, Settings.EMPTY);
assertThat(endpoint, nullValue());
}
public void testSpecificEndpoint() {
Settings settings = Settings.builder()
.put(AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING.getKey(), "ec2.endpoint")
.build();
String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings);
assertThat(endpoint, is("ec2.endpoint"));
}
public void testRegionWithAwsSettings() {
Settings settings = Settings.builder()
.put(AwsEc2Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
.build();
String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings);
assertThat(endpoint, is("ec2.eu-west-1.amazonaws.com"));
}
public void testRegionWithAwsAndEc2Settings() {
Settings settings = Settings.builder()
.put(AwsEc2Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
.put(AwsEc2Service.CLOUD_EC2.REGION_SETTING.getKey(), randomFrom("us-west", "us-west-1"))
.build();
String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings);
assertThat(endpoint, is("ec2.us-west-1.amazonaws.com"));
}
public void testInvalidRegion() {
Settings settings = Settings.builder()
.put(AwsEc2Service.REGION_SETTING.getKey(), "does-not-exist")
.build();
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
AwsEc2ServiceImpl.findEndpoint(logger, settings);
});
assertThat(e.getMessage(), containsString("No automatic endpoint could be derived from region"));
}
}