honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

S3 Repository: Remove region setting (#22853) 

This change removes the ability to set region for s3 repositories.
Endpoint should be used instead if a custom s3 location needs to be
used.

closes #22758
This commit is contained in:
Ryan Ernst 2017-01-30 14:34:59 -08:00 committed by GitHub
parent 4e613139dc
commit cf7747c338
7 changed files with 33 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/reference/migration/migrate_6_0/plugins.asciidoc
View File

@ -14,3 +14,9 @@ It must exist before the s3 repository is created.
* Support for specifying s3 credentials through environment variables and
system properties has been removed. Use the `elasticsearch-keystore` tool
to securely store the credentials.
* Specifying region has been removed. This includes the settings `cloud.aws.region`,
`cloud.aws.s3.region`, `repositories.s3.region`, and specifying
region inside the repository settings. Instead, specify the full endpoint if a custom
s3 location is needed, or rely on the default behavior which automatically locates
the region of the configured bucket.

14
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
View File

@ -77,11 +77,6 @@ public interface AwsS3Service extends LifecycleComponent {
*/
Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer",
Property.NodeScope, Property.Deprecated, Property.Shared);
/**
* cloud.aws.region: Region. Shared with discovery-ec2 plugin
*/
Setting<String> REGION_SETTING = new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT),
Property.NodeScope, Property.Deprecated, Property.Shared);
/**
* cloud.aws.read_timeout: Socket read timeout. Shared with discovery-ec2 plugin
*/
@ -153,14 +148,7 @@ public interface AwsS3Service extends LifecycleComponent {
new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(),
Property.NodeScope, Property.Deprecated);
/**
* cloud.aws.s3.region: Region specific for S3 API calls. Defaults to cloud.aws.region.
* @see AwsS3Service#REGION_SETTING
*/
Setting<String> REGION_SETTING =
new Setting<>("cloud.aws.s3.region", AwsS3Service.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT),
Property.NodeScope, Property.Deprecated);
/**
* cloud.aws.s3.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
* cloud.aws.s3.endpoint: Endpoint.
*/
Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
/**

81
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
View File

@ -162,25 +162,13 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
// pkg private for tests
/** Returns the endpoint the client should use, based on the available endpoint settings found. */
static String findEndpoint(Logger logger, Settings repositorySettings, Settings settings, String clientName) {
String region = getConfigValue(repositorySettings, settings, CLIENT_NAME.get(repositorySettings), S3Repository.REGION_SETTING,
S3Repository.Repository.REGION_SETTING, S3Repository.Repositories.REGION_SETTING);
String endpoint = getConfigValue(repositorySettings, settings, clientName, S3Repository.ENDPOINT_SETTING,
S3Repository.Repository.ENDPOINT_SETTING, S3Repository.Repositories.ENDPOINT_SETTING);
if (Strings.isNullOrEmpty(endpoint)) {
logger.debug("no repository level endpoint has been defined. Trying to guess from repository region [{}]", region);
if (!region.isEmpty()) {
endpoint = getEndpoint(region);
logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint);
} else {
// No region has been set so we will use the default endpoint
if (CLOUD_S3.ENDPOINT_SETTING.exists(settings)) {
endpoint = CLOUD_S3.ENDPOINT_SETTING.get(settings);
logger.debug("using explicit s3 endpoint [{}]", endpoint);
} else if (REGION_SETTING.exists(settings) || CLOUD_S3.REGION_SETTING.exists(settings)) {
region = CLOUD_S3.REGION_SETTING.get(settings);
endpoint = getEndpoint(region);
logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint);
}
}
} else {
logger.debug("using repository level endpoint [{}]", endpoint);
@ -189,75 +177,6 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
return endpoint;
}
private static String getEndpoint(String region) {
final String endpoint;
switch (region) {
case "us-east":
case "us-east-1":
endpoint = "s3.amazonaws.com";
break;
case "us-east-2":
endpoint = "s3.us-east-2.amazonaws.com";
break;
case "us-west":
case "us-west-1":
endpoint = "s3-us-west-1.amazonaws.com";
break;
case "us-west-2":
endpoint = "s3-us-west-2.amazonaws.com";
break;
case "ap-south":
case "ap-south-1":
endpoint = "s3-ap-south-1.amazonaws.com";
break;
case "ap-southeast":
case "ap-southeast-1":
endpoint = "s3-ap-southeast-1.amazonaws.com";
break;
case "ap-southeast-2":
endpoint = "s3-ap-southeast-2.amazonaws.com";
break;
case "ap-northeast":
case "ap-northeast-1":
endpoint = "s3-ap-northeast-1.amazonaws.com";
break;
case "ap-northeast-2":
endpoint = "s3-ap-northeast-2.amazonaws.com";
break;
case "eu-west":
case "eu-west-1":
endpoint = "s3-eu-west-1.amazonaws.com";
break;
case "eu-west-2":
endpoint = "s3-eu-west-2.amazonaws.com";
break;
case "eu-central":
case "eu-central-1":
endpoint = "s3.eu-central-1.amazonaws.com";
break;
case "sa-east":
case "sa-east-1":
endpoint = "s3-sa-east-1.amazonaws.com";
break;
case "cn-north":
case "cn-north-1":
endpoint = "s3.cn-north-1.amazonaws.com.cn";
break;
case "us-gov-west":
case "us-gov-west-1":
endpoint = "s3-us-gov-west-1.amazonaws.com";
break;
case "ca-central":
case "ca-central-1":
endpoint = "s3.ca-central-1.amazonaws.com";
break;
default:
throw new IllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]");
}
return endpoint;
}
/**
* Find the setting value, trying first with named configs,
* then falling back to repository and global repositories settings.

3
plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
View File

@ -90,7 +90,6 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin {
AwsS3Service.PROXY_USERNAME_SETTING,
AwsS3Service.PROXY_PASSWORD_SETTING,
AwsS3Service.SIGNER_SETTING,
AwsS3Service.REGION_SETTING,
AwsS3Service.READ_TIMEOUT,
// Register S3 specific settings: cloud.aws.s3
@ -102,7 +101,6 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin {
AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING,
AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING,
AwsS3Service.CLOUD_S3.SIGNER_SETTING,
AwsS3Service.CLOUD_S3.REGION_SETTING,
AwsS3Service.CLOUD_S3.ENDPOINT_SETTING,
AwsS3Service.CLOUD_S3.READ_TIMEOUT,
@ -110,7 +108,6 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin {
S3Repository.Repositories.KEY_SETTING,
S3Repository.Repositories.SECRET_SETTING,
S3Repository.Repositories.BUCKET_SETTING,
S3Repository.Repositories.REGION_SETTING,
S3Repository.Repositories.ENDPOINT_SETTING,
S3Repository.Repositories.PROTOCOL_SETTING,
S3Repository.Repositories.SERVER_SIDE_ENCRYPTION_SETTING,

16
plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View File

@ -53,7 +53,6 @@ import java.util.Locale;
* Shared file system repository supports the following settings
* <dl>
* <dt>{@code bucket}</dt><dd>S3 bucket</dd>
* <dt>{@code region}</dt><dd>S3 region. Defaults to us-east</dd>
* <dt>{@code base_path}</dt><dd>Specifies the path within bucket to repository data. Defaults to root directory.</dd>
* <dt>{@code concurrent_streams}</dt><dd>Number of concurrent read/write stream (per repository on each node). Defaults to 5.</dd>
* <dt>{@code chunk_size}</dt><dd>Large file can be divided into chunks. This parameter specifies the chunk size. Defaults to not chucked.</dd>
@ -75,10 +74,6 @@ public class S3Repository extends BlobStoreRepository {
public static final AffixSetting<SecureString> SECRET_KEY_SETTING = Setting.affixKeySetting(PREFIX, "secret_key",
key -> SecureSetting.secureString(key, Repositories.SECRET_SETTING, false));
/** The region the s3 repository bucket should exist in. */
public static final AffixSetting<String> REGION_SETTING = Setting.affixKeySetting(PREFIX, "region",
key -> new Setting<>(key, "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope));
/** An override for the s3 endpoint to connect to. */
public static final AffixSetting<String> ENDPOINT_SETTING = Setting.affixKeySetting(PREFIX, "endpoint",
key -> new Setting<>(key, Repositories.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope));
@ -126,12 +121,6 @@ public class S3Repository extends BlobStoreRepository {
Setting<SecureString> SECRET_SETTING = new Setting<>("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, SecureString::new,
Property.NodeScope, Property.Filtered, Property.Deprecated);
/**
* repositories.s3.region: Region specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.region.
* @see CLOUD_S3#REGION_SETTING
*/
Setting<String> REGION_SETTING = new Setting<>("repositories.s3.region", CLOUD_S3.REGION_SETTING,
s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated);
/**
* repositories.s3.endpoint: Endpoint specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.endpoint.
* @see CLOUD_S3#ENDPOINT_SETTING
@ -245,11 +234,6 @@ public class S3Repository extends BlobStoreRepository {
*/
Setting<Protocol> PROTOCOL_SETTING = new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
Property.Deprecated);
/**
* region
* @see Repositories#REGION_SETTING
*/
Setting<String> REGION_SETTING = new Setting<>("region", "", s -> s.toLowerCase(Locale.ROOT), Property.Deprecated);
/**
* server_side_encryption
* @see Repositories#SERVER_SIDE_ENCRYPTION_SETTING

111
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java
View File

@ -42,18 +42,15 @@ public class AwsS3ServiceImplTests extends ESTestCase {
}
public void testAwsCredsDefaultSettings() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString("s3.client.default.access_key", "aws_key");
secureSettings.setString("s3.client.default.secret_key", "aws_secret");
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret");
launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "aws_key", "aws_secret");
}
public void testAwsCredsExplicitConfigSettings() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
repositorySettings = Settings.builder().put(repositorySettings)
.put(InternalAwsS3Service.CLIENT_NAME.getKey(), "myconfig").build();
Settings repositorySettings = Settings.builder().put(InternalAwsS3Service.CLIENT_NAME.getKey(), "myconfig").build();
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString("s3.client.myconfig.access_key", "aws_key");
secureSettings.setString("s3.client.myconfig.secret_key", "aws_secret");
@ -64,36 +61,33 @@ public class AwsS3ServiceImplTests extends ESTestCase {
}
public void testAWSCredentialsWithElasticsearchAwsSettingsBackcompat() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
Settings settings = Settings.builder()
.put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
.put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
.build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret");
launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "aws_key", "aws_secret");
assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated");
}
public void testAWSCredentialsWithElasticsearchS3SettingsBackcompat() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
Settings settings = Settings.builder()
.put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key")
.put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
.build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "s3_key", "s3_secret");
assertWarnings("[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey() + "] setting was deprecated");
}
public void testAWSCredentialsWithElasticsearchAwsAndS3SettingsBackcompat() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
Settings settings = Settings.builder()
.put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
.put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
.put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key")
.put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
.build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "s3_key", "s3_secret");
assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated",
@ -101,25 +95,23 @@ public class AwsS3ServiceImplTests extends ESTestCase {
}
public void testAWSCredentialsWithElasticsearchRepositoriesSettingsBackcompat() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
Settings settings = Settings.builder()
.put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
.put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
.build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "repositories_key", "repositories_secret");
assertWarnings("[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repositories.SECRET_SETTING.getKey() + "] setting was deprecated");
}
public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettingsBackcompat() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
Settings settings = Settings.builder()
.put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
.put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
.put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
.put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
.build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "repositories_key", "repositories_secret");
assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
@ -127,7 +119,6 @@ public class AwsS3ServiceImplTests extends ESTestCase {
}
public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettingsBackcompat() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
Settings settings = Settings.builder()
.put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
.put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
@ -136,7 +127,7 @@ public class AwsS3ServiceImplTests extends ESTestCase {
.put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
.put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
.build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "repositories_key", "repositories_secret");
assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated",
@ -146,7 +137,7 @@ public class AwsS3ServiceImplTests extends ESTestCase {
}
public void testAWSCredentialsWithElasticsearchRepositoriesSettingsAndRepositorySettingsBackcompat() {
Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", null, null);
Settings settings = Settings.builder()
.put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
.put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
@ -157,7 +148,7 @@ public class AwsS3ServiceImplTests extends ESTestCase {
}
public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettingsAndRepositorySettingsBackcompat() {
Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", null, null);
Settings settings = Settings.builder()
.put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
.put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
@ -170,7 +161,7 @@ public class AwsS3ServiceImplTests extends ESTestCase {
}
public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettingsAndRepositorySettingsBackcompat() {
Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", null, null);
Settings settings = Settings.builder()
.put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
.put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
@ -194,13 +185,11 @@ public class AwsS3ServiceImplTests extends ESTestCase {
}
public void testAWSDefaultConfiguration() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
launchAWSConfigurationTest(Settings.EMPTY, repositorySettings, Protocol.HTTPS, null, -1, null, null, null, 3, false,
launchAWSConfigurationTest(Settings.EMPTY, Settings.EMPTY, Protocol.HTTPS, null, -1, null, null, null, 3, false,
ClientConfiguration.DEFAULT_SOCKET_TIMEOUT);
}
public void testAWSConfigurationWithAwsSettings() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString("s3.client.default.proxy.username", "aws_proxy_username");
secureSettings.setString("s3.client.default.proxy.password", "aws_proxy_password");
@ -211,12 +200,11 @@ public class AwsS3ServiceImplTests extends ESTestCase {
.put("s3.client.default.proxy.port", 8080)
.put("s3.client.default.read_timeout", "10s")
.build();
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
"aws_proxy_password", null, 3, false, 10000);
}
public void testAWSConfigurationWithAwsSettingsBackcompat() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
Settings settings = Settings.builder()
.put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http")
.put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
@ -226,7 +214,7 @@ public class AwsS3ServiceImplTests extends ESTestCase {
.put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
.put(AwsS3Service.READ_TIMEOUT.getKey(), "10s")
.build();
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
"aws_proxy_password", "AWS3SignerType", 3, false, 10000);
assertWarnings("[" + AwsS3Service.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated",
@ -238,7 +226,6 @@ public class AwsS3ServiceImplTests extends ESTestCase {
}
public void testAWSConfigurationWithAwsAndS3SettingsBackcompat() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
Settings settings = Settings.builder()
.put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http")
.put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
@ -255,7 +242,7 @@ public class AwsS3ServiceImplTests extends ESTestCase {
.put(AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey(), "NoOpSignerType")
.put(AwsS3Service.CLOUD_S3.READ_TIMEOUT.getKey(), "10s")
.build();
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username",
launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username",
"s3_proxy_password", "NoOpSignerType", 3, false, 10000);
assertWarnings("[" + AwsS3Service.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated",
@ -274,16 +261,15 @@ public class AwsS3ServiceImplTests extends ESTestCase {
}
public void testGlobalMaxRetries() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
Settings settings = Settings.builder()
.put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10)
.build();
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null,
launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, null, -1, null,
null, null, 10, false, 50000);
}
public void testRepositoryMaxRetries() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, 20);
Settings repositorySettings = generateRepositorySettings(null, null, null, 20);
Settings settings = Settings.builder()
.put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10)
.build();
@ -322,11 +308,8 @@ public class AwsS3ServiceImplTests extends ESTestCase {
assertThat(configuration.getSocketTimeout(), is(expectedReadTimeout));
}
private static Settings generateRepositorySettings(String key, String secret, String region, String endpoint, Integer maxRetries) {
private static Settings generateRepositorySettings(String key, String secret, String endpoint, Integer maxRetries) {
Settings.Builder builder = Settings.builder();
if (region != null) {
builder.put(S3Repository.Repository.REGION_SETTING.getKey(), region);
}
if (endpoint != null) {
builder.put(S3Repository.Repository.ENDPOINT_SETTING.getKey(), endpoint);
}
@ -343,78 +326,28 @@ public class AwsS3ServiceImplTests extends ESTestCase {
}
public void testDefaultEndpoint() {
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), Settings.EMPTY, "");
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null), Settings.EMPTY, "");
}
public void testEndpointSetting() {
Settings settings = Settings.builder()
.put("s3.client.default.endpoint", "s3.endpoint")
.build();
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, "s3.endpoint");
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null), settings, "s3.endpoint");
}
public void testEndpointSettingBackcompat() {
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", "repository.endpoint", null),
Settings.EMPTY, "repository.endpoint");
assertWarnings("[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
Settings settings = Settings.builder()
.put(S3Repository.Repositories.ENDPOINT_SETTING.getKey(), "repositories.endpoint")
.build();
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null), settings,
"repositories.endpoint");
assertWarnings("[" + S3Repository.Repositories.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
}
public void testRegionSetting() {
Settings settings = Settings.builder()
.put("s3.client.default.region", randomFrom("eu-west", "eu-west-1"))
.build();
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
"s3-eu-west-1.amazonaws.com");
}
public void testRegionSettingBackcompat() {
Settings settings = Settings.builder()
.put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
.build();
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
"s3-eu-west-1.amazonaws.com");
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
"s3.eu-central-1.amazonaws.com");
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
settings, "repository.endpoint");
assertWarnings("[" + InternalAwsS3Service.REGION_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repository.REGION_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
}
public void testRegionAndEndpointSettingBackcompatPrecedence() {
Settings settings = Settings.builder()
.put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
.put(InternalAwsS3Service.CLOUD_S3.REGION_SETTING.getKey(), randomFrom("us-west", "us-west-1"))
.build();
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
"s3-us-west-1.amazonaws.com");
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
"s3.eu-central-1.amazonaws.com");
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
settings, "repository.endpoint");
assertWarnings("[" + InternalAwsS3Service.REGION_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repository.REGION_SETTING.getKey() + "] setting was deprecated",
"[" + InternalAwsS3Service.CLOUD_S3.REGION_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
}
public void testInvalidRegion() {
Settings settings = Settings.builder()
.put("s3.client.default.region", "does-not-exist")
.build();
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, null);
});
assertThat(e.getMessage(), containsString("No automatic endpoint could be derived from region"));
}
private void assertEndpoint(Settings repositorySettings, Settings settings,
String expectedEndpoint) {
String configName = InternalAwsS3Service.CLIENT_NAME.get(repositorySettings);

2
plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
View File

@ -283,7 +283,6 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("s3").setSettings(Settings.builder()
.put(S3Repository.Repository.BASE_PATH_SETTING.getKey(), basePath)
.put(S3Repository.Repository.REGION_SETTING.getKey(), bucketSettings.get("region"))
.put(S3Repository.Repository.KEY_SETTING.getKey(), bucketSettings.get("access_key"))
.put(S3Repository.Repository.SECRET_SETTING.getKey(), bucketSettings.get("secret_key"))
.put(S3Repository.Repository.BUCKET_SETTING.getKey(), bucketSettings.get("bucket"))
@ -341,7 +340,6 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
.setType("s3").setSettings(Settings.builder()
.put(S3Repository.Repository.BASE_PATH_SETTING.getKey(), basePath)
.put(S3Repository.Repository.BUCKET_SETTING.getKey(), bucketSettings.get("bucket"))
.put(S3Repository.Repository.REGION_SETTING.getKey(), bucketSettings.get("region"))
).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));