From aad51d44ab7363e53da773afa4707e0165eebda0 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 27 Jan 2017 10:42:45 -0800 Subject: [PATCH] S3 repository: Add named configurations (#22762) * S3 repository: Add named configurations This change implements named configurations for s3 repository as proposed in #22520. The access/secret key secure settings which were added in #22479 are reverted, and the only secure settings are those with the new named configs. All other previously used settings for the connection are deprecated. closes #22520 --- .../elasticsearch/bootstrap/Bootstrap.java | 3 +- .../common/settings/SecureSetting.java | 4 +- .../common/settings/SecureString.java | 13 + .../common/settings/Setting.java | 10 +- .../common/settings/Settings.java | 3 + .../node/InternalSettingsPreparer.java | 1 + .../InternalSettingsPreparerTests.java | 13 + .../elasticsearch/cloud/aws/AwsS3Service.java | 82 ++--- .../cloud/aws/InternalAwsS3Service.java | 97 ++++-- .../repositories/s3/S3Repository.java | 107 +++++-- .../cloud/aws/AwsS3ServiceImplTests.java | 295 +++++++----------- .../aws/EnvironmentCredentialsTests.java | 2 +- .../cloud/aws/SyspropCredentialsTests.java | 2 +- .../cloud/aws/TestAwsS3Service.java | 12 +- .../s3/AbstractS3SnapshotRestoreTest.java | 15 +- .../repositories/s3/S3RepositoryTests.java | 18 +- .../org/elasticsearch/test/ESTestCase.java | 6 +- 17 files changed, 368 insertions(+), 315 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index 2b47908c352..d3766cc958c 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -338,12 +338,13 @@ final class Bootstrap { INSTANCE.setup(true, environment); + /* TODO: close this once s3 repository doesn't try to read during repository construction try { // any secure settings must be read during node construction IOUtils.close(keystore); } catch (IOException e) { throw new BootstrapException(e); - } + }*/ INSTANCE.start(); diff --git a/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java b/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java index 1a69896dcc1..c06dad2a2ea 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java +++ b/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java @@ -109,8 +109,8 @@ public abstract class SecureSetting extends Setting { * * This may be any sensitive string, e.g. a username, a password, an auth token, etc. */ - public static SecureSetting secureString(String name, SecureSetting fallback, - boolean allowLegacy, Property... properties) { + public static Setting secureString(String name, Setting fallback, + boolean allowLegacy, Property... properties) { final Setting legacy; if (allowLegacy) { Property[] legacyProperties = ArrayUtils.concat(properties, LEGACY_PROPERTIES, Property.class); diff --git a/core/src/main/java/org/elasticsearch/common/settings/SecureString.java b/core/src/main/java/org/elasticsearch/common/settings/SecureString.java index 3d423dca637..36982ddde1c 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/SecureString.java +++ b/core/src/main/java/org/elasticsearch/common/settings/SecureString.java @@ -39,6 +39,19 @@ public final class SecureString implements CharSequence, Closeable { this.chars = Objects.requireNonNull(chars); } + /** + * Constructs a new SecureString from an existing String. + * + * NOTE: This is not actually secure, since the provided String cannot be deallocated, but + * this constructor allows for easy compatibility between new and old apis. + * + * @deprecated Only use for compatibility between deprecated string settings and new secure strings + */ + @Deprecated + public SecureString(String s) { + this(s.toCharArray()); + } + /** Constant time equality to avoid potential timing attacks. */ @Override public synchronized boolean equals(Object o) { diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java index 74e10bd6e11..a1f25a5e98b 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -123,7 +123,7 @@ public class Setting extends ToXContentToBytes { private Setting(Key key, @Nullable Setting fallbackSetting, Function defaultValue, Function parser, Property... properties) { - assert this instanceof SecureSetting || parser.apply(defaultValue.apply(Settings.EMPTY)) != null || this.isGroupSetting() + assert this instanceof SecureSetting || this.isGroupSetting() || parser.apply(defaultValue.apply(Settings.EMPTY)) != null : "parser returned null"; this.key = key; this.fallbackSetting = fallbackSetting; @@ -527,6 +527,14 @@ public class Setting extends ToXContentToBytes { } } + /** + * Get a setting with the given namespace filled in for prefix and suffix. + */ + public Setting getConcreteSettingForNamespace(String namespace) { + String fullKey = key.toConcreteKey(namespace).toString(); + return getConcreteSetting(fullKey); + } + @Override public void diff(Settings.Builder builder, Settings source, Settings defaultSettings) { matchStream(defaultSettings).forEach((key) -> getConcreteSetting(key).diff(builder, source, defaultSettings)); diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java index 516b5bab45d..8bd55e19e07 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -897,6 +897,9 @@ public final class Settings implements ToXContent { public Builder put(Settings settings) { removeNonArraysFieldsIfNewSettingsContainsFieldAsArray(settings.getAsMap()); map.putAll(settings.getAsMap()); + if (settings.getSecureSettings() != null) { + setSecureSettings(settings.getSecureSettings()); + } return this; } diff --git a/core/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java b/core/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java index 1b3ffe4327b..1c5e59fe379 100644 --- a/core/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java +++ b/core/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java @@ -89,6 +89,7 @@ public class InternalSettingsPreparer { initializeSettings(output, input, true, properties); Environment environment = new Environment(output.build()); + output = Settings.builder(); // start with a fresh output boolean settingsFileFound = false; Set foundSuffixes = new HashSet<>(); for (String allowedSuffix : ALLOWED_SUFFIXES) { diff --git a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java index 94b3f3737cb..58b6fd39594 100644 --- a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java +++ b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java @@ -21,6 +21,10 @@ package org.elasticsearch.node.internal; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.SecureSetting; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.env.Environment; @@ -161,4 +165,13 @@ public class InternalSettingsPreparerTests extends ESTestCase { assertTrue(e.getMessage(), e.getMessage().contains(".properties")); } } + + public void testSecureSettings() { + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("foo", "secret"); + Settings input = Settings.builder().put(baseEnvSettings).setSecureSettings(secureSettings).build(); + Environment env = InternalSettingsPreparer.prepareEnvironment(input, null); + Setting fakeSetting = SecureSetting.secureString("foo", null, false); + assertEquals("secret", fakeSetting.get(env.settings()).toString()); + } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index e02556ccc2b..44b4357371c 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -19,139 +19,146 @@ package org.elasticsearch.cloud.aws; +import java.util.Locale; +import java.util.function.Function; + import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.services.s3.AmazonS3; import org.elasticsearch.common.component.LifecycleComponent; -import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import java.util.Locale; -import java.util.function.Function; - public interface AwsS3Service extends LifecycleComponent { - // Global AWS settings (shared between discovery-ec2 and repository-s3) + // Legacy global AWS settings (shared between discovery-ec2 and repository-s3) // Each setting starting with `cloud.aws` also exists in discovery-ec2 project. Don't forget to update // the code there if you change anything here. /** * cloud.aws.access_key: AWS Access key. Shared with discovery-ec2 plugin */ - SecureSetting KEY_SETTING = SecureSetting.secureString("cloud.aws.access_key", null, true, Property.Shared); - + Setting KEY_SETTING = new Setting<>("cloud.aws.access_key", "", SecureString::new, + Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared); /** * cloud.aws.secret_key: AWS Secret key. Shared with discovery-ec2 plugin */ - SecureSetting SECRET_SETTING = SecureSetting.secureString("cloud.aws.secret_key", null, true, Property.Shared); + Setting SECRET_SETTING = new Setting<>("cloud.aws.secret_key", "", SecureString::new, + Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared); /** * cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with discovery-ec2 plugin */ - Setting PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), - Property.NodeScope, Property.Shared); + Setting PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", + s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated, Property.Shared); /** * cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with discovery-ec2 plugin */ - Setting PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope, Property.Shared); + Setting PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", + Property.NodeScope, Property.Deprecated, Property.Shared); /** * cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with discovery-ec2 plugin */ - Setting PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope, - Property.Shared); + Setting PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, + Property.NodeScope, Property.Deprecated, Property.Shared); /** * cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with discovery-ec2 plugin */ - SecureSetting PROXY_USERNAME_SETTING = - SecureSetting.secureString("cloud.aws.proxy.username", null, true, Property.Shared); - + Setting PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.proxy.username", "", SecureString::new, + Property.NodeScope, Property.Deprecated, Property.Shared); /** * cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with discovery-ec2 plugin */ - SecureSetting PROXY_PASSWORD_SETTING = - SecureSetting.secureString("cloud.aws.proxy.password", null, true, Property.Shared); + Setting PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.proxy.password", "", SecureString::new, + Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared); /** * cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with discovery-ec2 plugin */ - Setting SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared); + Setting SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", + Property.NodeScope, Property.Deprecated, Property.Shared); /** * cloud.aws.region: Region. Shared with discovery-ec2 plugin */ - Setting REGION_SETTING = - new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared); + Setting REGION_SETTING = new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), + Property.NodeScope, Property.Deprecated, Property.Shared); /** * cloud.aws.read_timeout: Socket read timeout. Shared with discovery-ec2 plugin */ Setting READ_TIMEOUT = Setting.timeSetting("cloud.aws.read_timeout", - TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Shared); + TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Deprecated, Property.Shared); /** * Defines specific s3 settings starting with cloud.aws.s3. + * NOTE: These are legacy settings. Use the named client configs in {@link org.elasticsearch.repositories.s3.S3Repository}. */ interface CLOUD_S3 { /** * cloud.aws.s3.access_key: AWS Access key specific for S3 API calls. Defaults to cloud.aws.access_key. * @see AwsS3Service#KEY_SETTING */ - SecureSetting KEY_SETTING = SecureSetting.secureString("cloud.aws.s3.access_key", AwsS3Service.KEY_SETTING, true); + Setting KEY_SETTING = + new Setting<>("cloud.aws.s3.access_key", AwsS3Service.KEY_SETTING, SecureString::new, + Property.NodeScope, Property.Filtered, Property.Deprecated); /** * cloud.aws.s3.secret_key: AWS Secret key specific for S3 API calls. Defaults to cloud.aws.secret_key. * @see AwsS3Service#SECRET_SETTING */ - SecureSetting SECRET_SETTING = SecureSetting.secureString("cloud.aws.s3.secret_key", - AwsS3Service.SECRET_SETTING, true); + Setting SECRET_SETTING = + new Setting<>("cloud.aws.s3.secret_key", AwsS3Service.SECRET_SETTING, SecureString::new, + Property.NodeScope, Property.Filtered, Property.Deprecated); /** * cloud.aws.s3.protocol: Protocol for AWS API specific for S3 API calls: http or https. Defaults to cloud.aws.protocol. * @see AwsS3Service#PROTOCOL_SETTING */ Setting PROTOCOL_SETTING = new Setting<>("cloud.aws.s3.protocol", AwsS3Service.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), - Property.NodeScope); + Property.NodeScope, Property.Deprecated); /** * cloud.aws.s3.proxy.host: In case of proxy, define its hostname/IP specific for S3 API calls. Defaults to cloud.aws.proxy.host. * @see AwsS3Service#PROXY_HOST_SETTING */ Setting PROXY_HOST_SETTING = new Setting<>("cloud.aws.s3.proxy.host", AwsS3Service.PROXY_HOST_SETTING, Function.identity(), - Property.NodeScope); + Property.NodeScope, Property.Deprecated); /** * cloud.aws.s3.proxy.port: In case of proxy, define its port specific for S3 API calls. Defaults to cloud.aws.proxy.port. * @see AwsS3Service#PROXY_PORT_SETTING */ Setting PROXY_PORT_SETTING = new Setting<>("cloud.aws.s3.proxy.port", AwsS3Service.PROXY_PORT_SETTING, - s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.s3.proxy.port"), Property.NodeScope); + s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.s3.proxy.port"), Property.NodeScope, Property.Deprecated); /** * cloud.aws.s3.proxy.username: In case of proxy with auth, define the username specific for S3 API calls. * Defaults to cloud.aws.proxy.username. * @see AwsS3Service#PROXY_USERNAME_SETTING */ - SecureSetting PROXY_USERNAME_SETTING = - SecureSetting.secureString("cloud.aws.s3.proxy.username", AwsS3Service.PROXY_USERNAME_SETTING, true); + Setting PROXY_USERNAME_SETTING = + new Setting<>("cloud.aws.s3.proxy.username", AwsS3Service.PROXY_USERNAME_SETTING, SecureString::new, + Property.NodeScope, Property.Deprecated); /** * cloud.aws.s3.proxy.password: In case of proxy with auth, define the password specific for S3 API calls. * Defaults to cloud.aws.proxy.password. * @see AwsS3Service#PROXY_PASSWORD_SETTING */ - SecureSetting PROXY_PASSWORD_SETTING = - SecureSetting.secureString("cloud.aws.s3.proxy.password", AwsS3Service.PROXY_PASSWORD_SETTING, true); - + Setting PROXY_PASSWORD_SETTING = + new Setting<>("cloud.aws.s3.proxy.password", AwsS3Service.PROXY_PASSWORD_SETTING, SecureString::new, + Property.NodeScope, Property.Filtered, Property.Deprecated); /** * cloud.aws.s3.signer: If you are using an old AWS API version, you can define a Signer. Specific for S3 API calls. * Defaults to cloud.aws.signer. * @see AwsS3Service#SIGNER_SETTING */ Setting SIGNER_SETTING = - new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(), Property.NodeScope); + new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(), + Property.NodeScope, Property.Deprecated); /** * cloud.aws.s3.region: Region specific for S3 API calls. Defaults to cloud.aws.region. * @see AwsS3Service#REGION_SETTING */ Setting REGION_SETTING = new Setting<>("cloud.aws.s3.region", AwsS3Service.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT), - Property.NodeScope); + Property.NodeScope, Property.Deprecated); /** * cloud.aws.s3.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting. */ @@ -161,9 +168,8 @@ public interface AwsS3Service extends LifecycleComponent { * @see AwsS3Service#READ_TIMEOUT */ Setting READ_TIMEOUT = - Setting.timeSetting("cloud.aws.s3.read_timeout", AwsS3Service.READ_TIMEOUT, Property.NodeScope); + Setting.timeSetting("cloud.aws.s3.read_timeout", AwsS3Service.READ_TIMEOUT, Property.NodeScope, Property.Deprecated); } - AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries, - boolean useThrottleRetries, Boolean pathStyleAccess); + AmazonS3 client(Settings repositorySettings, Integer maxRetries, boolean useThrottleRetries, Boolean pathStyleAccess); } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index c14aba91365..fd2effd5798 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -19,42 +19,40 @@ package org.elasticsearch.cloud.aws; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; import com.amazonaws.auth.InstanceProfileCredentialsProvider; import com.amazonaws.auth.SystemPropertiesCredentialsProvider; -import com.amazonaws.auth.profile.ProfileCredentialsProvider; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.S3ClientOptions; import org.apache.logging.log4j.Logger; -import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.repositories.s3.S3Repository; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Supplier; - -import static org.elasticsearch.repositories.s3.S3Repository.getValue; - public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service { + // pkg private for tests + static final Setting CLIENT_NAME = new Setting<>("client", "default", Function.identity()); + /** * (acceskey, endpoint) -> client */ @@ -65,11 +63,12 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements } @Override - public synchronized AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries, - boolean useThrottleRetries, Boolean pathStyleAccess) { - String foundEndpoint = findEndpoint(logger, settings, endpoint, region); + public synchronized AmazonS3 client(Settings repositorySettings, Integer maxRetries, + boolean useThrottleRetries, Boolean pathStyleAccess) { + String clientName = CLIENT_NAME.get(repositorySettings); + String foundEndpoint = findEndpoint(logger, repositorySettings, settings, clientName); - AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, settings, repositorySettings); + AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, settings, repositorySettings, clientName); Tuple clientDescriptor = new Tuple<>(foundEndpoint, credentials.getCredentials().getAWSAccessKeyId()); AmazonS3Client client = clients.get(clientDescriptor); @@ -79,7 +78,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements client = new AmazonS3Client( credentials, - buildConfiguration(logger, settings, protocol, maxRetries, foundEndpoint, useThrottleRetries)); + buildConfiguration(logger, repositorySettings, settings, clientName, maxRetries, foundEndpoint, useThrottleRetries)); if (pathStyleAccess != null) { client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess)); @@ -93,19 +92,27 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements return client; } - public static ClientConfiguration buildConfiguration(Logger logger, Settings settings, Protocol protocol, Integer maxRetries, - String endpoint, boolean useThrottleRetries) { + // pkg private for tests + static ClientConfiguration buildConfiguration(Logger logger, Settings repositorySettings, Settings settings, + String clientName, Integer maxRetries, String endpoint, + boolean useThrottleRetries) { ClientConfiguration clientConfiguration = new ClientConfiguration(); // the response metadata cache is only there for diagnostics purposes, // but can force objects from every response to the old generation. clientConfiguration.setResponseMetadataCacheSize(0); + Protocol protocol = getConfigValue(repositorySettings, settings, clientName, S3Repository.PROTOCOL_SETTING, + S3Repository.Repository.PROTOCOL_SETTING, S3Repository.Repositories.PROTOCOL_SETTING); clientConfiguration.setProtocol(protocol); - String proxyHost = CLOUD_S3.PROXY_HOST_SETTING.get(settings); + String proxyHost = getConfigValue(null, settings, clientName, + S3Repository.PROXY_HOST_SETTING, null, CLOUD_S3.PROXY_HOST_SETTING); if (Strings.hasText(proxyHost)) { - Integer proxyPort = CLOUD_S3.PROXY_PORT_SETTING.get(settings); - try (SecureString proxyUsername = CLOUD_S3.PROXY_USERNAME_SETTING.get(settings); - SecureString proxyPassword = CLOUD_S3.PROXY_PASSWORD_SETTING.get(settings)) { + Integer proxyPort = getConfigValue(null, settings, clientName, + S3Repository.PROXY_PORT_SETTING, null, CLOUD_S3.PROXY_PORT_SETTING); + try (SecureString proxyUsername = getConfigValue(null, settings, clientName, + S3Repository.PROXY_USERNAME_SETTING, null, CLOUD_S3.PROXY_USERNAME_SETTING); + SecureString proxyPassword = getConfigValue(null, settings, clientName, + S3Repository.PROXY_PASSWORD_SETTING, null, CLOUD_S3.PROXY_PASSWORD_SETTING)) { clientConfiguration .withProxyHost(proxyHost) @@ -128,18 +135,20 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsSigner.configureSigner(awsSigner, clientConfiguration, endpoint); } - clientConfiguration.setSocketTimeout((int) CLOUD_S3.READ_TIMEOUT.get(settings).millis()); + TimeValue readTimeout = getConfigValue(null, settings, clientName, + S3Repository.READ_TIMEOUT_SETTING, null, CLOUD_S3.READ_TIMEOUT); + clientConfiguration.setSocketTimeout((int)readTimeout.millis()); return clientConfiguration; } public static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger deprecationLogger, - Settings settings, Settings repositorySettings) { + Settings settings, Settings repositorySettings, String clientName) { AWSCredentialsProvider credentials; - try (SecureString key = getValue(repositorySettings, settings, - S3Repository.Repository.KEY_SETTING, S3Repository.Repositories.KEY_SETTING); - SecureString secret = getValue(repositorySettings, settings, - S3Repository.Repository.SECRET_SETTING, S3Repository.Repositories.SECRET_SETTING)) { + try (SecureString key = getConfigValue(repositorySettings, settings, clientName, S3Repository.ACCESS_KEY_SETTING, + S3Repository.Repository.KEY_SETTING, S3Repository.Repositories.KEY_SETTING); + SecureString secret = getConfigValue(repositorySettings, settings, clientName, S3Repository.SECRET_KEY_SETTING, + S3Repository.Repository.SECRET_SETTING, S3Repository.Repositories.SECRET_SETTING)) { if (key.length() == 0 && secret.length() == 0) { // create a "manual" chain of providers here, so we can log deprecation of unsupported methods @@ -183,7 +192,12 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements return null; } - protected static String findEndpoint(Logger logger, Settings settings, String endpoint, String region) { + // pkg private for tests + /** Returns the endpoint the client should use, based on the available endpoint settings found. */ + static String findEndpoint(Logger logger, Settings repositorySettings, Settings settings, String clientName) { + String region = getRegion(repositorySettings, settings); + String endpoint = getConfigValue(repositorySettings, settings, clientName, S3Repository.ENDPOINT_SETTING, + S3Repository.Repository.ENDPOINT_SETTING, S3Repository.Repositories.ENDPOINT_SETTING); if (Strings.isNullOrEmpty(endpoint)) { logger.debug("no repository level endpoint has been defined. Trying to guess from repository region [{}]", region); if (!region.isEmpty()) { @@ -207,6 +221,14 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements return endpoint; } + /** + * Return the region configured, or empty string. + * TODO: remove after https://github.com/elastic/elasticsearch/issues/22761 */ + public static String getRegion(Settings repositorySettings, Settings settings) { + return getConfigValue(repositorySettings, settings, CLIENT_NAME.get(repositorySettings), S3Repository.REGION_SETTING, + S3Repository.Repository.REGION_SETTING, S3Repository.Repositories.REGION_SETTING); + } + private static String getEndpoint(String region) { final String endpoint; switch (region) { @@ -276,6 +298,23 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements return endpoint; } + /** + * Find the setting value, trying first with named configs, + * then falling back to repository and global repositories settings. + */ + private static T getConfigValue(Settings repositorySettings, Settings globalSettings, String clientName, + Setting.AffixSetting configSetting, Setting repositorySetting, Setting globalSetting) { + Setting concreteSetting = configSetting.getConcreteSettingForNamespace(clientName); + if (concreteSetting.exists(globalSettings)) { + return concreteSetting.get(globalSettings); + } else if (repositorySetting == null) { + // no repository setting, just use global setting + return globalSetting.get(globalSettings); + } else { + return S3Repository.getValue(repositorySettings, globalSettings, repositorySetting, globalSetting); + } + } + @Override protected void doStart() throws ElasticsearchException { } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 63973f106d4..2c243ac30e5 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -21,8 +21,10 @@ package org.elasticsearch.repositories.s3; import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; +import com.amazonaws.services.s3.AmazonS3; import org.elasticsearch.cloud.aws.AwsS3Service; import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3; +import org.elasticsearch.cloud.aws.InternalAwsS3Service; import org.elasticsearch.cloud.aws.blobstore.S3BlobStore; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.Strings; @@ -31,10 +33,12 @@ import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.AffixSetting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.repositories.RepositoryException; @@ -42,7 +46,6 @@ import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import java.io.IOException; import java.util.Locale; -import java.util.function.Function; /** * Shared file system implementation of the BlobStoreRepository @@ -61,40 +64,86 @@ public class S3Repository extends BlobStoreRepository { public static final String TYPE = "s3"; + // prefix for s3 client settings + private static final String PREFIX = "s3.client."; + + /** The access key (ie login id) for connecting to s3. */ + public static final AffixSetting ACCESS_KEY_SETTING = Setting.affixKeySetting(PREFIX, "access_key", + key -> SecureSetting.secureString(key, Repositories.KEY_SETTING, false)); + + /** The secret key (ie password) for connecting to s3. */ + public static final AffixSetting SECRET_KEY_SETTING = Setting.affixKeySetting(PREFIX, "secret_key", + key -> SecureSetting.secureString(key, Repositories.SECRET_SETTING, false)); + + /** The region the s3 repository bucket should exist in. */ + public static final AffixSetting REGION_SETTING = Setting.affixKeySetting(PREFIX, "region", + key -> new Setting<>(key, "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope)); + + /** An override for the s3 endpoint to connect to. */ + public static final AffixSetting ENDPOINT_SETTING = Setting.affixKeySetting(PREFIX, "endpoint", + key -> new Setting<>(key, Repositories.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope)); + + /** The protocol to use to connec to to s3. */ + public static final AffixSetting PROTOCOL_SETTING = Setting.affixKeySetting(PREFIX, "protocol", + key -> new Setting<>(key, "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope)); + + /** The host name of a proxy to connect to s3 through. */ + public static final AffixSetting PROXY_HOST_SETTING = Setting.affixKeySetting(PREFIX, "proxy.host", + key -> Setting.simpleString(key, Property.NodeScope)); + + /** The port of a proxy to connect to s3 through. */ + public static final AffixSetting PROXY_PORT_SETTING = Setting.affixKeySetting(PREFIX, "proxy.port", + key -> Setting.intSetting(key, 80, 0, 1<<16, Property.NodeScope)); + + /** The username of a proxy to connect to s3 through. */ + public static final AffixSetting PROXY_USERNAME_SETTING = Setting.affixKeySetting(PREFIX, "proxy.username", + key -> SecureSetting.secureString(key, AwsS3Service.PROXY_USERNAME_SETTING, false)); + + /** The password of a proxy to connect to s3 through. */ + public static final AffixSetting PROXY_PASSWORD_SETTING = Setting.affixKeySetting(PREFIX, "proxy.password", + key -> SecureSetting.secureString(key, AwsS3Service.PROXY_PASSWORD_SETTING, false)); + + /** The socket timeout for connecting to s3. */ + public static final AffixSetting READ_TIMEOUT_SETTING = Setting.affixKeySetting(PREFIX, "read_timeout", + key -> Setting.timeSetting(key, TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope)); + /** * Global S3 repositories settings. Starting with: repositories.s3 + * NOTE: These are legacy settings. Use the named client config settings above. */ public interface Repositories { /** * repositories.s3.access_key: AWS Access key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.access_key. * @see CLOUD_S3#KEY_SETTING */ - SecureSetting KEY_SETTING = SecureSetting.secureString("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, true); + Setting KEY_SETTING = new Setting<>("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, SecureString::new, + Property.NodeScope, Property.Filtered, Property.Deprecated); /** * repositories.s3.secret_key: AWS Secret key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.secret_key. * @see CLOUD_S3#SECRET_SETTING */ - SecureSetting SECRET_SETTING = SecureSetting.secureString("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, true); + Setting SECRET_SETTING = new Setting<>("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, SecureString::new, + Property.NodeScope, Property.Filtered, Property.Deprecated); /** * repositories.s3.region: Region specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.region. * @see CLOUD_S3#REGION_SETTING */ - Setting REGION_SETTING = - new Setting<>("repositories.s3.region", CLOUD_S3.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope); + Setting REGION_SETTING = new Setting<>("repositories.s3.region", CLOUD_S3.REGION_SETTING, + s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated); /** * repositories.s3.endpoint: Endpoint specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.endpoint. * @see CLOUD_S3#ENDPOINT_SETTING */ - Setting ENDPOINT_SETTING = - new Setting<>("repositories.s3.endpoint", CLOUD_S3.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope); + Setting ENDPOINT_SETTING = new Setting<>("repositories.s3.endpoint", CLOUD_S3.ENDPOINT_SETTING, + s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated); /** * repositories.s3.protocol: Protocol specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.protocol. * @see CLOUD_S3#PROTOCOL_SETTING */ - Setting PROTOCOL_SETTING = - new Setting<>("repositories.s3.protocol", CLOUD_S3.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope); + Setting PROTOCOL_SETTING = new Setting<>("repositories.s3.protocol", CLOUD_S3.PROTOCOL_SETTING, + s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated); /** * repositories.s3.bucket: The name of the bucket to be used for snapshots. */ @@ -177,37 +226,30 @@ public class S3Repository extends BlobStoreRepository { * If undefined, they use the repositories.s3.xxx equivalent setting. */ public interface Repository { - /** - * access_key - * @see Repositories#KEY_SETTING - */ - SecureSetting KEY_SETTING = SecureSetting.secureString("access_key", null, true); + Setting KEY_SETTING = new Setting<>("access_key", "", SecureString::new, + Property.Filtered, Property.Deprecated); + + + Setting SECRET_SETTING = new Setting<>("secret_key", "", SecureString::new, + Property.Filtered, Property.Deprecated); - /** - * secret_key - * @see Repositories#SECRET_SETTING - */ - SecureSetting SECRET_SETTING = SecureSetting.secureString("secret_key", null, true); - /** - * bucket - * @see Repositories#BUCKET_SETTING - */ Setting BUCKET_SETTING = Setting.simpleString("bucket"); /** * endpoint * @see Repositories#ENDPOINT_SETTING */ - Setting ENDPOINT_SETTING = Setting.simpleString("endpoint"); + Setting ENDPOINT_SETTING = Setting.simpleString("endpoint", Property.Deprecated); /** * protocol * @see Repositories#PROTOCOL_SETTING */ - Setting PROTOCOL_SETTING = new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT))); + Setting PROTOCOL_SETTING = new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), + Property.Deprecated); /** * region * @see Repositories#REGION_SETTING */ - Setting REGION_SETTING = new Setting<>("region", "", s -> s.toLowerCase(Locale.ROOT)); + Setting REGION_SETTING = new Setting<>("region", "", s -> s.toLowerCase(Locale.ROOT), Property.Deprecated); /** * server_side_encryption * @see Repositories#SERVER_SIDE_ENCRYPTION_SETTING @@ -286,10 +328,6 @@ public class S3Repository extends BlobStoreRepository { throw new RepositoryException(metadata.name(), "No bucket defined for s3 gateway"); } - String endpoint = getValue(metadata.settings(), settings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING); - Protocol protocol = getValue(metadata.settings(), settings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING); - String region = getValue(metadata.settings(), settings, Repository.REGION_SETTING, Repositories.REGION_SETTING); - boolean serverSideEncryption = getValue(metadata.settings(), settings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING); ByteSizeValue bufferSize = getValue(metadata.settings(), settings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING); Integer maxRetries = getValue(metadata.settings(), settings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING); @@ -315,13 +353,14 @@ public class S3Repository extends BlobStoreRepository { pathStyleAccess = getValue(metadata.settings(), settings, Repository.PATH_STYLE_ACCESS_SETTING, Repositories.PATH_STYLE_ACCESS_SETTING); } - logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " + + logger.debug("using bucket [{}], chunk_size [{}], server_side_encryption [{}], " + "buffer_size [{}], max_retries [{}], use_throttle_retries [{}], cannedACL [{}], storageClass [{}], path_style_access [{}]", - bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL, + bucket, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL, storageClass, pathStyleAccess); - blobStore = new S3BlobStore(settings, - s3Service.client(metadata.settings(), endpoint, protocol, region, maxRetries, useThrottleRetries, pathStyleAccess), + AmazonS3 client = s3Service.client(metadata.settings(), maxRetries, useThrottleRetries, pathStyleAccess); + String region = InternalAwsS3Service.getRegion(metadata.settings(), settings); + blobStore = new S3BlobStore(settings, client, bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass); String basePath = getValue(metadata.settings(), settings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java index 094db8f423b..bfa9b82017c 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java @@ -23,7 +23,6 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.auth.InstanceProfileCredentialsProvider; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; @@ -38,104 +37,30 @@ public class AwsS3ServiceImplTests extends ESTestCase { public void testAWSCredentialsWithSystemProviders() { AWSCredentialsProvider credentialsProvider = - InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY); + InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY, "default"); assertThat(credentialsProvider, instanceOf(InstanceProfileCredentialsProvider.class)); } - public void testAWSCredentialsWithElasticsearchAwsSettings() { + public void testAwsCredsDefaultSettings() { Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key"); - secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret"); + secureSettings.setString("s3.client.default.access_key", "aws_key"); + secureSettings.setString("s3.client.default.secret_key", "aws_secret"); Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret"); } - public void testAWSCredentialsWithElasticsearchS3Settings() { + public void testAwsCredsExplicitConfigSettings() { Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); + repositorySettings = Settings.builder().put(repositorySettings) + .put(InternalAwsS3Service.CLIENT_NAME.getKey(), "myconfig").build(); MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key"); - secureSettings.setString(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret"); + secureSettings.setString("s3.client.myconfig.access_key", "aws_key"); + secureSettings.setString("s3.client.myconfig.secret_key", "aws_secret"); + secureSettings.setString("s3.client.default.access_key", "wrong_key"); + secureSettings.setString("s3.client.default.secret_key", "wrong_secret"); Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret"); - } - - public void testAWSCredentialsWithElasticsearchAwsAndS3Settings() { - Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key"); - secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret"); - secureSettings.setString(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key"); - secureSettings.setString(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret"); - } - - public void testAWSCredentialsWithElasticsearchRepositoriesSettings() { - Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key"); - secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret"); - } - - - public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettings() { - Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key"); - secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret"); - secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key"); - secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret"); - } - - public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettings() { - Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key"); - secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret"); - secureSettings.setString(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key"); - secureSettings.setString(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret"); - secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key"); - secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret"); - } - - public void testAWSCredentialsWithElasticsearchRepositoriesSettingsAndRepositorySettings() { - Settings repositorySettings = generateSecureRepositorySettings("repository_key", "repository_secret", "eu-central", null, null); - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key"); - secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret"); - } - - public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettingsAndRepositorySettings() { - Settings repositorySettings = generateSecureRepositorySettings("repository_key", "repository_secret", "eu-central", null, null); - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key"); - secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret"); - secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key"); - secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret"); - } - - public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettingsAndRepositorySettings() { - Settings repositorySettings = generateSecureRepositorySettings("repository_key", "repository_secret", "eu-central", null, null); - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key"); - secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret"); - secureSettings.setString(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key"); - secureSettings.setString(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret"); - secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key"); - secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret"); + launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret"); } public void testAWSCredentialsWithElasticsearchAwsSettingsBackcompat() { @@ -169,7 +94,9 @@ public class AwsS3ServiceImplTests extends ESTestCase { .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret") .build(); launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret"); - assertWarnings("[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated", + assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated", "[" + AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey() + "] setting was deprecated"); } @@ -193,7 +120,9 @@ public class AwsS3ServiceImplTests extends ESTestCase { .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret") .build(); launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret"); - assertWarnings("[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated", + assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated", + "[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated", "[" + S3Repository.Repositories.SECRET_SETTING.getKey() + "] setting was deprecated"); } @@ -208,7 +137,11 @@ public class AwsS3ServiceImplTests extends ESTestCase { .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret") .build(); launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret"); - assertWarnings("[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated", + assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey() + "] setting was deprecated", + "[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated", "[" + S3Repository.Repositories.SECRET_SETTING.getKey() + "] setting was deprecated"); } @@ -253,8 +186,9 @@ public class AwsS3ServiceImplTests extends ESTestCase { protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings singleRepositorySettings, Settings settings, String expectedKey, String expectedSecret) { - AWSCredentials credentials = - InternalAwsS3Service.buildCredentials(logger, deprecationLogger, settings, singleRepositorySettings).getCredentials(); + String configName = InternalAwsS3Service.CLIENT_NAME.get(singleRepositorySettings); + AWSCredentials credentials = InternalAwsS3Service.buildCredentials(logger, deprecationLogger, settings, + singleRepositorySettings, configName).getCredentials(); assertThat(credentials.getAWSAccessKeyId(), is(expectedKey)); assertThat(credentials.getAWSSecretKey(), is(expectedSecret)); } @@ -268,41 +202,17 @@ public class AwsS3ServiceImplTests extends ESTestCase { public void testAWSConfigurationWithAwsSettings() { Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username"); - secureSettings.setString(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password"); + secureSettings.setString("s3.client.default.proxy.username", "aws_proxy_username"); + secureSettings.setString("s3.client.default.proxy.password", "aws_proxy_password"); Settings settings = Settings.builder() .setSecureSettings(secureSettings) - .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http") - .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host") - .put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080) - .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType") - .put(AwsS3Service.READ_TIMEOUT.getKey(), "10s") + .put("s3.client.default.protocol", "http") + .put("s3.client.default.proxy.host", "aws_proxy_host") + .put("s3.client.default.proxy.port", 8080) + .put("s3.client.default.read_timeout", "10s") .build(); launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", - "aws_proxy_password", "AWS3SignerType", 3, false, 10000); - } - - public void testAWSConfigurationWithAwsAndS3Settings() { - Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username"); - secureSettings.setString(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password"); - secureSettings.setString(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey(), "s3_proxy_username"); - secureSettings.setString(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey(), "s3_proxy_password"); - Settings settings = Settings.builder() - .setSecureSettings(secureSettings) - .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http") - .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host") - .put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080) - .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType") - .put(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey(), "https") - .put(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey(), "s3_proxy_host") - .put(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey(), 8081) - .put(AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey(), "NoOpSignerType") - .put(AwsS3Service.CLOUD_S3.READ_TIMEOUT.getKey(), "10s") - .build(); - launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username", - "s3_proxy_password", "NoOpSignerType", 3, false, 10000); + "aws_proxy_password", null, 3, false, 10000); } public void testAWSConfigurationWithAwsSettingsBackcompat() { @@ -319,7 +229,12 @@ public class AwsS3ServiceImplTests extends ESTestCase { launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", "aws_proxy_password", "AWS3SignerType", 3, false, 10000); assertWarnings("[" + AwsS3Service.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated", - "[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated"); + "[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.PROTOCOL_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.PROXY_HOST_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.PROXY_PORT_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.SIGNER_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.READ_TIMEOUT.getKey() + "] setting was deprecated"); } public void testAWSConfigurationWithAwsAndS3SettingsBackcompat() { @@ -331,6 +246,7 @@ public class AwsS3ServiceImplTests extends ESTestCase { .put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username") .put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password") .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType") + .put(AwsS3Service.READ_TIMEOUT.getKey(), "5s") .put(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey(), "https") .put(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey(), "s3_proxy_host") .put(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey(), 8081) @@ -341,8 +257,38 @@ public class AwsS3ServiceImplTests extends ESTestCase { .build(); launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username", "s3_proxy_password", "NoOpSignerType", 3, false, 10000); - assertWarnings("[" + AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated", - "[" + AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated"); + assertWarnings("[" + AwsS3Service.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.PROTOCOL_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.PROXY_HOST_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.PROXY_PORT_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.SIGNER_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.READ_TIMEOUT.getKey() + "] setting was deprecated", + "[" + AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey() + "] setting was deprecated", + "[" + AwsS3Service.CLOUD_S3.READ_TIMEOUT.getKey() + "] setting was deprecated"); + } + + public void testGlobalMaxRetries() { + Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); + Settings settings = Settings.builder() + .put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10) + .build(); + launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null, + null, null, 10, false, 50000); + } + + public void testRepositoryMaxRetries() { + Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, 20); + Settings settings = Settings.builder() + .put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10) + .build(); + launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null, + null, null, 20, false, 50000); } protected void launchAWSConfigurationTest(Settings settings, @@ -356,15 +302,13 @@ public class AwsS3ServiceImplTests extends ESTestCase { Integer expectedMaxRetries, boolean expectedUseThrottleRetries, int expectedReadTimeout) { - Protocol protocol = S3Repository.getValue(singleRepositorySettings, settings, - S3Repository.Repository.PROTOCOL_SETTING, S3Repository.Repositories.PROTOCOL_SETTING); Integer maxRetries = S3Repository.getValue(singleRepositorySettings, settings, S3Repository.Repository.MAX_RETRIES_SETTING, S3Repository.Repositories.MAX_RETRIES_SETTING); Boolean useThrottleRetries = S3Repository.getValue(singleRepositorySettings, settings, S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING, S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING); - ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(logger, settings, protocol, maxRetries, null, - useThrottleRetries); + ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(logger, singleRepositorySettings, settings, + "default", maxRetries, null, useThrottleRetries); assertThat(configuration.getResponseMetadataCacheSize(), is(0)); assertThat(configuration.getProtocol(), is(expectedProtocol)); @@ -398,84 +342,83 @@ public class AwsS3ServiceImplTests extends ESTestCase { return builder.build(); } - private static Settings generateSecureRepositorySettings(String key, String secret, String region, String endpoint, - Integer maxRetries) { - Settings settings = generateRepositorySettings(null, null, region, endpoint, maxRetries); - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(S3Repository.Repository.KEY_SETTING.getKey(), key); - secureSettings.setString(S3Repository.Repository.SECRET_SETTING.getKey(), secret); - return Settings.builder().put(settings).setSecureSettings(secureSettings).build(); - } - public void testDefaultEndpoint() { - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), Settings.EMPTY, ""); - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), Settings.EMPTY, - "s3.eu-central-1.amazonaws.com"); - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), - Settings.EMPTY, "repository.endpoint"); + assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), Settings.EMPTY, ""); } - public void testSpecificEndpoint() { + public void testEndpointSetting() { Settings settings = Settings.builder() - .put(InternalAwsS3Service.CLOUD_S3.ENDPOINT_SETTING.getKey(), "ec2.endpoint") + .put("s3.client.default.endpoint", "s3.endpoint") .build(); - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, - "ec2.endpoint"); - // Endpoint has precedence on region. Whatever region we set, we won't use it - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings, - "ec2.endpoint"); - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), - settings, "repository.endpoint"); + assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, "s3.endpoint"); } - public void testRegionWithAwsSettings() { + public void testEndpointSettingBackcompat() { + assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), + Settings.EMPTY, "repository.endpoint"); + assertWarnings("[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated"); + Settings settings = Settings.builder() + .put(S3Repository.Repositories.ENDPOINT_SETTING.getKey(), "repositories.endpoint") + .build(); + assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, + "repositories.endpoint"); + assertWarnings("[" + S3Repository.Repositories.ENDPOINT_SETTING.getKey() + "] setting was deprecated"); + } + + public void testRegionSetting() { + Settings settings = Settings.builder() + .put("s3.client.default.region", randomFrom("eu-west", "eu-west-1")) + .build(); + assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, + "s3-eu-west-1.amazonaws.com"); + } + + public void testRegionSettingBackcompat() { Settings settings = Settings.builder() .put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1")) .build(); - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, + assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, "s3-eu-west-1.amazonaws.com"); - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings, + assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings, "s3.eu-central-1.amazonaws.com"); - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), + assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), settings, "repository.endpoint"); + assertWarnings("[" + InternalAwsS3Service.REGION_SETTING.getKey() + "] setting was deprecated", + "[" + S3Repository.Repository.REGION_SETTING.getKey() + "] setting was deprecated", + "[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated"); } - public void testRegionWithAwsAndS3Settings() { + public void testRegionAndEndpointSettingBackcompatPrecedence() { Settings settings = Settings.builder() .put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1")) .put(InternalAwsS3Service.CLOUD_S3.REGION_SETTING.getKey(), randomFrom("us-west", "us-west-1")) .build(); - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, + assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, "s3-us-west-1.amazonaws.com"); - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings, + assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings, "s3.eu-central-1.amazonaws.com"); - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), + assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), settings, "repository.endpoint"); + assertWarnings("[" + InternalAwsS3Service.REGION_SETTING.getKey() + "] setting was deprecated", + "[" + S3Repository.Repository.REGION_SETTING.getKey() + "] setting was deprecated", + "[" + InternalAwsS3Service.CLOUD_S3.REGION_SETTING.getKey() + "] setting was deprecated", + "[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated"); } public void testInvalidRegion() { Settings settings = Settings.builder() - .put(InternalAwsS3Service.REGION_SETTING.getKey(), "does-not-exist") + .put("s3.client.default.region", "does-not-exist") .build(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, null); + assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, null); }); assertThat(e.getMessage(), containsString("No automatic endpoint could be derived from region")); - - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings, - "s3.eu-central-1.amazonaws.com"); - launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), - settings, "repository.endpoint"); } - protected void launchAWSEndpointTest(Settings singleRepositorySettings, Settings settings, - String expectedEndpoint) { - String region = S3Repository.getValue(singleRepositorySettings, settings, - S3Repository.Repository.REGION_SETTING, S3Repository.Repositories.REGION_SETTING); - String endpoint = S3Repository.getValue(singleRepositorySettings, settings, - S3Repository.Repository.ENDPOINT_SETTING, S3Repository.Repositories.ENDPOINT_SETTING); - - String foundEndpoint = InternalAwsS3Service.findEndpoint(logger, settings, endpoint, region); + private void assertEndpoint(Settings repositorySettings, Settings settings, + String expectedEndpoint) { + String configName = InternalAwsS3Service.CLIENT_NAME.get(repositorySettings); + String foundEndpoint = InternalAwsS3Service.findEndpoint(logger, repositorySettings, settings, configName); assertThat(foundEndpoint, is(expectedEndpoint)); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/EnvironmentCredentialsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/EnvironmentCredentialsTests.java index e4d0d34efac..c98d3d14fb0 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/EnvironmentCredentialsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/EnvironmentCredentialsTests.java @@ -27,7 +27,7 @@ public class EnvironmentCredentialsTests extends ESTestCase { public void test() { AWSCredentialsProvider provider = - InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY); + InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY, "default"); // NOTE: env vars are setup by the test runner in gradle assertEquals("env_access", provider.getCredentials().getAWSAccessKeyId()); assertEquals("env_secret", provider.getCredentials().getAWSSecretKey()); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/SyspropCredentialsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/SyspropCredentialsTests.java index f8525b3dc13..608e7328e42 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/SyspropCredentialsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/SyspropCredentialsTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.test.ESTestCase; public class SyspropCredentialsTests extends ESTestCase { public void test() { AWSCredentialsProvider provider = - InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY); + InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY, "default"); // NOTE: sys props are setup by the test runner in gradle assertEquals("sysprop_access", provider.getCredentials().getAWSAccessKeyId()); assertEquals("sysprop_secret", provider.getCredentials().getAWSSecretKey()); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java index e7a57958a89..554e913c676 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java @@ -18,14 +18,14 @@ */ package org.elasticsearch.cloud.aws; -import com.amazonaws.Protocol; +import java.util.IdentityHashMap; + import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin; -import java.util.IdentityHashMap; - public class TestAwsS3Service extends InternalAwsS3Service { public static class TestPlugin extends S3RepositoryPlugin { @Override @@ -42,9 +42,9 @@ public class TestAwsS3Service extends InternalAwsS3Service { @Override - public synchronized AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries, - boolean useThrottleRetries, Boolean pathStyleAccess) { - return cachedWrapper(super.client(repositorySettings, endpoint, protocol, region, maxRetries, useThrottleRetries, pathStyleAccess)); + public synchronized AmazonS3 client(Settings repositorySettings, Integer maxRetries, + boolean useThrottleRetries, Boolean pathStyleAccess) { + return cachedWrapper(super.client(repositorySettings, maxRetries, useThrottleRetries, pathStyleAccess)); } private AmazonS3 cachedWrapper(AmazonS3 client) { diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index 4d7f30ed9db..fa41b978e20 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -197,12 +197,7 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase Settings settings = internalCluster().getInstance(Settings.class); Settings bucket = settings.getByPrefix("repositories.s3."); - AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client( - repositorySettings, - null, - S3Repository.Repositories.PROTOCOL_SETTING.get(settings), - S3Repository.Repositories.REGION_SETTING.get(settings), - null, randomBoolean(), null); + AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(repositorySettings, null, randomBoolean(), null); String bucketName = bucket.get("bucket"); logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath); @@ -467,16 +462,12 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase settings.getByPrefix("repositories.s3.external-bucket.") }; for (Settings bucket : buckets) { - String endpoint = bucket.get("endpoint", S3Repository.Repositories.ENDPOINT_SETTING.get(settings)); - Protocol protocol = S3Repository.Repositories.PROTOCOL_SETTING.get(settings); - String region = bucket.get("region", S3Repository.Repositories.REGION_SETTING.get(settings)); String bucketName = bucket.get("bucket"); // We check that settings has been set in elasticsearch.yml integration test file // as described in README assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue()); - AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(Settings.EMPTY, endpoint, protocol, region, null, - randomBoolean(), null); + AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(Settings.EMPTY, null, randomBoolean(), null); try { ObjectListing prevListing = null; //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html @@ -513,7 +504,7 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase client.deleteObjects(multiObjectDeleteRequest); } } catch (Exception ex) { - logger.warn((Supplier) () -> new ParameterizedMessage("Failed to delete S3 repository [{}] in [{}]", bucketName, region), ex); + logger.warn((Supplier) () -> new ParameterizedMessage("Failed to delete S3 repository [{}]", bucketName), ex); } } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java index 0446425229e..425adb7bf51 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java @@ -19,13 +19,13 @@ package org.elasticsearch.repositories.s3; -import com.amazonaws.Protocol; +import java.io.IOException; + import com.amazonaws.services.s3.AbstractAmazonS3; import com.amazonaws.services.s3.AmazonS3; import org.elasticsearch.cloud.aws.AwsS3Service; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -35,8 +35,6 @@ import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import java.io.IOException; - import static org.elasticsearch.repositories.s3.S3Repository.Repositories; import static org.elasticsearch.repositories.s3.S3Repository.Repository; import static org.elasticsearch.repositories.s3.S3Repository.getValue; @@ -62,19 +60,15 @@ public class S3RepositoryTests extends ESTestCase { @Override protected void doClose() {} @Override - public AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries, + public AmazonS3 client(Settings repositorySettings, Integer maxRetries, boolean useThrottleRetries, Boolean pathStyleAccess) { return new DummyS3Client(); } } public void testSettingsResolution() throws Exception { - MockSecureSettings secureSettings1 = new MockSecureSettings(); - secureSettings1.setString(Repository.KEY_SETTING.getKey(), "key1"); - Settings localSettings = Settings.builder().setSecureSettings(secureSettings1).build(); - MockSecureSettings secureSettings2 = new MockSecureSettings(); - secureSettings2.setString(Repositories.KEY_SETTING.getKey(), "key2"); - Settings globalSettings = Settings.builder().setSecureSettings(secureSettings2).build(); + Settings localSettings = Settings.builder().put(Repository.KEY_SETTING.getKey(), "key1").build(); + Settings globalSettings = Settings.builder().put(Repositories.KEY_SETTING.getKey(), "key2").build(); assertEquals(new SecureString("key1".toCharArray()), getValue(localSettings, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING)); @@ -84,6 +78,8 @@ public class S3RepositoryTests extends ESTestCase { getValue(Settings.EMPTY, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING)); assertEquals(new SecureString("".toCharArray()), getValue(Settings.EMPTY, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING)); + assertWarnings("[" + Repository.KEY_SETTING.getKey() + "] setting was deprecated", + "[" + Repositories.KEY_SETTING.getKey() + "] setting was deprecated"); } public void testInvalidChunkBufferSizeSettings() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index d4c24ccb617..133c87eb4b9 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -306,12 +306,12 @@ public abstract class ESTestCase extends LuceneTestCase { } try { final List actualWarnings = threadContext.getResponseHeaders().get(DeprecationLogger.WARNING_HEADER); - assertEquals("Expected " + expectedWarnings.length + " warnings but found " + actualWarnings.size() + "\nExpected: " - + Arrays.asList(expectedWarnings) + "\nActual: " + actualWarnings, - expectedWarnings.length, actualWarnings.size()); for (String msg : expectedWarnings) { assertThat(actualWarnings, hasItem(containsString(msg))); } + assertEquals("Expected " + expectedWarnings.length + " warnings but found " + actualWarnings.size() + "\nExpected: " + + Arrays.asList(expectedWarnings) + "\nActual: " + actualWarnings, + expectedWarnings.length, actualWarnings.size()); } finally { resetDeprecationLogger(); }