honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

S3 repository: Add named configurations (#22762) 

* S3 repository: Add named configurations

This change implements named configurations for s3 repository as
proposed in #22520. The access/secret key secure settings which were
added in #22479 are reverted, and the only secure settings are those
with the new named configs. All other previously used settings for the
connection are deprecated.

closes #22520
This commit is contained in:
Ryan Ernst 2017-01-27 10:42:45 -08:00 committed by GitHub
parent 0f58f3f34b
commit aad51d44ab
17 changed files with 368 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
View File

@ -338,12 +338,13 @@ final class Bootstrap {
INSTANCE.setup(true, environment);
/* TODO: close this once s3 repository doesn't try to read during repository construction
try {
// any secure settings must be read during node construction
IOUtils.close(keystore);
} catch (IOException e) {
throw new BootstrapException(e);
}
}*/
INSTANCE.start();

4
core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java
View File

@ -109,8 +109,8 @@ public abstract class SecureSetting<T> extends Setting<T> {
*
* This may be any sensitive string, e.g. a username, a password, an auth token, etc.
*/
public static SecureSetting<SecureString> secureString(String name, SecureSetting<SecureString> fallback,
boolean allowLegacy, Property... properties) {
public static Setting<SecureString> secureString(String name, Setting<SecureString> fallback,
boolean allowLegacy, Property... properties) {
final Setting<String> legacy;
if (allowLegacy) {
Property[] legacyProperties = ArrayUtils.concat(properties, LEGACY_PROPERTIES, Property.class);

13
core/src/main/java/org/elasticsearch/common/settings/SecureString.java
View File

@ -39,6 +39,19 @@ public final class SecureString implements CharSequence, Closeable {
this.chars = Objects.requireNonNull(chars);
}
/**
* Constructs a new SecureString from an existing String.
*
* NOTE: This is not actually secure, since the provided String cannot be deallocated, but
* this constructor allows for easy compatibility between new and old apis.
*
* @deprecated Only use for compatibility between deprecated string settings and new secure strings
*/
@Deprecated
public SecureString(String s) {
this(s.toCharArray());
}
/** Constant time equality to avoid potential timing attacks. */
@Override
public synchronized boolean equals(Object o) {

10
core/src/main/java/org/elasticsearch/common/settings/Setting.java
View File

@ -123,7 +123,7 @@ public class Setting<T> extends ToXContentToBytes {
private Setting(Key key, @Nullable Setting<T> fallbackSetting, Function<Settings, String> defaultValue, Function<String, T> parser,
Property... properties) {
assert this instanceof SecureSetting || parser.apply(defaultValue.apply(Settings.EMPTY)) != null || this.isGroupSetting()
assert this instanceof SecureSetting || this.isGroupSetting() || parser.apply(defaultValue.apply(Settings.EMPTY)) != null
: "parser returned null";
this.key = key;
this.fallbackSetting = fallbackSetting;
@ -527,6 +527,14 @@ public class Setting<T> extends ToXContentToBytes {
}
}
/**
* Get a setting with the given namespace filled in for prefix and suffix.
*/
public Setting<T> getConcreteSettingForNamespace(String namespace) {
String fullKey = key.toConcreteKey(namespace).toString();
return getConcreteSetting(fullKey);
}
@Override
public void diff(Settings.Builder builder, Settings source, Settings defaultSettings) {
matchStream(defaultSettings).forEach((key) -> getConcreteSetting(key).diff(builder, source, defaultSettings));

3
core/src/main/java/org/elasticsearch/common/settings/Settings.java
View File

@ -897,6 +897,9 @@ public final class Settings implements ToXContent {
public Builder put(Settings settings) {
removeNonArraysFieldsIfNewSettingsContainsFieldAsArray(settings.getAsMap());
map.putAll(settings.getAsMap());
if (settings.getSecureSettings() != null) {
setSecureSettings(settings.getSecureSettings());
}
return this;
}

1
core/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java
View File

@ -89,6 +89,7 @@ public class InternalSettingsPreparer {
initializeSettings(output, input, true, properties);
Environment environment = new Environment(output.build());
output = Settings.builder(); // start with a fresh output
boolean settingsFileFound = false;
Set<String> foundSuffixes = new HashSet<>();
for (String allowedSuffix : ALLOWED_SUFFIXES) {

13
core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
View File

@ -21,6 +21,10 @@ package org.elasticsearch.node.internal;
import org.elasticsearch.cli.MockTerminal;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.SecureSetting;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.env.Environment;
@ -161,4 +165,13 @@ public class InternalSettingsPreparerTests extends ESTestCase {
assertTrue(e.getMessage(), e.getMessage().contains(".properties"));
}
}
public void testSecureSettings() {
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString("foo", "secret");
Settings input = Settings.builder().put(baseEnvSettings).setSecureSettings(secureSettings).build();
Environment env = InternalSettingsPreparer.prepareEnvironment(input, null);
Setting<SecureString> fakeSetting = SecureSetting.secureString("foo", null, false);
assertEquals("secret", fakeSetting.get(env.settings()).toString());
}
}

82
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
View File

@ -19,139 +19,146 @@
package org.elasticsearch.cloud.aws;
import java.util.Locale;
import java.util.function.Function;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.settings.SecureSetting;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import java.util.Locale;
import java.util.function.Function;
public interface AwsS3Service extends LifecycleComponent {
// Global AWS settings (shared between discovery-ec2 and repository-s3)
// Legacy global AWS settings (shared between discovery-ec2 and repository-s3)
// Each setting starting with `cloud.aws` also exists in discovery-ec2 project. Don't forget to update
// the code there if you change anything here.
/**
* cloud.aws.access_key: AWS Access key. Shared with discovery-ec2 plugin
*/
SecureSetting<SecureString> KEY_SETTING = SecureSetting.secureString("cloud.aws.access_key", null, true, Property.Shared);
Setting<SecureString> KEY_SETTING = new Setting<>("cloud.aws.access_key", "", SecureString::new,
Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared);
/**
* cloud.aws.secret_key: AWS Secret key. Shared with discovery-ec2 plugin
*/
SecureSetting<SecureString> SECRET_SETTING = SecureSetting.secureString("cloud.aws.secret_key", null, true, Property.Shared);
Setting<SecureString> SECRET_SETTING = new Setting<>("cloud.aws.secret_key", "", SecureString::new,
Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared);
/**
* cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with discovery-ec2 plugin
*/
Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
Property.NodeScope, Property.Shared);
Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https",
s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated, Property.Shared);
/**
* cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with discovery-ec2 plugin
*/
Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope, Property.Shared);
Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host",
Property.NodeScope, Property.Deprecated, Property.Shared);
/**
* cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with discovery-ec2 plugin
*/
Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope,
Property.Shared);
Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16,
Property.NodeScope, Property.Deprecated, Property.Shared);
/**
* cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with discovery-ec2 plugin
*/
SecureSetting<SecureString> PROXY_USERNAME_SETTING =
SecureSetting.secureString("cloud.aws.proxy.username", null, true, Property.Shared);
Setting<SecureString> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.proxy.username", "", SecureString::new,
Property.NodeScope, Property.Deprecated, Property.Shared);
/**
* cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with discovery-ec2 plugin
*/
SecureSetting<SecureString> PROXY_PASSWORD_SETTING =
SecureSetting.secureString("cloud.aws.proxy.password", null, true, Property.Shared);
Setting<SecureString> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.proxy.password", "", SecureString::new,
Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared);
/**
* cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with discovery-ec2 plugin
*/
Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared);
Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer",
Property.NodeScope, Property.Deprecated, Property.Shared);
/**
* cloud.aws.region: Region. Shared with discovery-ec2 plugin
*/
Setting<String> REGION_SETTING =
new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared);
Setting<String> REGION_SETTING = new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT),
Property.NodeScope, Property.Deprecated, Property.Shared);
/**
* cloud.aws.read_timeout: Socket read timeout. Shared with discovery-ec2 plugin
*/
Setting<TimeValue> READ_TIMEOUT = Setting.timeSetting("cloud.aws.read_timeout",
TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Shared);
TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Deprecated, Property.Shared);
/**
* Defines specific s3 settings starting with cloud.aws.s3.
* NOTE: These are legacy settings. Use the named client configs in {@link org.elasticsearch.repositories.s3.S3Repository}.
*/
interface CLOUD_S3 {
/**
* cloud.aws.s3.access_key: AWS Access key specific for S3 API calls. Defaults to cloud.aws.access_key.
* @see AwsS3Service#KEY_SETTING
*/
SecureSetting<SecureString> KEY_SETTING = SecureSetting.secureString("cloud.aws.s3.access_key", AwsS3Service.KEY_SETTING, true);
Setting<SecureString> KEY_SETTING =
new Setting<>("cloud.aws.s3.access_key", AwsS3Service.KEY_SETTING, SecureString::new,
Property.NodeScope, Property.Filtered, Property.Deprecated);
/**
* cloud.aws.s3.secret_key: AWS Secret key specific for S3 API calls. Defaults to cloud.aws.secret_key.
* @see AwsS3Service#SECRET_SETTING
*/
SecureSetting<SecureString> SECRET_SETTING = SecureSetting.secureString("cloud.aws.s3.secret_key",
AwsS3Service.SECRET_SETTING, true);
Setting<SecureString> SECRET_SETTING =
new Setting<>("cloud.aws.s3.secret_key", AwsS3Service.SECRET_SETTING, SecureString::new,
Property.NodeScope, Property.Filtered, Property.Deprecated);
/**
* cloud.aws.s3.protocol: Protocol for AWS API specific for S3 API calls: http or https. Defaults to cloud.aws.protocol.
* @see AwsS3Service#PROTOCOL_SETTING
*/
Setting<Protocol> PROTOCOL_SETTING =
new Setting<>("cloud.aws.s3.protocol", AwsS3Service.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
Property.NodeScope);
Property.NodeScope, Property.Deprecated);
/**
* cloud.aws.s3.proxy.host: In case of proxy, define its hostname/IP specific for S3 API calls. Defaults to cloud.aws.proxy.host.
* @see AwsS3Service#PROXY_HOST_SETTING
*/
Setting<String> PROXY_HOST_SETTING =
new Setting<>("cloud.aws.s3.proxy.host", AwsS3Service.PROXY_HOST_SETTING, Function.identity(),
Property.NodeScope);
Property.NodeScope, Property.Deprecated);
/**
* cloud.aws.s3.proxy.port: In case of proxy, define its port specific for S3 API calls. Defaults to cloud.aws.proxy.port.
* @see AwsS3Service#PROXY_PORT_SETTING
*/
Setting<Integer> PROXY_PORT_SETTING =
new Setting<>("cloud.aws.s3.proxy.port", AwsS3Service.PROXY_PORT_SETTING,
s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.s3.proxy.port"), Property.NodeScope);
s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.s3.proxy.port"), Property.NodeScope, Property.Deprecated);
/**
* cloud.aws.s3.proxy.username: In case of proxy with auth, define the username specific for S3 API calls.
* Defaults to cloud.aws.proxy.username.
* @see AwsS3Service#PROXY_USERNAME_SETTING
*/
SecureSetting<SecureString> PROXY_USERNAME_SETTING =
SecureSetting.secureString("cloud.aws.s3.proxy.username", AwsS3Service.PROXY_USERNAME_SETTING, true);
Setting<SecureString> PROXY_USERNAME_SETTING =
new Setting<>("cloud.aws.s3.proxy.username", AwsS3Service.PROXY_USERNAME_SETTING, SecureString::new,
Property.NodeScope, Property.Deprecated);
/**
* cloud.aws.s3.proxy.password: In case of proxy with auth, define the password specific for S3 API calls.
* Defaults to cloud.aws.proxy.password.
* @see AwsS3Service#PROXY_PASSWORD_SETTING
*/
SecureSetting<SecureString> PROXY_PASSWORD_SETTING =
SecureSetting.secureString("cloud.aws.s3.proxy.password", AwsS3Service.PROXY_PASSWORD_SETTING, true);
Setting<SecureString> PROXY_PASSWORD_SETTING =
new Setting<>("cloud.aws.s3.proxy.password", AwsS3Service.PROXY_PASSWORD_SETTING, SecureString::new,
Property.NodeScope, Property.Filtered, Property.Deprecated);
/**
* cloud.aws.s3.signer: If you are using an old AWS API version, you can define a Signer. Specific for S3 API calls.
* Defaults to cloud.aws.signer.
* @see AwsS3Service#SIGNER_SETTING
*/
Setting<String> SIGNER_SETTING =
new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(), Property.NodeScope);
new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(),
Property.NodeScope, Property.Deprecated);
/**
* cloud.aws.s3.region: Region specific for S3 API calls. Defaults to cloud.aws.region.
* @see AwsS3Service#REGION_SETTING
*/
Setting<String> REGION_SETTING =
new Setting<>("cloud.aws.s3.region", AwsS3Service.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT),
Property.NodeScope);
Property.NodeScope, Property.Deprecated);
/**
* cloud.aws.s3.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
*/
@ -161,9 +168,8 @@ public interface AwsS3Service extends LifecycleComponent {
* @see AwsS3Service#READ_TIMEOUT
*/
Setting<TimeValue> READ_TIMEOUT =
Setting.timeSetting("cloud.aws.s3.read_timeout", AwsS3Service.READ_TIMEOUT, Property.NodeScope);
Setting.timeSetting("cloud.aws.s3.read_timeout", AwsS3Service.READ_TIMEOUT, Property.NodeScope, Property.Deprecated);
}
AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
boolean useThrottleRetries, Boolean pathStyleAccess);
AmazonS3 client(Settings repositorySettings, Integer maxRetries, boolean useThrottleRetries, Boolean pathStyleAccess);
}

97
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
View File

@ -19,42 +19,40 @@
package org.elasticsearch.cloud.aws;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Function;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.http.IdleConnectionReaper;
import com.amazonaws.internal.StaticCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.S3ClientOptions;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.repositories.s3.S3Repository;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Supplier;
import static org.elasticsearch.repositories.s3.S3Repository.getValue;
public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service {
// pkg private for tests
static final Setting<String> CLIENT_NAME = new Setting<>("client", "default", Function.identity());
/**
* (acceskey, endpoint) -&gt; client
*/
@ -65,11 +63,12 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
}
@Override
public synchronized AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
boolean useThrottleRetries, Boolean pathStyleAccess) {
String foundEndpoint = findEndpoint(logger, settings, endpoint, region);
public synchronized AmazonS3 client(Settings repositorySettings, Integer maxRetries,
boolean useThrottleRetries, Boolean pathStyleAccess) {
String clientName = CLIENT_NAME.get(repositorySettings);
String foundEndpoint = findEndpoint(logger, repositorySettings, settings, clientName);
AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, settings, repositorySettings);
AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, settings, repositorySettings, clientName);
Tuple<String, String> clientDescriptor = new Tuple<>(foundEndpoint, credentials.getCredentials().getAWSAccessKeyId());
AmazonS3Client client = clients.get(clientDescriptor);
@ -79,7 +78,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
client = new AmazonS3Client(
credentials,
buildConfiguration(logger, settings, protocol, maxRetries, foundEndpoint, useThrottleRetries));
buildConfiguration(logger, repositorySettings, settings, clientName, maxRetries, foundEndpoint, useThrottleRetries));
if (pathStyleAccess != null) {
client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess));
@ -93,19 +92,27 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
return client;
}
public static ClientConfiguration buildConfiguration(Logger logger, Settings settings, Protocol protocol, Integer maxRetries,
String endpoint, boolean useThrottleRetries) {
// pkg private for tests
static ClientConfiguration buildConfiguration(Logger logger, Settings repositorySettings, Settings settings,
String clientName, Integer maxRetries, String endpoint,
boolean useThrottleRetries) {
ClientConfiguration clientConfiguration = new ClientConfiguration();
// the response metadata cache is only there for diagnostics purposes,
// but can force objects from every response to the old generation.
clientConfiguration.setResponseMetadataCacheSize(0);
Protocol protocol = getConfigValue(repositorySettings, settings, clientName, S3Repository.PROTOCOL_SETTING,
S3Repository.Repository.PROTOCOL_SETTING, S3Repository.Repositories.PROTOCOL_SETTING);
clientConfiguration.setProtocol(protocol);
String proxyHost = CLOUD_S3.PROXY_HOST_SETTING.get(settings);
String proxyHost = getConfigValue(null, settings, clientName,
S3Repository.PROXY_HOST_SETTING, null, CLOUD_S3.PROXY_HOST_SETTING);
if (Strings.hasText(proxyHost)) {
Integer proxyPort = CLOUD_S3.PROXY_PORT_SETTING.get(settings);
try (SecureString proxyUsername = CLOUD_S3.PROXY_USERNAME_SETTING.get(settings);
SecureString proxyPassword = CLOUD_S3.PROXY_PASSWORD_SETTING.get(settings)) {
Integer proxyPort = getConfigValue(null, settings, clientName,
S3Repository.PROXY_PORT_SETTING, null, CLOUD_S3.PROXY_PORT_SETTING);
try (SecureString proxyUsername = getConfigValue(null, settings, clientName,
S3Repository.PROXY_USERNAME_SETTING, null, CLOUD_S3.PROXY_USERNAME_SETTING);
SecureString proxyPassword = getConfigValue(null, settings, clientName,
S3Repository.PROXY_PASSWORD_SETTING, null, CLOUD_S3.PROXY_PASSWORD_SETTING)) {
clientConfiguration
.withProxyHost(proxyHost)
@ -128,18 +135,20 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
AwsSigner.configureSigner(awsSigner, clientConfiguration, endpoint);
}
clientConfiguration.setSocketTimeout((int) CLOUD_S3.READ_TIMEOUT.get(settings).millis());
TimeValue readTimeout = getConfigValue(null, settings, clientName,
S3Repository.READ_TIMEOUT_SETTING, null, CLOUD_S3.READ_TIMEOUT);
clientConfiguration.setSocketTimeout((int)readTimeout.millis());
return clientConfiguration;
}
public static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger deprecationLogger,
Settings settings, Settings repositorySettings) {
Settings settings, Settings repositorySettings, String clientName) {
AWSCredentialsProvider credentials;
try (SecureString key = getValue(repositorySettings, settings,
S3Repository.Repository.KEY_SETTING, S3Repository.Repositories.KEY_SETTING);
SecureString secret = getValue(repositorySettings, settings,
S3Repository.Repository.SECRET_SETTING, S3Repository.Repositories.SECRET_SETTING)) {
try (SecureString key = getConfigValue(repositorySettings, settings, clientName, S3Repository.ACCESS_KEY_SETTING,
S3Repository.Repository.KEY_SETTING, S3Repository.Repositories.KEY_SETTING);
SecureString secret = getConfigValue(repositorySettings, settings, clientName, S3Repository.SECRET_KEY_SETTING,
S3Repository.Repository.SECRET_SETTING, S3Repository.Repositories.SECRET_SETTING)) {
if (key.length() == 0 && secret.length() == 0) {
// create a "manual" chain of providers here, so we can log deprecation of unsupported methods
@ -183,7 +192,12 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
return null;
}
protected static String findEndpoint(Logger logger, Settings settings, String endpoint, String region) {
// pkg private for tests
/** Returns the endpoint the client should use, based on the available endpoint settings found. */
static String findEndpoint(Logger logger, Settings repositorySettings, Settings settings, String clientName) {
String region = getRegion(repositorySettings, settings);
String endpoint = getConfigValue(repositorySettings, settings, clientName, S3Repository.ENDPOINT_SETTING,
S3Repository.Repository.ENDPOINT_SETTING, S3Repository.Repositories.ENDPOINT_SETTING);
if (Strings.isNullOrEmpty(endpoint)) {
logger.debug("no repository level endpoint has been defined. Trying to guess from repository region [{}]", region);
if (!region.isEmpty()) {
@ -207,6 +221,14 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
return endpoint;
}
/**
* Return the region configured, or empty string.
* TODO: remove after https://github.com/elastic/elasticsearch/issues/22761 */
public static String getRegion(Settings repositorySettings, Settings settings) {
return getConfigValue(repositorySettings, settings, CLIENT_NAME.get(repositorySettings), S3Repository.REGION_SETTING,
S3Repository.Repository.REGION_SETTING, S3Repository.Repositories.REGION_SETTING);
}
private static String getEndpoint(String region) {
final String endpoint;
switch (region) {
@ -276,6 +298,23 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
return endpoint;
}
/**
* Find the setting value, trying first with named configs,
* then falling back to repository and global repositories settings.
*/
private static <T> T getConfigValue(Settings repositorySettings, Settings globalSettings, String clientName,
Setting.AffixSetting<T> configSetting, Setting<T> repositorySetting, Setting<T> globalSetting) {
Setting<T> concreteSetting = configSetting.getConcreteSettingForNamespace(clientName);
if (concreteSetting.exists(globalSettings)) {
return concreteSetting.get(globalSettings);
} else if (repositorySetting == null) {
// no repository setting, just use global setting
return globalSetting.get(globalSettings);
} else {
return S3Repository.getValue(repositorySettings, globalSettings, repositorySetting, globalSetting);
}
}
@Override
protected void doStart() throws ElasticsearchException {
}

107
plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View File

@ -21,8 +21,10 @@ package org.elasticsearch.repositories.s3;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3;
import org.elasticsearch.cloud.aws.AwsS3Service;
import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3;
import org.elasticsearch.cloud.aws.InternalAwsS3Service;
import org.elasticsearch.cloud.aws.blobstore.S3BlobStore;
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
import org.elasticsearch.common.Strings;
@ -31,10 +33,12 @@ import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.settings.SecureSetting;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.AffixSetting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.repositories.RepositoryException;
@ -42,7 +46,6 @@ import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
import java.io.IOException;
import java.util.Locale;
import java.util.function.Function;
/**
* Shared file system implementation of the BlobStoreRepository
@ -61,40 +64,86 @@ public class S3Repository extends BlobStoreRepository {
public static final String TYPE = "s3";
// prefix for s3 client settings
private static final String PREFIX = "s3.client.";
/** The access key (ie login id) for connecting to s3. */
public static final AffixSetting<SecureString> ACCESS_KEY_SETTING = Setting.affixKeySetting(PREFIX, "access_key",
key -> SecureSetting.secureString(key, Repositories.KEY_SETTING, false));
/** The secret key (ie password) for connecting to s3. */
public static final AffixSetting<SecureString> SECRET_KEY_SETTING = Setting.affixKeySetting(PREFIX, "secret_key",
key -> SecureSetting.secureString(key, Repositories.SECRET_SETTING, false));
/** The region the s3 repository bucket should exist in. */
public static final AffixSetting<String> REGION_SETTING = Setting.affixKeySetting(PREFIX, "region",
key -> new Setting<>(key, "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope));
/** An override for the s3 endpoint to connect to. */
public static final AffixSetting<String> ENDPOINT_SETTING = Setting.affixKeySetting(PREFIX, "endpoint",
key -> new Setting<>(key, Repositories.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope));
/** The protocol to use to connec to to s3. */
public static final AffixSetting<Protocol> PROTOCOL_SETTING = Setting.affixKeySetting(PREFIX, "protocol",
key -> new Setting<>(key, "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope));
/** The host name of a proxy to connect to s3 through. */
public static final AffixSetting<String> PROXY_HOST_SETTING = Setting.affixKeySetting(PREFIX, "proxy.host",
key -> Setting.simpleString(key, Property.NodeScope));
/** The port of a proxy to connect to s3 through. */
public static final AffixSetting<Integer> PROXY_PORT_SETTING = Setting.affixKeySetting(PREFIX, "proxy.port",
key -> Setting.intSetting(key, 80, 0, 1<<16, Property.NodeScope));
/** The username of a proxy to connect to s3 through. */
public static final AffixSetting<SecureString> PROXY_USERNAME_SETTING = Setting.affixKeySetting(PREFIX, "proxy.username",
key -> SecureSetting.secureString(key, AwsS3Service.PROXY_USERNAME_SETTING, false));
/** The password of a proxy to connect to s3 through. */
public static final AffixSetting<SecureString> PROXY_PASSWORD_SETTING = Setting.affixKeySetting(PREFIX, "proxy.password",
key -> SecureSetting.secureString(key, AwsS3Service.PROXY_PASSWORD_SETTING, false));
/** The socket timeout for connecting to s3. */
public static final AffixSetting<TimeValue> READ_TIMEOUT_SETTING = Setting.affixKeySetting(PREFIX, "read_timeout",
key -> Setting.timeSetting(key, TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope));
/**
* Global S3 repositories settings. Starting with: repositories.s3
* NOTE: These are legacy settings. Use the named client config settings above.
*/
public interface Repositories {
/**
* repositories.s3.access_key: AWS Access key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.access_key.
* @see CLOUD_S3#KEY_SETTING
*/
SecureSetting<SecureString> KEY_SETTING = SecureSetting.secureString("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, true);
Setting<SecureString> KEY_SETTING = new Setting<>("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, SecureString::new,
Property.NodeScope, Property.Filtered, Property.Deprecated);
/**
* repositories.s3.secret_key: AWS Secret key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.secret_key.
* @see CLOUD_S3#SECRET_SETTING
*/
SecureSetting<SecureString> SECRET_SETTING = SecureSetting.secureString("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, true);
Setting<SecureString> SECRET_SETTING = new Setting<>("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, SecureString::new,
Property.NodeScope, Property.Filtered, Property.Deprecated);
/**
* repositories.s3.region: Region specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.region.
* @see CLOUD_S3#REGION_SETTING
*/
Setting<String> REGION_SETTING =
new Setting<>("repositories.s3.region", CLOUD_S3.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
Setting<String> REGION_SETTING = new Setting<>("repositories.s3.region", CLOUD_S3.REGION_SETTING,
s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated);
/**
* repositories.s3.endpoint: Endpoint specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.endpoint.
* @see CLOUD_S3#ENDPOINT_SETTING
*/
Setting<String> ENDPOINT_SETTING =
new Setting<>("repositories.s3.endpoint", CLOUD_S3.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
Setting<String> ENDPOINT_SETTING = new Setting<>("repositories.s3.endpoint", CLOUD_S3.ENDPOINT_SETTING,
s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated);
/**
* repositories.s3.protocol: Protocol specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.protocol.
* @see CLOUD_S3#PROTOCOL_SETTING
*/
Setting<Protocol> PROTOCOL_SETTING =
new Setting<>("repositories.s3.protocol", CLOUD_S3.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope);
Setting<Protocol> PROTOCOL_SETTING = new Setting<>("repositories.s3.protocol", CLOUD_S3.PROTOCOL_SETTING,
s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated);
/**
* repositories.s3.bucket: The name of the bucket to be used for snapshots.
*/
@ -177,37 +226,30 @@ public class S3Repository extends BlobStoreRepository {
* If undefined, they use the repositories.s3.xxx equivalent setting.
*/
public interface Repository {
/**
* access_key
* @see Repositories#KEY_SETTING
*/
SecureSetting<SecureString> KEY_SETTING = SecureSetting.secureString("access_key", null, true);
Setting<SecureString> KEY_SETTING = new Setting<>("access_key", "", SecureString::new,
Property.Filtered, Property.Deprecated);
Setting<SecureString> SECRET_SETTING = new Setting<>("secret_key", "", SecureString::new,
Property.Filtered, Property.Deprecated);
/**
* secret_key
* @see Repositories#SECRET_SETTING
*/
SecureSetting<SecureString> SECRET_SETTING = SecureSetting.secureString("secret_key", null, true);
/**
* bucket
* @see Repositories#BUCKET_SETTING
*/
Setting<String> BUCKET_SETTING = Setting.simpleString("bucket");
/**
* endpoint
* @see Repositories#ENDPOINT_SETTING
*/
Setting<String> ENDPOINT_SETTING = Setting.simpleString("endpoint");
Setting<String> ENDPOINT_SETTING = Setting.simpleString("endpoint", Property.Deprecated);
/**
* protocol
* @see Repositories#PROTOCOL_SETTING
*/
Setting<Protocol> PROTOCOL_SETTING = new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)));
Setting<Protocol> PROTOCOL_SETTING = new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
Property.Deprecated);
/**
* region
* @see Repositories#REGION_SETTING
*/
Setting<String> REGION_SETTING = new Setting<>("region", "", s -> s.toLowerCase(Locale.ROOT));
Setting<String> REGION_SETTING = new Setting<>("region", "", s -> s.toLowerCase(Locale.ROOT), Property.Deprecated);
/**
* server_side_encryption
* @see Repositories#SERVER_SIDE_ENCRYPTION_SETTING
@ -286,10 +328,6 @@ public class S3Repository extends BlobStoreRepository {
throw new RepositoryException(metadata.name(), "No bucket defined for s3 gateway");
}
String endpoint = getValue(metadata.settings(), settings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING);
Protocol protocol = getValue(metadata.settings(), settings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING);
String region = getValue(metadata.settings(), settings, Repository.REGION_SETTING, Repositories.REGION_SETTING);
boolean serverSideEncryption = getValue(metadata.settings(), settings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
ByteSizeValue bufferSize = getValue(metadata.settings(), settings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING);
Integer maxRetries = getValue(metadata.settings(), settings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING);
@ -315,13 +353,14 @@ public class S3Repository extends BlobStoreRepository {
pathStyleAccess = getValue(metadata.settings(), settings, Repository.PATH_STYLE_ACCESS_SETTING, Repositories.PATH_STYLE_ACCESS_SETTING);
}
logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " +
logger.debug("using bucket [{}], chunk_size [{}], server_side_encryption [{}], " +
"buffer_size [{}], max_retries [{}], use_throttle_retries [{}], cannedACL [{}], storageClass [{}], path_style_access [{}]",
bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
bucket, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
storageClass, pathStyleAccess);
blobStore = new S3BlobStore(settings,
s3Service.client(metadata.settings(), endpoint, protocol, region, maxRetries, useThrottleRetries, pathStyleAccess),
AmazonS3 client = s3Service.client(metadata.settings(), maxRetries, useThrottleRetries, pathStyleAccess);
String region = InternalAwsS3Service.getRegion(metadata.settings(), settings);
blobStore = new S3BlobStore(settings, client,
bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
String basePath = getValue(metadata.settings(), settings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);

295
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java
View File

@ -23,7 +23,6 @@ import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.Settings;
@ -38,104 +37,30 @@ public class AwsS3ServiceImplTests extends ESTestCase {
public void testAWSCredentialsWithSystemProviders() {
AWSCredentialsProvider credentialsProvider =
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY);
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY, "default");
assertThat(credentialsProvider, instanceOf(InstanceProfileCredentialsProvider.class));
}
public void testAWSCredentialsWithElasticsearchAwsSettings() {
public void testAwsCredsDefaultSettings() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key");
secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret");
secureSettings.setString("s3.client.default.access_key", "aws_key");
secureSettings.setString("s3.client.default.secret_key", "aws_secret");
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret");
}
public void testAWSCredentialsWithElasticsearchS3Settings() {
public void testAwsCredsExplicitConfigSettings() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
repositorySettings = Settings.builder().put(repositorySettings)
.put(InternalAwsS3Service.CLIENT_NAME.getKey(), "myconfig").build();
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key");
secureSettings.setString(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret");
secureSettings.setString("s3.client.myconfig.access_key", "aws_key");
secureSettings.setString("s3.client.myconfig.secret_key", "aws_secret");
secureSettings.setString("s3.client.default.access_key", "wrong_key");
secureSettings.setString("s3.client.default.secret_key", "wrong_secret");
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
}
public void testAWSCredentialsWithElasticsearchAwsAndS3Settings() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key");
secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret");
secureSettings.setString(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key");
secureSettings.setString(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret");
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
}
public void testAWSCredentialsWithElasticsearchRepositoriesSettings() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key");
secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret");
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
}
public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettings() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key");
secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret");
secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key");
secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret");
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
}
public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettings() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key");
secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret");
secureSettings.setString(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key");
secureSettings.setString(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret");
secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key");
secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret");
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
}
public void testAWSCredentialsWithElasticsearchRepositoriesSettingsAndRepositorySettings() {
Settings repositorySettings = generateSecureRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key");
secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret");
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
}
public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettingsAndRepositorySettings() {
Settings repositorySettings = generateSecureRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key");
secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret");
secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key");
secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret");
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
}
public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettingsAndRepositorySettings() {
Settings repositorySettings = generateSecureRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key");
secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret");
secureSettings.setString(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key");
secureSettings.setString(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret");
secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key");
secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret");
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret");
}
public void testAWSCredentialsWithElasticsearchAwsSettingsBackcompat() {
@ -169,7 +94,9 @@ public class AwsS3ServiceImplTests extends ESTestCase {
.put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
.build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
assertWarnings("[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated",
assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey() + "] setting was deprecated");
}
@ -193,7 +120,9 @@ public class AwsS3ServiceImplTests extends ESTestCase {
.put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
.build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
assertWarnings("[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repositories.SECRET_SETTING.getKey() + "] setting was deprecated");
}
@ -208,7 +137,11 @@ public class AwsS3ServiceImplTests extends ESTestCase {
.put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
.build();
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
assertWarnings("[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repositories.SECRET_SETTING.getKey() + "] setting was deprecated");
}
@ -253,8 +186,9 @@ public class AwsS3ServiceImplTests extends ESTestCase {
protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings singleRepositorySettings, Settings settings,
String expectedKey, String expectedSecret) {
AWSCredentials credentials =
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, settings, singleRepositorySettings).getCredentials();
String configName = InternalAwsS3Service.CLIENT_NAME.get(singleRepositorySettings);
AWSCredentials credentials = InternalAwsS3Service.buildCredentials(logger, deprecationLogger, settings,
singleRepositorySettings, configName).getCredentials();
assertThat(credentials.getAWSAccessKeyId(), is(expectedKey));
assertThat(credentials.getAWSSecretKey(), is(expectedSecret));
}
@ -268,41 +202,17 @@ public class AwsS3ServiceImplTests extends ESTestCase {
public void testAWSConfigurationWithAwsSettings() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username");
secureSettings.setString(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password");
secureSettings.setString("s3.client.default.proxy.username", "aws_proxy_username");
secureSettings.setString("s3.client.default.proxy.password", "aws_proxy_password");
Settings settings = Settings.builder()
.setSecureSettings(secureSettings)
.put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http")
.put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
.put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080)
.put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
.put(AwsS3Service.READ_TIMEOUT.getKey(), "10s")
.put("s3.client.default.protocol", "http")
.put("s3.client.default.proxy.host", "aws_proxy_host")
.put("s3.client.default.proxy.port", 8080)
.put("s3.client.default.read_timeout", "10s")
.build();
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
"aws_proxy_password", "AWS3SignerType", 3, false, 10000);
}
public void testAWSConfigurationWithAwsAndS3Settings() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username");
secureSettings.setString(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password");
secureSettings.setString(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey(), "s3_proxy_username");
secureSettings.setString(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey(), "s3_proxy_password");
Settings settings = Settings.builder()
.setSecureSettings(secureSettings)
.put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http")
.put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
.put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080)
.put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
.put(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey(), "https")
.put(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey(), "s3_proxy_host")
.put(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey(), 8081)
.put(AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey(), "NoOpSignerType")
.put(AwsS3Service.CLOUD_S3.READ_TIMEOUT.getKey(), "10s")
.build();
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username",
"s3_proxy_password", "NoOpSignerType", 3, false, 10000);
"aws_proxy_password", null, 3, false, 10000);
}
public void testAWSConfigurationWithAwsSettingsBackcompat() {
@ -319,7 +229,12 @@ public class AwsS3ServiceImplTests extends ESTestCase {
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
"aws_proxy_password", "AWS3SignerType", 3, false, 10000);
assertWarnings("[" + AwsS3Service.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated");
"[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.PROTOCOL_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.PROXY_HOST_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.PROXY_PORT_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.SIGNER_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.READ_TIMEOUT.getKey() + "] setting was deprecated");
}
public void testAWSConfigurationWithAwsAndS3SettingsBackcompat() {
@ -331,6 +246,7 @@ public class AwsS3ServiceImplTests extends ESTestCase {
.put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
.put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
.put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
.put(AwsS3Service.READ_TIMEOUT.getKey(), "5s")
.put(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey(), "https")
.put(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey(), "s3_proxy_host")
.put(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey(), 8081)
@ -341,8 +257,38 @@ public class AwsS3ServiceImplTests extends ESTestCase {
.build();
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username",
"s3_proxy_password", "NoOpSignerType", 3, false, 10000);
assertWarnings("[" + AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated");
assertWarnings("[" + AwsS3Service.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.PROTOCOL_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.PROXY_HOST_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.PROXY_PORT_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.SIGNER_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.READ_TIMEOUT.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey() + "] setting was deprecated",
"[" + AwsS3Service.CLOUD_S3.READ_TIMEOUT.getKey() + "] setting was deprecated");
}
public void testGlobalMaxRetries() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
Settings settings = Settings.builder()
.put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10)
.build();
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null,
null, null, 10, false, 50000);
}
public void testRepositoryMaxRetries() {
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, 20);
Settings settings = Settings.builder()
.put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10)
.build();
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null,
null, null, 20, false, 50000);
}
protected void launchAWSConfigurationTest(Settings settings,
@ -356,15 +302,13 @@ public class AwsS3ServiceImplTests extends ESTestCase {
Integer expectedMaxRetries,
boolean expectedUseThrottleRetries,
int expectedReadTimeout) {
Protocol protocol = S3Repository.getValue(singleRepositorySettings, settings,
S3Repository.Repository.PROTOCOL_SETTING, S3Repository.Repositories.PROTOCOL_SETTING);
Integer maxRetries = S3Repository.getValue(singleRepositorySettings, settings,
S3Repository.Repository.MAX_RETRIES_SETTING, S3Repository.Repositories.MAX_RETRIES_SETTING);
Boolean useThrottleRetries = S3Repository.getValue(singleRepositorySettings, settings,
S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING, S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING);
ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(logger, settings, protocol, maxRetries, null,
useThrottleRetries);
ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(logger, singleRepositorySettings, settings,
"default", maxRetries, null, useThrottleRetries);
assertThat(configuration.getResponseMetadataCacheSize(), is(0));
assertThat(configuration.getProtocol(), is(expectedProtocol));
@ -398,84 +342,83 @@ public class AwsS3ServiceImplTests extends ESTestCase {
return builder.build();
}
private static Settings generateSecureRepositorySettings(String key, String secret, String region, String endpoint,
Integer maxRetries) {
Settings settings = generateRepositorySettings(null, null, region, endpoint, maxRetries);
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString(S3Repository.Repository.KEY_SETTING.getKey(), key);
secureSettings.setString(S3Repository.Repository.SECRET_SETTING.getKey(), secret);
return Settings.builder().put(settings).setSecureSettings(secureSettings).build();
}
public void testDefaultEndpoint() {
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), Settings.EMPTY, "");
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), Settings.EMPTY,
"s3.eu-central-1.amazonaws.com");
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
Settings.EMPTY, "repository.endpoint");
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), Settings.EMPTY, "");
}
public void testSpecificEndpoint() {
public void testEndpointSetting() {
Settings settings = Settings.builder()
.put(InternalAwsS3Service.CLOUD_S3.ENDPOINT_SETTING.getKey(), "ec2.endpoint")
.put("s3.client.default.endpoint", "s3.endpoint")
.build();
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
"ec2.endpoint");
// Endpoint has precedence on region. Whatever region we set, we won't use it
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
"ec2.endpoint");
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
settings, "repository.endpoint");
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, "s3.endpoint");
}
public void testRegionWithAwsSettings() {
public void testEndpointSettingBackcompat() {
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
Settings.EMPTY, "repository.endpoint");
assertWarnings("[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
Settings settings = Settings.builder()
.put(S3Repository.Repositories.ENDPOINT_SETTING.getKey(), "repositories.endpoint")
.build();
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
"repositories.endpoint");
assertWarnings("[" + S3Repository.Repositories.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
}
public void testRegionSetting() {
Settings settings = Settings.builder()
.put("s3.client.default.region", randomFrom("eu-west", "eu-west-1"))
.build();
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
"s3-eu-west-1.amazonaws.com");
}
public void testRegionSettingBackcompat() {
Settings settings = Settings.builder()
.put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
.build();
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
"s3-eu-west-1.amazonaws.com");
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
"s3.eu-central-1.amazonaws.com");
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
settings, "repository.endpoint");
assertWarnings("[" + InternalAwsS3Service.REGION_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repository.REGION_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
}
public void testRegionWithAwsAndS3Settings() {
public void testRegionAndEndpointSettingBackcompatPrecedence() {
Settings settings = Settings.builder()
.put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
.put(InternalAwsS3Service.CLOUD_S3.REGION_SETTING.getKey(), randomFrom("us-west", "us-west-1"))
.build();
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
"s3-us-west-1.amazonaws.com");
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
"s3.eu-central-1.amazonaws.com");
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
settings, "repository.endpoint");
assertWarnings("[" + InternalAwsS3Service.REGION_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repository.REGION_SETTING.getKey() + "] setting was deprecated",
"[" + InternalAwsS3Service.CLOUD_S3.REGION_SETTING.getKey() + "] setting was deprecated",
"[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
}
public void testInvalidRegion() {
Settings settings = Settings.builder()
.put(InternalAwsS3Service.REGION_SETTING.getKey(), "does-not-exist")
.put("s3.client.default.region", "does-not-exist")
.build();
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, null);
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, null);
});
assertThat(e.getMessage(), containsString("No automatic endpoint could be derived from region"));
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
"s3.eu-central-1.amazonaws.com");
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
settings, "repository.endpoint");
}
protected void launchAWSEndpointTest(Settings singleRepositorySettings, Settings settings,
String expectedEndpoint) {
String region = S3Repository.getValue(singleRepositorySettings, settings,
S3Repository.Repository.REGION_SETTING, S3Repository.Repositories.REGION_SETTING);
String endpoint = S3Repository.getValue(singleRepositorySettings, settings,
S3Repository.Repository.ENDPOINT_SETTING, S3Repository.Repositories.ENDPOINT_SETTING);
String foundEndpoint = InternalAwsS3Service.findEndpoint(logger, settings, endpoint, region);
private void assertEndpoint(Settings repositorySettings, Settings settings,
String expectedEndpoint) {
String configName = InternalAwsS3Service.CLIENT_NAME.get(repositorySettings);
String foundEndpoint = InternalAwsS3Service.findEndpoint(logger, repositorySettings, settings, configName);
assertThat(foundEndpoint, is(expectedEndpoint));
}

2
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/EnvironmentCredentialsTests.java
View File

@ -27,7 +27,7 @@ public class EnvironmentCredentialsTests extends ESTestCase {
public void test() {
AWSCredentialsProvider provider =
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY);
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY, "default");
// NOTE: env vars are setup by the test runner in gradle
assertEquals("env_access", provider.getCredentials().getAWSAccessKeyId());
assertEquals("env_secret", provider.getCredentials().getAWSSecretKey());

2
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/SyspropCredentialsTests.java
View File

@ -26,7 +26,7 @@ import org.elasticsearch.test.ESTestCase;
public class SyspropCredentialsTests extends ESTestCase {
public void test() {
AWSCredentialsProvider provider =
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY);
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY, "default");
// NOTE: sys props are setup by the test runner in gradle
assertEquals("sysprop_access", provider.getCredentials().getAWSAccessKeyId());
assertEquals("sysprop_secret", provider.getCredentials().getAWSSecretKey());

12
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
View File

@ -18,14 +18,14 @@
*/
package org.elasticsearch.cloud.aws;
import com.amazonaws.Protocol;
import java.util.IdentityHashMap;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin;
import java.util.IdentityHashMap;
public class TestAwsS3Service extends InternalAwsS3Service {
public static class TestPlugin extends S3RepositoryPlugin {
@Override
@ -42,9 +42,9 @@ public class TestAwsS3Service extends InternalAwsS3Service {
@Override
public synchronized AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
boolean useThrottleRetries, Boolean pathStyleAccess) {
return cachedWrapper(super.client(repositorySettings, endpoint, protocol, region, maxRetries, useThrottleRetries, pathStyleAccess));
public synchronized AmazonS3 client(Settings repositorySettings, Integer maxRetries,
boolean useThrottleRetries, Boolean pathStyleAccess) {
return cachedWrapper(super.client(repositorySettings, maxRetries, useThrottleRetries, pathStyleAccess));
}
private AmazonS3 cachedWrapper(AmazonS3 client) {

15
plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
View File

@ -197,12 +197,7 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
Settings settings = internalCluster().getInstance(Settings.class);
Settings bucket = settings.getByPrefix("repositories.s3.");
AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(
repositorySettings,
null,
S3Repository.Repositories.PROTOCOL_SETTING.get(settings),
S3Repository.Repositories.REGION_SETTING.get(settings),
null, randomBoolean(), null);
AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(repositorySettings, null, randomBoolean(), null);
String bucketName = bucket.get("bucket");
logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
@ -467,16 +462,12 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
settings.getByPrefix("repositories.s3.external-bucket.")
};
for (Settings bucket : buckets) {
String endpoint = bucket.get("endpoint", S3Repository.Repositories.ENDPOINT_SETTING.get(settings));
Protocol protocol = S3Repository.Repositories.PROTOCOL_SETTING.get(settings);
String region = bucket.get("region", S3Repository.Repositories.REGION_SETTING.get(settings));
String bucketName = bucket.get("bucket");
// We check that settings has been set in elasticsearch.yml integration test file
// as described in README
assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(Settings.EMPTY, endpoint, protocol, region, null,
randomBoolean(), null);
AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(Settings.EMPTY, null, randomBoolean(), null);
try {
ObjectListing prevListing = null;
//From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
@ -513,7 +504,7 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
client.deleteObjects(multiObjectDeleteRequest);
}
} catch (Exception ex) {
logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to delete S3 repository [{}] in [{}]", bucketName, region), ex);
logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to delete S3 repository [{}]", bucketName), ex);
}
}
}

18
plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
View File

@ -19,13 +19,13 @@
package org.elasticsearch.repositories.s3;
import com.amazonaws.Protocol;
import java.io.IOException;
import com.amazonaws.services.s3.AbstractAmazonS3;
import com.amazonaws.services.s3.AmazonS3;
import org.elasticsearch.cloud.aws.AwsS3Service;
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
@ -35,8 +35,6 @@ import org.elasticsearch.repositories.RepositoryException;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.Matchers;
import java.io.IOException;
import static org.elasticsearch.repositories.s3.S3Repository.Repositories;
import static org.elasticsearch.repositories.s3.S3Repository.Repository;
import static org.elasticsearch.repositories.s3.S3Repository.getValue;
@ -62,19 +60,15 @@ public class S3RepositoryTests extends ESTestCase {
@Override
protected void doClose() {}
@Override
public AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
public AmazonS3 client(Settings repositorySettings, Integer maxRetries,
boolean useThrottleRetries, Boolean pathStyleAccess) {
return new DummyS3Client();
}
}
public void testSettingsResolution() throws Exception {
MockSecureSettings secureSettings1 = new MockSecureSettings();
secureSettings1.setString(Repository.KEY_SETTING.getKey(), "key1");
Settings localSettings = Settings.builder().setSecureSettings(secureSettings1).build();
MockSecureSettings secureSettings2 = new MockSecureSettings();
secureSettings2.setString(Repositories.KEY_SETTING.getKey(), "key2");
Settings globalSettings = Settings.builder().setSecureSettings(secureSettings2).build();
Settings localSettings = Settings.builder().put(Repository.KEY_SETTING.getKey(), "key1").build();
Settings globalSettings = Settings.builder().put(Repositories.KEY_SETTING.getKey(), "key2").build();
assertEquals(new SecureString("key1".toCharArray()),
getValue(localSettings, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
@ -84,6 +78,8 @@ public class S3RepositoryTests extends ESTestCase {
getValue(Settings.EMPTY, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
assertEquals(new SecureString("".toCharArray()),
getValue(Settings.EMPTY, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING));
assertWarnings("[" + Repository.KEY_SETTING.getKey() + "] setting was deprecated",
"[" + Repositories.KEY_SETTING.getKey() + "] setting was deprecated");
}
public void testInvalidChunkBufferSizeSettings() throws IOException {

6
test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
View File

@ -306,12 +306,12 @@ public abstract class ESTestCase extends LuceneTestCase {
}
try {
final List<String> actualWarnings = threadContext.getResponseHeaders().get(DeprecationLogger.WARNING_HEADER);
assertEquals("Expected " + expectedWarnings.length + " warnings but found " + actualWarnings.size() + "\nExpected: "
+ Arrays.asList(expectedWarnings) + "\nActual: " + actualWarnings,
expectedWarnings.length, actualWarnings.size());
for (String msg : expectedWarnings) {
assertThat(actualWarnings, hasItem(containsString(msg)));
}
assertEquals("Expected " + expectedWarnings.length + " warnings but found " + actualWarnings.size() + "\nExpected: "
+ Arrays.asList(expectedWarnings) + "\nActual: " + actualWarnings,
expectedWarnings.length, actualWarnings.size());
} finally {
resetDeprecationLogger();
}