mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-03-31 04:18:39 +00:00
S3 repository: Add named configurations (#22762)
* S3 repository: Add named configurations This change implements named configurations for s3 repository as proposed in #22520. The access/secret key secure settings which were added in #22479 are reverted, and the only secure settings are those with the new named configs. All other previously used settings for the connection are deprecated. closes #22520
This commit is contained in:
Ryan Ernst
GitHub
parent
0f58f3f34b
commit
aad51d44ab
core/src
main/java/org/elasticsearch
bootstrap
common/settings
node
test/java/org/elasticsearch/node/internal
plugins/repository-s3/src
main/java/org/elasticsearch
test/java/org/elasticsearch
test/framework/src/main/java/org/elasticsearch/test
@ -338,12 +338,13 @@ final class Bootstrap {
|
||||
|
||||
INSTANCE.setup(true, environment);
|
||||
|
||||
/* TODO: close this once s3 repository doesn't try to read during repository construction
|
||||
try {
|
||||
// any secure settings must be read during node construction
|
||||
IOUtils.close(keystore);
|
||||
} catch (IOException e) {
|
||||
throw new BootstrapException(e);
|
||||
}
|
||||
}*/
|
||||
|
||||
INSTANCE.start();
|
||||
|
||||
|
||||
@ -109,8 +109,8 @@ public abstract class SecureSetting<T> extends Setting<T> {
|
||||
*
|
||||
* This may be any sensitive string, e.g. a username, a password, an auth token, etc.
|
||||
*/
|
||||
public static SecureSetting<SecureString> secureString(String name, SecureSetting<SecureString> fallback,
|
||||
boolean allowLegacy, Property... properties) {
|
||||
public static Setting<SecureString> secureString(String name, Setting<SecureString> fallback,
|
||||
boolean allowLegacy, Property... properties) {
|
||||
final Setting<String> legacy;
|
||||
if (allowLegacy) {
|
||||
Property[] legacyProperties = ArrayUtils.concat(properties, LEGACY_PROPERTIES, Property.class);
|
||||
|
||||
@ -39,6 +39,19 @@ public final class SecureString implements CharSequence, Closeable {
|
||||
this.chars = Objects.requireNonNull(chars);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a new SecureString from an existing String.
|
||||
*
|
||||
* NOTE: This is not actually secure, since the provided String cannot be deallocated, but
|
||||
* this constructor allows for easy compatibility between new and old apis.
|
||||
*
|
||||
* @deprecated Only use for compatibility between deprecated string settings and new secure strings
|
||||
*/
|
||||
@Deprecated
|
||||
public SecureString(String s) {
|
||||
this(s.toCharArray());
|
||||
}
|
||||
|
||||
/** Constant time equality to avoid potential timing attacks. */
|
||||
@Override
|
||||
public synchronized boolean equals(Object o) {
|
||||
|
||||
@ -123,7 +123,7 @@ public class Setting<T> extends ToXContentToBytes {
|
||||
|
||||
private Setting(Key key, @Nullable Setting<T> fallbackSetting, Function<Settings, String> defaultValue, Function<String, T> parser,
|
||||
Property... properties) {
|
||||
assert this instanceof SecureSetting || parser.apply(defaultValue.apply(Settings.EMPTY)) != null || this.isGroupSetting()
|
||||
assert this instanceof SecureSetting || this.isGroupSetting() || parser.apply(defaultValue.apply(Settings.EMPTY)) != null
|
||||
: "parser returned null";
|
||||
this.key = key;
|
||||
this.fallbackSetting = fallbackSetting;
|
||||
@ -527,6 +527,14 @@ public class Setting<T> extends ToXContentToBytes {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a setting with the given namespace filled in for prefix and suffix.
|
||||
*/
|
||||
public Setting<T> getConcreteSettingForNamespace(String namespace) {
|
||||
String fullKey = key.toConcreteKey(namespace).toString();
|
||||
return getConcreteSetting(fullKey);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void diff(Settings.Builder builder, Settings source, Settings defaultSettings) {
|
||||
matchStream(defaultSettings).forEach((key) -> getConcreteSetting(key).diff(builder, source, defaultSettings));
|
||||
|
||||
@ -897,6 +897,9 @@ public final class Settings implements ToXContent {
|
||||
public Builder put(Settings settings) {
|
||||
removeNonArraysFieldsIfNewSettingsContainsFieldAsArray(settings.getAsMap());
|
||||
map.putAll(settings.getAsMap());
|
||||
if (settings.getSecureSettings() != null) {
|
||||
setSecureSettings(settings.getSecureSettings());
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
@ -89,6 +89,7 @@ public class InternalSettingsPreparer {
|
||||
initializeSettings(output, input, true, properties);
|
||||
Environment environment = new Environment(output.build());
|
||||
|
||||
output = Settings.builder(); // start with a fresh output
|
||||
boolean settingsFileFound = false;
|
||||
Set<String> foundSuffixes = new HashSet<>();
|
||||
for (String allowedSuffix : ALLOWED_SUFFIXES) {
|
||||
|
||||
@ -21,6 +21,10 @@ package org.elasticsearch.node.internal;
|
||||
|
||||
import org.elasticsearch.cli.MockTerminal;
|
||||
import org.elasticsearch.cluster.ClusterName;
|
||||
import org.elasticsearch.common.settings.MockSecureSettings;
|
||||
import org.elasticsearch.common.settings.SecureSetting;
|
||||
import org.elasticsearch.common.settings.SecureString;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.settings.SettingsException;
|
||||
import org.elasticsearch.env.Environment;
|
||||
@ -161,4 +165,13 @@ public class InternalSettingsPreparerTests extends ESTestCase {
|
||||
assertTrue(e.getMessage(), e.getMessage().contains(".properties"));
|
||||
}
|
||||
}
|
||||
|
||||
public void testSecureSettings() {
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString("foo", "secret");
|
||||
Settings input = Settings.builder().put(baseEnvSettings).setSecureSettings(secureSettings).build();
|
||||
Environment env = InternalSettingsPreparer.prepareEnvironment(input, null);
|
||||
Setting<SecureString> fakeSetting = SecureSetting.secureString("foo", null, false);
|
||||
assertEquals("secret", fakeSetting.get(env.settings()).toString());
|
||||
}
|
||||
}
|
||||
|
||||
@ -19,139 +19,146 @@
|
||||
|
||||
package org.elasticsearch.cloud.aws;
|
||||
|
||||
import java.util.Locale;
|
||||
import java.util.function.Function;
|
||||
|
||||
import com.amazonaws.ClientConfiguration;
|
||||
import com.amazonaws.Protocol;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import org.elasticsearch.common.component.LifecycleComponent;
|
||||
import org.elasticsearch.common.settings.SecureSetting;
|
||||
import org.elasticsearch.common.settings.SecureString;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Setting.Property;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
|
||||
import java.util.Locale;
|
||||
import java.util.function.Function;
|
||||
|
||||
public interface AwsS3Service extends LifecycleComponent {
|
||||
|
||||
// Global AWS settings (shared between discovery-ec2 and repository-s3)
|
||||
// Legacy global AWS settings (shared between discovery-ec2 and repository-s3)
|
||||
// Each setting starting with `cloud.aws` also exists in discovery-ec2 project. Don't forget to update
|
||||
// the code there if you change anything here.
|
||||
/**
|
||||
* cloud.aws.access_key: AWS Access key. Shared with discovery-ec2 plugin
|
||||
*/
|
||||
SecureSetting<SecureString> KEY_SETTING = SecureSetting.secureString("cloud.aws.access_key", null, true, Property.Shared);
|
||||
|
||||
Setting<SecureString> KEY_SETTING = new Setting<>("cloud.aws.access_key", "", SecureString::new,
|
||||
Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared);
|
||||
/**
|
||||
* cloud.aws.secret_key: AWS Secret key. Shared with discovery-ec2 plugin
|
||||
*/
|
||||
SecureSetting<SecureString> SECRET_SETTING = SecureSetting.secureString("cloud.aws.secret_key", null, true, Property.Shared);
|
||||
Setting<SecureString> SECRET_SETTING = new Setting<>("cloud.aws.secret_key", "", SecureString::new,
|
||||
Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared);
|
||||
/**
|
||||
* cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with discovery-ec2 plugin
|
||||
*/
|
||||
Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
|
||||
Property.NodeScope, Property.Shared);
|
||||
Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https",
|
||||
s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated, Property.Shared);
|
||||
/**
|
||||
* cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with discovery-ec2 plugin
|
||||
*/
|
||||
Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope, Property.Shared);
|
||||
Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host",
|
||||
Property.NodeScope, Property.Deprecated, Property.Shared);
|
||||
/**
|
||||
* cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with discovery-ec2 plugin
|
||||
*/
|
||||
Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope,
|
||||
Property.Shared);
|
||||
Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16,
|
||||
Property.NodeScope, Property.Deprecated, Property.Shared);
|
||||
/**
|
||||
* cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with discovery-ec2 plugin
|
||||
*/
|
||||
SecureSetting<SecureString> PROXY_USERNAME_SETTING =
|
||||
SecureSetting.secureString("cloud.aws.proxy.username", null, true, Property.Shared);
|
||||
|
||||
Setting<SecureString> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.proxy.username", "", SecureString::new,
|
||||
Property.NodeScope, Property.Deprecated, Property.Shared);
|
||||
/**
|
||||
* cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with discovery-ec2 plugin
|
||||
*/
|
||||
SecureSetting<SecureString> PROXY_PASSWORD_SETTING =
|
||||
SecureSetting.secureString("cloud.aws.proxy.password", null, true, Property.Shared);
|
||||
Setting<SecureString> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.proxy.password", "", SecureString::new,
|
||||
Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared);
|
||||
/**
|
||||
* cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with discovery-ec2 plugin
|
||||
*/
|
||||
Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared);
|
||||
Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer",
|
||||
Property.NodeScope, Property.Deprecated, Property.Shared);
|
||||
/**
|
||||
* cloud.aws.region: Region. Shared with discovery-ec2 plugin
|
||||
*/
|
||||
Setting<String> REGION_SETTING =
|
||||
new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared);
|
||||
Setting<String> REGION_SETTING = new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT),
|
||||
Property.NodeScope, Property.Deprecated, Property.Shared);
|
||||
/**
|
||||
* cloud.aws.read_timeout: Socket read timeout. Shared with discovery-ec2 plugin
|
||||
*/
|
||||
Setting<TimeValue> READ_TIMEOUT = Setting.timeSetting("cloud.aws.read_timeout",
|
||||
TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Shared);
|
||||
TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Deprecated, Property.Shared);
|
||||
|
||||
/**
|
||||
* Defines specific s3 settings starting with cloud.aws.s3.
|
||||
* NOTE: These are legacy settings. Use the named client configs in {@link org.elasticsearch.repositories.s3.S3Repository}.
|
||||
*/
|
||||
interface CLOUD_S3 {
|
||||
/**
|
||||
* cloud.aws.s3.access_key: AWS Access key specific for S3 API calls. Defaults to cloud.aws.access_key.
|
||||
* @see AwsS3Service#KEY_SETTING
|
||||
*/
|
||||
SecureSetting<SecureString> KEY_SETTING = SecureSetting.secureString("cloud.aws.s3.access_key", AwsS3Service.KEY_SETTING, true);
|
||||
Setting<SecureString> KEY_SETTING =
|
||||
new Setting<>("cloud.aws.s3.access_key", AwsS3Service.KEY_SETTING, SecureString::new,
|
||||
Property.NodeScope, Property.Filtered, Property.Deprecated);
|
||||
/**
|
||||
* cloud.aws.s3.secret_key: AWS Secret key specific for S3 API calls. Defaults to cloud.aws.secret_key.
|
||||
* @see AwsS3Service#SECRET_SETTING
|
||||
*/
|
||||
SecureSetting<SecureString> SECRET_SETTING = SecureSetting.secureString("cloud.aws.s3.secret_key",
|
||||
AwsS3Service.SECRET_SETTING, true);
|
||||
Setting<SecureString> SECRET_SETTING =
|
||||
new Setting<>("cloud.aws.s3.secret_key", AwsS3Service.SECRET_SETTING, SecureString::new,
|
||||
Property.NodeScope, Property.Filtered, Property.Deprecated);
|
||||
/**
|
||||
* cloud.aws.s3.protocol: Protocol for AWS API specific for S3 API calls: http or https. Defaults to cloud.aws.protocol.
|
||||
* @see AwsS3Service#PROTOCOL_SETTING
|
||||
*/
|
||||
Setting<Protocol> PROTOCOL_SETTING =
|
||||
new Setting<>("cloud.aws.s3.protocol", AwsS3Service.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
|
||||
Property.NodeScope);
|
||||
Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* cloud.aws.s3.proxy.host: In case of proxy, define its hostname/IP specific for S3 API calls. Defaults to cloud.aws.proxy.host.
|
||||
* @see AwsS3Service#PROXY_HOST_SETTING
|
||||
*/
|
||||
Setting<String> PROXY_HOST_SETTING =
|
||||
new Setting<>("cloud.aws.s3.proxy.host", AwsS3Service.PROXY_HOST_SETTING, Function.identity(),
|
||||
Property.NodeScope);
|
||||
Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* cloud.aws.s3.proxy.port: In case of proxy, define its port specific for S3 API calls. Defaults to cloud.aws.proxy.port.
|
||||
* @see AwsS3Service#PROXY_PORT_SETTING
|
||||
*/
|
||||
Setting<Integer> PROXY_PORT_SETTING =
|
||||
new Setting<>("cloud.aws.s3.proxy.port", AwsS3Service.PROXY_PORT_SETTING,
|
||||
s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.s3.proxy.port"), Property.NodeScope);
|
||||
s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.s3.proxy.port"), Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* cloud.aws.s3.proxy.username: In case of proxy with auth, define the username specific for S3 API calls.
|
||||
* Defaults to cloud.aws.proxy.username.
|
||||
* @see AwsS3Service#PROXY_USERNAME_SETTING
|
||||
*/
|
||||
SecureSetting<SecureString> PROXY_USERNAME_SETTING =
|
||||
SecureSetting.secureString("cloud.aws.s3.proxy.username", AwsS3Service.PROXY_USERNAME_SETTING, true);
|
||||
Setting<SecureString> PROXY_USERNAME_SETTING =
|
||||
new Setting<>("cloud.aws.s3.proxy.username", AwsS3Service.PROXY_USERNAME_SETTING, SecureString::new,
|
||||
Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* cloud.aws.s3.proxy.password: In case of proxy with auth, define the password specific for S3 API calls.
|
||||
* Defaults to cloud.aws.proxy.password.
|
||||
* @see AwsS3Service#PROXY_PASSWORD_SETTING
|
||||
*/
|
||||
SecureSetting<SecureString> PROXY_PASSWORD_SETTING =
|
||||
SecureSetting.secureString("cloud.aws.s3.proxy.password", AwsS3Service.PROXY_PASSWORD_SETTING, true);
|
||||
|
||||
Setting<SecureString> PROXY_PASSWORD_SETTING =
|
||||
new Setting<>("cloud.aws.s3.proxy.password", AwsS3Service.PROXY_PASSWORD_SETTING, SecureString::new,
|
||||
Property.NodeScope, Property.Filtered, Property.Deprecated);
|
||||
/**
|
||||
* cloud.aws.s3.signer: If you are using an old AWS API version, you can define a Signer. Specific for S3 API calls.
|
||||
* Defaults to cloud.aws.signer.
|
||||
* @see AwsS3Service#SIGNER_SETTING
|
||||
*/
|
||||
Setting<String> SIGNER_SETTING =
|
||||
new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(), Property.NodeScope);
|
||||
new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(),
|
||||
Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* cloud.aws.s3.region: Region specific for S3 API calls. Defaults to cloud.aws.region.
|
||||
* @see AwsS3Service#REGION_SETTING
|
||||
*/
|
||||
Setting<String> REGION_SETTING =
|
||||
new Setting<>("cloud.aws.s3.region", AwsS3Service.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT),
|
||||
Property.NodeScope);
|
||||
Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* cloud.aws.s3.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
|
||||
*/
|
||||
@ -161,9 +168,8 @@ public interface AwsS3Service extends LifecycleComponent {
|
||||
* @see AwsS3Service#READ_TIMEOUT
|
||||
*/
|
||||
Setting<TimeValue> READ_TIMEOUT =
|
||||
Setting.timeSetting("cloud.aws.s3.read_timeout", AwsS3Service.READ_TIMEOUT, Property.NodeScope);
|
||||
Setting.timeSetting("cloud.aws.s3.read_timeout", AwsS3Service.READ_TIMEOUT, Property.NodeScope, Property.Deprecated);
|
||||
}
|
||||
|
||||
AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
|
||||
boolean useThrottleRetries, Boolean pathStyleAccess);
|
||||
AmazonS3 client(Settings repositorySettings, Integer maxRetries, boolean useThrottleRetries, Boolean pathStyleAccess);
|
||||
}
|
||||
|
||||
@ -19,42 +19,40 @@
|
||||
|
||||
package org.elasticsearch.cloud.aws;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
|
||||
import com.amazonaws.ClientConfiguration;
|
||||
import com.amazonaws.Protocol;
|
||||
import com.amazonaws.auth.AWSCredentials;
|
||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
|
||||
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
|
||||
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
|
||||
import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
|
||||
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
|
||||
import com.amazonaws.http.IdleConnectionReaper;
|
||||
import com.amazonaws.internal.StaticCredentialsProvider;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3Client;
|
||||
import com.amazonaws.services.s3.S3ClientOptions;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.elasticsearch.ElasticsearchException;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.collect.Tuple;
|
||||
import org.elasticsearch.common.component.AbstractLifecycleComponent;
|
||||
import org.elasticsearch.common.logging.DeprecationLogger;
|
||||
import org.elasticsearch.common.settings.SecureString;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.repositories.s3.S3Repository;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static org.elasticsearch.repositories.s3.S3Repository.getValue;
|
||||
|
||||
public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service {
|
||||
|
||||
// pkg private for tests
|
||||
static final Setting<String> CLIENT_NAME = new Setting<>("client", "default", Function.identity());
|
||||
|
||||
/**
|
||||
* (acceskey, endpoint) -> client
|
||||
*/
|
||||
@ -65,11 +63,12 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
|
||||
boolean useThrottleRetries, Boolean pathStyleAccess) {
|
||||
String foundEndpoint = findEndpoint(logger, settings, endpoint, region);
|
||||
public synchronized AmazonS3 client(Settings repositorySettings, Integer maxRetries,
|
||||
boolean useThrottleRetries, Boolean pathStyleAccess) {
|
||||
String clientName = CLIENT_NAME.get(repositorySettings);
|
||||
String foundEndpoint = findEndpoint(logger, repositorySettings, settings, clientName);
|
||||
|
||||
AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, settings, repositorySettings);
|
||||
AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, settings, repositorySettings, clientName);
|
||||
|
||||
Tuple<String, String> clientDescriptor = new Tuple<>(foundEndpoint, credentials.getCredentials().getAWSAccessKeyId());
|
||||
AmazonS3Client client = clients.get(clientDescriptor);
|
||||
@ -79,7 +78,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
|
||||
|
||||
client = new AmazonS3Client(
|
||||
credentials,
|
||||
buildConfiguration(logger, settings, protocol, maxRetries, foundEndpoint, useThrottleRetries));
|
||||
buildConfiguration(logger, repositorySettings, settings, clientName, maxRetries, foundEndpoint, useThrottleRetries));
|
||||
|
||||
if (pathStyleAccess != null) {
|
||||
client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess));
|
||||
@ -93,19 +92,27 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
|
||||
return client;
|
||||
}
|
||||
|
||||
public static ClientConfiguration buildConfiguration(Logger logger, Settings settings, Protocol protocol, Integer maxRetries,
|
||||
String endpoint, boolean useThrottleRetries) {
|
||||
// pkg private for tests
|
||||
static ClientConfiguration buildConfiguration(Logger logger, Settings repositorySettings, Settings settings,
|
||||
String clientName, Integer maxRetries, String endpoint,
|
||||
boolean useThrottleRetries) {
|
||||
ClientConfiguration clientConfiguration = new ClientConfiguration();
|
||||
// the response metadata cache is only there for diagnostics purposes,
|
||||
// but can force objects from every response to the old generation.
|
||||
clientConfiguration.setResponseMetadataCacheSize(0);
|
||||
Protocol protocol = getConfigValue(repositorySettings, settings, clientName, S3Repository.PROTOCOL_SETTING,
|
||||
S3Repository.Repository.PROTOCOL_SETTING, S3Repository.Repositories.PROTOCOL_SETTING);
|
||||
clientConfiguration.setProtocol(protocol);
|
||||
|
||||
String proxyHost = CLOUD_S3.PROXY_HOST_SETTING.get(settings);
|
||||
String proxyHost = getConfigValue(null, settings, clientName,
|
||||
S3Repository.PROXY_HOST_SETTING, null, CLOUD_S3.PROXY_HOST_SETTING);
|
||||
if (Strings.hasText(proxyHost)) {
|
||||
Integer proxyPort = CLOUD_S3.PROXY_PORT_SETTING.get(settings);
|
||||
try (SecureString proxyUsername = CLOUD_S3.PROXY_USERNAME_SETTING.get(settings);
|
||||
SecureString proxyPassword = CLOUD_S3.PROXY_PASSWORD_SETTING.get(settings)) {
|
||||
Integer proxyPort = getConfigValue(null, settings, clientName,
|
||||
S3Repository.PROXY_PORT_SETTING, null, CLOUD_S3.PROXY_PORT_SETTING);
|
||||
try (SecureString proxyUsername = getConfigValue(null, settings, clientName,
|
||||
S3Repository.PROXY_USERNAME_SETTING, null, CLOUD_S3.PROXY_USERNAME_SETTING);
|
||||
SecureString proxyPassword = getConfigValue(null, settings, clientName,
|
||||
S3Repository.PROXY_PASSWORD_SETTING, null, CLOUD_S3.PROXY_PASSWORD_SETTING)) {
|
||||
|
||||
clientConfiguration
|
||||
.withProxyHost(proxyHost)
|
||||
@ -128,18 +135,20 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
|
||||
AwsSigner.configureSigner(awsSigner, clientConfiguration, endpoint);
|
||||
}
|
||||
|
||||
clientConfiguration.setSocketTimeout((int) CLOUD_S3.READ_TIMEOUT.get(settings).millis());
|
||||
TimeValue readTimeout = getConfigValue(null, settings, clientName,
|
||||
S3Repository.READ_TIMEOUT_SETTING, null, CLOUD_S3.READ_TIMEOUT);
|
||||
clientConfiguration.setSocketTimeout((int)readTimeout.millis());
|
||||
|
||||
return clientConfiguration;
|
||||
}
|
||||
|
||||
public static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger deprecationLogger,
|
||||
Settings settings, Settings repositorySettings) {
|
||||
Settings settings, Settings repositorySettings, String clientName) {
|
||||
AWSCredentialsProvider credentials;
|
||||
try (SecureString key = getValue(repositorySettings, settings,
|
||||
S3Repository.Repository.KEY_SETTING, S3Repository.Repositories.KEY_SETTING);
|
||||
SecureString secret = getValue(repositorySettings, settings,
|
||||
S3Repository.Repository.SECRET_SETTING, S3Repository.Repositories.SECRET_SETTING)) {
|
||||
try (SecureString key = getConfigValue(repositorySettings, settings, clientName, S3Repository.ACCESS_KEY_SETTING,
|
||||
S3Repository.Repository.KEY_SETTING, S3Repository.Repositories.KEY_SETTING);
|
||||
SecureString secret = getConfigValue(repositorySettings, settings, clientName, S3Repository.SECRET_KEY_SETTING,
|
||||
S3Repository.Repository.SECRET_SETTING, S3Repository.Repositories.SECRET_SETTING)) {
|
||||
|
||||
if (key.length() == 0 && secret.length() == 0) {
|
||||
// create a "manual" chain of providers here, so we can log deprecation of unsupported methods
|
||||
@ -183,7 +192,12 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
|
||||
return null;
|
||||
}
|
||||
|
||||
protected static String findEndpoint(Logger logger, Settings settings, String endpoint, String region) {
|
||||
// pkg private for tests
|
||||
/** Returns the endpoint the client should use, based on the available endpoint settings found. */
|
||||
static String findEndpoint(Logger logger, Settings repositorySettings, Settings settings, String clientName) {
|
||||
String region = getRegion(repositorySettings, settings);
|
||||
String endpoint = getConfigValue(repositorySettings, settings, clientName, S3Repository.ENDPOINT_SETTING,
|
||||
S3Repository.Repository.ENDPOINT_SETTING, S3Repository.Repositories.ENDPOINT_SETTING);
|
||||
if (Strings.isNullOrEmpty(endpoint)) {
|
||||
logger.debug("no repository level endpoint has been defined. Trying to guess from repository region [{}]", region);
|
||||
if (!region.isEmpty()) {
|
||||
@ -207,6 +221,14 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
|
||||
return endpoint;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the region configured, or empty string.
|
||||
* TODO: remove after https://github.com/elastic/elasticsearch/issues/22761 */
|
||||
public static String getRegion(Settings repositorySettings, Settings settings) {
|
||||
return getConfigValue(repositorySettings, settings, CLIENT_NAME.get(repositorySettings), S3Repository.REGION_SETTING,
|
||||
S3Repository.Repository.REGION_SETTING, S3Repository.Repositories.REGION_SETTING);
|
||||
}
|
||||
|
||||
private static String getEndpoint(String region) {
|
||||
final String endpoint;
|
||||
switch (region) {
|
||||
@ -276,6 +298,23 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
|
||||
return endpoint;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the setting value, trying first with named configs,
|
||||
* then falling back to repository and global repositories settings.
|
||||
*/
|
||||
private static <T> T getConfigValue(Settings repositorySettings, Settings globalSettings, String clientName,
|
||||
Setting.AffixSetting<T> configSetting, Setting<T> repositorySetting, Setting<T> globalSetting) {
|
||||
Setting<T> concreteSetting = configSetting.getConcreteSettingForNamespace(clientName);
|
||||
if (concreteSetting.exists(globalSettings)) {
|
||||
return concreteSetting.get(globalSettings);
|
||||
} else if (repositorySetting == null) {
|
||||
// no repository setting, just use global setting
|
||||
return globalSetting.get(globalSettings);
|
||||
} else {
|
||||
return S3Repository.getValue(repositorySettings, globalSettings, repositorySetting, globalSetting);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doStart() throws ElasticsearchException {
|
||||
}
|
||||
|
||||
@ -21,8 +21,10 @@ package org.elasticsearch.repositories.s3;
|
||||
|
||||
import com.amazonaws.ClientConfiguration;
|
||||
import com.amazonaws.Protocol;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import org.elasticsearch.cloud.aws.AwsS3Service;
|
||||
import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3;
|
||||
import org.elasticsearch.cloud.aws.InternalAwsS3Service;
|
||||
import org.elasticsearch.cloud.aws.blobstore.S3BlobStore;
|
||||
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
|
||||
import org.elasticsearch.common.Strings;
|
||||
@ -31,10 +33,12 @@ import org.elasticsearch.common.blobstore.BlobStore;
|
||||
import org.elasticsearch.common.settings.SecureSetting;
|
||||
import org.elasticsearch.common.settings.SecureString;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Setting.AffixSetting;
|
||||
import org.elasticsearch.common.settings.Setting.Property;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.ByteSizeUnit;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
|
||||
import org.elasticsearch.monitor.jvm.JvmInfo;
|
||||
import org.elasticsearch.repositories.RepositoryException;
|
||||
@ -42,7 +46,6 @@ import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Locale;
|
||||
import java.util.function.Function;
|
||||
|
||||
/**
|
||||
* Shared file system implementation of the BlobStoreRepository
|
||||
@ -61,40 +64,86 @@ public class S3Repository extends BlobStoreRepository {
|
||||
|
||||
public static final String TYPE = "s3";
|
||||
|
||||
// prefix for s3 client settings
|
||||
private static final String PREFIX = "s3.client.";
|
||||
|
||||
/** The access key (ie login id) for connecting to s3. */
|
||||
public static final AffixSetting<SecureString> ACCESS_KEY_SETTING = Setting.affixKeySetting(PREFIX, "access_key",
|
||||
key -> SecureSetting.secureString(key, Repositories.KEY_SETTING, false));
|
||||
|
||||
/** The secret key (ie password) for connecting to s3. */
|
||||
public static final AffixSetting<SecureString> SECRET_KEY_SETTING = Setting.affixKeySetting(PREFIX, "secret_key",
|
||||
key -> SecureSetting.secureString(key, Repositories.SECRET_SETTING, false));
|
||||
|
||||
/** The region the s3 repository bucket should exist in. */
|
||||
public static final AffixSetting<String> REGION_SETTING = Setting.affixKeySetting(PREFIX, "region",
|
||||
key -> new Setting<>(key, "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope));
|
||||
|
||||
/** An override for the s3 endpoint to connect to. */
|
||||
public static final AffixSetting<String> ENDPOINT_SETTING = Setting.affixKeySetting(PREFIX, "endpoint",
|
||||
key -> new Setting<>(key, Repositories.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope));
|
||||
|
||||
/** The protocol to use to connec to to s3. */
|
||||
public static final AffixSetting<Protocol> PROTOCOL_SETTING = Setting.affixKeySetting(PREFIX, "protocol",
|
||||
key -> new Setting<>(key, "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope));
|
||||
|
||||
/** The host name of a proxy to connect to s3 through. */
|
||||
public static final AffixSetting<String> PROXY_HOST_SETTING = Setting.affixKeySetting(PREFIX, "proxy.host",
|
||||
key -> Setting.simpleString(key, Property.NodeScope));
|
||||
|
||||
/** The port of a proxy to connect to s3 through. */
|
||||
public static final AffixSetting<Integer> PROXY_PORT_SETTING = Setting.affixKeySetting(PREFIX, "proxy.port",
|
||||
key -> Setting.intSetting(key, 80, 0, 1<<16, Property.NodeScope));
|
||||
|
||||
/** The username of a proxy to connect to s3 through. */
|
||||
public static final AffixSetting<SecureString> PROXY_USERNAME_SETTING = Setting.affixKeySetting(PREFIX, "proxy.username",
|
||||
key -> SecureSetting.secureString(key, AwsS3Service.PROXY_USERNAME_SETTING, false));
|
||||
|
||||
/** The password of a proxy to connect to s3 through. */
|
||||
public static final AffixSetting<SecureString> PROXY_PASSWORD_SETTING = Setting.affixKeySetting(PREFIX, "proxy.password",
|
||||
key -> SecureSetting.secureString(key, AwsS3Service.PROXY_PASSWORD_SETTING, false));
|
||||
|
||||
/** The socket timeout for connecting to s3. */
|
||||
public static final AffixSetting<TimeValue> READ_TIMEOUT_SETTING = Setting.affixKeySetting(PREFIX, "read_timeout",
|
||||
key -> Setting.timeSetting(key, TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope));
|
||||
|
||||
/**
|
||||
* Global S3 repositories settings. Starting with: repositories.s3
|
||||
* NOTE: These are legacy settings. Use the named client config settings above.
|
||||
*/
|
||||
public interface Repositories {
|
||||
/**
|
||||
* repositories.s3.access_key: AWS Access key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.access_key.
|
||||
* @see CLOUD_S3#KEY_SETTING
|
||||
*/
|
||||
SecureSetting<SecureString> KEY_SETTING = SecureSetting.secureString("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, true);
|
||||
Setting<SecureString> KEY_SETTING = new Setting<>("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, SecureString::new,
|
||||
Property.NodeScope, Property.Filtered, Property.Deprecated);
|
||||
|
||||
/**
|
||||
* repositories.s3.secret_key: AWS Secret key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.secret_key.
|
||||
* @see CLOUD_S3#SECRET_SETTING
|
||||
*/
|
||||
SecureSetting<SecureString> SECRET_SETTING = SecureSetting.secureString("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, true);
|
||||
Setting<SecureString> SECRET_SETTING = new Setting<>("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, SecureString::new,
|
||||
Property.NodeScope, Property.Filtered, Property.Deprecated);
|
||||
|
||||
/**
|
||||
* repositories.s3.region: Region specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.region.
|
||||
* @see CLOUD_S3#REGION_SETTING
|
||||
*/
|
||||
Setting<String> REGION_SETTING =
|
||||
new Setting<>("repositories.s3.region", CLOUD_S3.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
|
||||
Setting<String> REGION_SETTING = new Setting<>("repositories.s3.region", CLOUD_S3.REGION_SETTING,
|
||||
s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* repositories.s3.endpoint: Endpoint specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.endpoint.
|
||||
* @see CLOUD_S3#ENDPOINT_SETTING
|
||||
*/
|
||||
Setting<String> ENDPOINT_SETTING =
|
||||
new Setting<>("repositories.s3.endpoint", CLOUD_S3.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
|
||||
Setting<String> ENDPOINT_SETTING = new Setting<>("repositories.s3.endpoint", CLOUD_S3.ENDPOINT_SETTING,
|
||||
s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* repositories.s3.protocol: Protocol specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.protocol.
|
||||
* @see CLOUD_S3#PROTOCOL_SETTING
|
||||
*/
|
||||
Setting<Protocol> PROTOCOL_SETTING =
|
||||
new Setting<>("repositories.s3.protocol", CLOUD_S3.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope);
|
||||
Setting<Protocol> PROTOCOL_SETTING = new Setting<>("repositories.s3.protocol", CLOUD_S3.PROTOCOL_SETTING,
|
||||
s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* repositories.s3.bucket: The name of the bucket to be used for snapshots.
|
||||
*/
|
||||
@ -177,37 +226,30 @@ public class S3Repository extends BlobStoreRepository {
|
||||
* If undefined, they use the repositories.s3.xxx equivalent setting.
|
||||
*/
|
||||
public interface Repository {
|
||||
/**
|
||||
* access_key
|
||||
* @see Repositories#KEY_SETTING
|
||||
*/
|
||||
SecureSetting<SecureString> KEY_SETTING = SecureSetting.secureString("access_key", null, true);
|
||||
Setting<SecureString> KEY_SETTING = new Setting<>("access_key", "", SecureString::new,
|
||||
Property.Filtered, Property.Deprecated);
|
||||
|
||||
|
||||
Setting<SecureString> SECRET_SETTING = new Setting<>("secret_key", "", SecureString::new,
|
||||
Property.Filtered, Property.Deprecated);
|
||||
|
||||
/**
|
||||
* secret_key
|
||||
* @see Repositories#SECRET_SETTING
|
||||
*/
|
||||
SecureSetting<SecureString> SECRET_SETTING = SecureSetting.secureString("secret_key", null, true);
|
||||
/**
|
||||
* bucket
|
||||
* @see Repositories#BUCKET_SETTING
|
||||
*/
|
||||
Setting<String> BUCKET_SETTING = Setting.simpleString("bucket");
|
||||
/**
|
||||
* endpoint
|
||||
* @see Repositories#ENDPOINT_SETTING
|
||||
*/
|
||||
Setting<String> ENDPOINT_SETTING = Setting.simpleString("endpoint");
|
||||
Setting<String> ENDPOINT_SETTING = Setting.simpleString("endpoint", Property.Deprecated);
|
||||
/**
|
||||
* protocol
|
||||
* @see Repositories#PROTOCOL_SETTING
|
||||
*/
|
||||
Setting<Protocol> PROTOCOL_SETTING = new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)));
|
||||
Setting<Protocol> PROTOCOL_SETTING = new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
|
||||
Property.Deprecated);
|
||||
/**
|
||||
* region
|
||||
* @see Repositories#REGION_SETTING
|
||||
*/
|
||||
Setting<String> REGION_SETTING = new Setting<>("region", "", s -> s.toLowerCase(Locale.ROOT));
|
||||
Setting<String> REGION_SETTING = new Setting<>("region", "", s -> s.toLowerCase(Locale.ROOT), Property.Deprecated);
|
||||
/**
|
||||
* server_side_encryption
|
||||
* @see Repositories#SERVER_SIDE_ENCRYPTION_SETTING
|
||||
@ -286,10 +328,6 @@ public class S3Repository extends BlobStoreRepository {
|
||||
throw new RepositoryException(metadata.name(), "No bucket defined for s3 gateway");
|
||||
}
|
||||
|
||||
String endpoint = getValue(metadata.settings(), settings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING);
|
||||
Protocol protocol = getValue(metadata.settings(), settings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING);
|
||||
String region = getValue(metadata.settings(), settings, Repository.REGION_SETTING, Repositories.REGION_SETTING);
|
||||
|
||||
boolean serverSideEncryption = getValue(metadata.settings(), settings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
|
||||
ByteSizeValue bufferSize = getValue(metadata.settings(), settings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING);
|
||||
Integer maxRetries = getValue(metadata.settings(), settings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING);
|
||||
@ -315,13 +353,14 @@ public class S3Repository extends BlobStoreRepository {
|
||||
pathStyleAccess = getValue(metadata.settings(), settings, Repository.PATH_STYLE_ACCESS_SETTING, Repositories.PATH_STYLE_ACCESS_SETTING);
|
||||
}
|
||||
|
||||
logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " +
|
||||
logger.debug("using bucket [{}], chunk_size [{}], server_side_encryption [{}], " +
|
||||
"buffer_size [{}], max_retries [{}], use_throttle_retries [{}], cannedACL [{}], storageClass [{}], path_style_access [{}]",
|
||||
bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
|
||||
bucket, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
|
||||
storageClass, pathStyleAccess);
|
||||
|
||||
blobStore = new S3BlobStore(settings,
|
||||
s3Service.client(metadata.settings(), endpoint, protocol, region, maxRetries, useThrottleRetries, pathStyleAccess),
|
||||
AmazonS3 client = s3Service.client(metadata.settings(), maxRetries, useThrottleRetries, pathStyleAccess);
|
||||
String region = InternalAwsS3Service.getRegion(metadata.settings(), settings);
|
||||
blobStore = new S3BlobStore(settings, client,
|
||||
bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
|
||||
|
||||
String basePath = getValue(metadata.settings(), settings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);
|
||||
|
||||
@ -23,7 +23,6 @@ import com.amazonaws.ClientConfiguration;
|
||||
import com.amazonaws.Protocol;
|
||||
import com.amazonaws.auth.AWSCredentials;
|
||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
|
||||
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
|
||||
import org.elasticsearch.common.settings.MockSecureSettings;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
@ -38,104 +37,30 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
|
||||
public void testAWSCredentialsWithSystemProviders() {
|
||||
AWSCredentialsProvider credentialsProvider =
|
||||
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY);
|
||||
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY, "default");
|
||||
assertThat(credentialsProvider, instanceOf(InstanceProfileCredentialsProvider.class));
|
||||
}
|
||||
|
||||
public void testAWSCredentialsWithElasticsearchAwsSettings() {
|
||||
public void testAwsCredsDefaultSettings() {
|
||||
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key");
|
||||
secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret");
|
||||
secureSettings.setString("s3.client.default.access_key", "aws_key");
|
||||
secureSettings.setString("s3.client.default.secret_key", "aws_secret");
|
||||
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret");
|
||||
}
|
||||
|
||||
public void testAWSCredentialsWithElasticsearchS3Settings() {
|
||||
public void testAwsCredsExplicitConfigSettings() {
|
||||
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
|
||||
repositorySettings = Settings.builder().put(repositorySettings)
|
||||
.put(InternalAwsS3Service.CLIENT_NAME.getKey(), "myconfig").build();
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key");
|
||||
secureSettings.setString(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret");
|
||||
secureSettings.setString("s3.client.myconfig.access_key", "aws_key");
|
||||
secureSettings.setString("s3.client.myconfig.secret_key", "aws_secret");
|
||||
secureSettings.setString("s3.client.default.access_key", "wrong_key");
|
||||
secureSettings.setString("s3.client.default.secret_key", "wrong_secret");
|
||||
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
|
||||
}
|
||||
|
||||
public void testAWSCredentialsWithElasticsearchAwsAndS3Settings() {
|
||||
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key");
|
||||
secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret");
|
||||
secureSettings.setString(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key");
|
||||
secureSettings.setString(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret");
|
||||
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
|
||||
}
|
||||
|
||||
public void testAWSCredentialsWithElasticsearchRepositoriesSettings() {
|
||||
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key");
|
||||
secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret");
|
||||
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
|
||||
}
|
||||
|
||||
|
||||
public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettings() {
|
||||
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key");
|
||||
secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret");
|
||||
secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key");
|
||||
secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret");
|
||||
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
|
||||
}
|
||||
|
||||
public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettings() {
|
||||
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key");
|
||||
secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret");
|
||||
secureSettings.setString(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key");
|
||||
secureSettings.setString(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret");
|
||||
secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key");
|
||||
secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret");
|
||||
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
|
||||
}
|
||||
|
||||
public void testAWSCredentialsWithElasticsearchRepositoriesSettingsAndRepositorySettings() {
|
||||
Settings repositorySettings = generateSecureRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key");
|
||||
secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret");
|
||||
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
|
||||
}
|
||||
|
||||
public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettingsAndRepositorySettings() {
|
||||
Settings repositorySettings = generateSecureRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key");
|
||||
secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret");
|
||||
secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key");
|
||||
secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret");
|
||||
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
|
||||
}
|
||||
|
||||
public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettingsAndRepositorySettings() {
|
||||
Settings repositorySettings = generateSecureRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString(AwsS3Service.KEY_SETTING.getKey(), "aws_key");
|
||||
secureSettings.setString(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret");
|
||||
secureSettings.setString(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key");
|
||||
secureSettings.setString(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret");
|
||||
secureSettings.setString(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key");
|
||||
secureSettings.setString(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret");
|
||||
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret");
|
||||
}
|
||||
|
||||
public void testAWSCredentialsWithElasticsearchAwsSettingsBackcompat() {
|
||||
@ -169,7 +94,9 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
.put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
|
||||
.build();
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
|
||||
assertWarnings("[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated",
|
||||
assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey() + "] setting was deprecated");
|
||||
}
|
||||
|
||||
@ -193,7 +120,9 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
.put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
|
||||
.build();
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
|
||||
assertWarnings("[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
|
||||
assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + S3Repository.Repositories.SECRET_SETTING.getKey() + "] setting was deprecated");
|
||||
}
|
||||
|
||||
@ -208,7 +137,11 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
.put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
|
||||
.build();
|
||||
launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
|
||||
assertWarnings("[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
|
||||
assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + S3Repository.Repositories.SECRET_SETTING.getKey() + "] setting was deprecated");
|
||||
}
|
||||
|
||||
@ -253,8 +186,9 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
|
||||
protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings singleRepositorySettings, Settings settings,
|
||||
String expectedKey, String expectedSecret) {
|
||||
AWSCredentials credentials =
|
||||
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, settings, singleRepositorySettings).getCredentials();
|
||||
String configName = InternalAwsS3Service.CLIENT_NAME.get(singleRepositorySettings);
|
||||
AWSCredentials credentials = InternalAwsS3Service.buildCredentials(logger, deprecationLogger, settings,
|
||||
singleRepositorySettings, configName).getCredentials();
|
||||
assertThat(credentials.getAWSAccessKeyId(), is(expectedKey));
|
||||
assertThat(credentials.getAWSSecretKey(), is(expectedSecret));
|
||||
}
|
||||
@ -268,41 +202,17 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
public void testAWSConfigurationWithAwsSettings() {
|
||||
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username");
|
||||
secureSettings.setString(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password");
|
||||
secureSettings.setString("s3.client.default.proxy.username", "aws_proxy_username");
|
||||
secureSettings.setString("s3.client.default.proxy.password", "aws_proxy_password");
|
||||
Settings settings = Settings.builder()
|
||||
.setSecureSettings(secureSettings)
|
||||
.put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http")
|
||||
.put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
|
||||
.put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080)
|
||||
.put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
|
||||
.put(AwsS3Service.READ_TIMEOUT.getKey(), "10s")
|
||||
.put("s3.client.default.protocol", "http")
|
||||
.put("s3.client.default.proxy.host", "aws_proxy_host")
|
||||
.put("s3.client.default.proxy.port", 8080)
|
||||
.put("s3.client.default.read_timeout", "10s")
|
||||
.build();
|
||||
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
|
||||
"aws_proxy_password", "AWS3SignerType", 3, false, 10000);
|
||||
}
|
||||
|
||||
public void testAWSConfigurationWithAwsAndS3Settings() {
|
||||
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username");
|
||||
secureSettings.setString(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password");
|
||||
secureSettings.setString(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey(), "s3_proxy_username");
|
||||
secureSettings.setString(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey(), "s3_proxy_password");
|
||||
Settings settings = Settings.builder()
|
||||
.setSecureSettings(secureSettings)
|
||||
.put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http")
|
||||
.put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
|
||||
.put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080)
|
||||
.put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
|
||||
.put(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey(), "https")
|
||||
.put(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey(), "s3_proxy_host")
|
||||
.put(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey(), 8081)
|
||||
.put(AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey(), "NoOpSignerType")
|
||||
.put(AwsS3Service.CLOUD_S3.READ_TIMEOUT.getKey(), "10s")
|
||||
.build();
|
||||
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username",
|
||||
"s3_proxy_password", "NoOpSignerType", 3, false, 10000);
|
||||
"aws_proxy_password", null, 3, false, 10000);
|
||||
}
|
||||
|
||||
public void testAWSConfigurationWithAwsSettingsBackcompat() {
|
||||
@ -319,7 +229,12 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
|
||||
"aws_proxy_password", "AWS3SignerType", 3, false, 10000);
|
||||
assertWarnings("[" + AwsS3Service.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated");
|
||||
"[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.PROTOCOL_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.PROXY_HOST_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.PROXY_PORT_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.SIGNER_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.READ_TIMEOUT.getKey() + "] setting was deprecated");
|
||||
}
|
||||
|
||||
public void testAWSConfigurationWithAwsAndS3SettingsBackcompat() {
|
||||
@ -331,6 +246,7 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
.put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
|
||||
.put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
|
||||
.put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
|
||||
.put(AwsS3Service.READ_TIMEOUT.getKey(), "5s")
|
||||
.put(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey(), "https")
|
||||
.put(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey(), "s3_proxy_host")
|
||||
.put(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey(), 8081)
|
||||
@ -341,8 +257,38 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
.build();
|
||||
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username",
|
||||
"s3_proxy_password", "NoOpSignerType", 3, false, 10000);
|
||||
assertWarnings("[" + AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated");
|
||||
assertWarnings("[" + AwsS3Service.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.PROTOCOL_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.PROXY_HOST_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.PROXY_PORT_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.SIGNER_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.READ_TIMEOUT.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + AwsS3Service.CLOUD_S3.READ_TIMEOUT.getKey() + "] setting was deprecated");
|
||||
}
|
||||
|
||||
public void testGlobalMaxRetries() {
|
||||
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
|
||||
Settings settings = Settings.builder()
|
||||
.put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10)
|
||||
.build();
|
||||
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null,
|
||||
null, null, 10, false, 50000);
|
||||
}
|
||||
|
||||
public void testRepositoryMaxRetries() {
|
||||
Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, 20);
|
||||
Settings settings = Settings.builder()
|
||||
.put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10)
|
||||
.build();
|
||||
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null,
|
||||
null, null, 20, false, 50000);
|
||||
}
|
||||
|
||||
protected void launchAWSConfigurationTest(Settings settings,
|
||||
@ -356,15 +302,13 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
Integer expectedMaxRetries,
|
||||
boolean expectedUseThrottleRetries,
|
||||
int expectedReadTimeout) {
|
||||
Protocol protocol = S3Repository.getValue(singleRepositorySettings, settings,
|
||||
S3Repository.Repository.PROTOCOL_SETTING, S3Repository.Repositories.PROTOCOL_SETTING);
|
||||
Integer maxRetries = S3Repository.getValue(singleRepositorySettings, settings,
|
||||
S3Repository.Repository.MAX_RETRIES_SETTING, S3Repository.Repositories.MAX_RETRIES_SETTING);
|
||||
Boolean useThrottleRetries = S3Repository.getValue(singleRepositorySettings, settings,
|
||||
S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING, S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING);
|
||||
|
||||
ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(logger, settings, protocol, maxRetries, null,
|
||||
useThrottleRetries);
|
||||
ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(logger, singleRepositorySettings, settings,
|
||||
"default", maxRetries, null, useThrottleRetries);
|
||||
|
||||
assertThat(configuration.getResponseMetadataCacheSize(), is(0));
|
||||
assertThat(configuration.getProtocol(), is(expectedProtocol));
|
||||
@ -398,84 +342,83 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private static Settings generateSecureRepositorySettings(String key, String secret, String region, String endpoint,
|
||||
Integer maxRetries) {
|
||||
Settings settings = generateRepositorySettings(null, null, region, endpoint, maxRetries);
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString(S3Repository.Repository.KEY_SETTING.getKey(), key);
|
||||
secureSettings.setString(S3Repository.Repository.SECRET_SETTING.getKey(), secret);
|
||||
return Settings.builder().put(settings).setSecureSettings(secureSettings).build();
|
||||
}
|
||||
|
||||
public void testDefaultEndpoint() {
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), Settings.EMPTY, "");
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), Settings.EMPTY,
|
||||
"s3.eu-central-1.amazonaws.com");
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
|
||||
Settings.EMPTY, "repository.endpoint");
|
||||
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), Settings.EMPTY, "");
|
||||
}
|
||||
|
||||
public void testSpecificEndpoint() {
|
||||
public void testEndpointSetting() {
|
||||
Settings settings = Settings.builder()
|
||||
.put(InternalAwsS3Service.CLOUD_S3.ENDPOINT_SETTING.getKey(), "ec2.endpoint")
|
||||
.put("s3.client.default.endpoint", "s3.endpoint")
|
||||
.build();
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
|
||||
"ec2.endpoint");
|
||||
// Endpoint has precedence on region. Whatever region we set, we won't use it
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
|
||||
"ec2.endpoint");
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
|
||||
settings, "repository.endpoint");
|
||||
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, "s3.endpoint");
|
||||
}
|
||||
|
||||
public void testRegionWithAwsSettings() {
|
||||
public void testEndpointSettingBackcompat() {
|
||||
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
|
||||
Settings.EMPTY, "repository.endpoint");
|
||||
assertWarnings("[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
|
||||
Settings settings = Settings.builder()
|
||||
.put(S3Repository.Repositories.ENDPOINT_SETTING.getKey(), "repositories.endpoint")
|
||||
.build();
|
||||
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
|
||||
"repositories.endpoint");
|
||||
assertWarnings("[" + S3Repository.Repositories.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
|
||||
}
|
||||
|
||||
public void testRegionSetting() {
|
||||
Settings settings = Settings.builder()
|
||||
.put("s3.client.default.region", randomFrom("eu-west", "eu-west-1"))
|
||||
.build();
|
||||
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
|
||||
"s3-eu-west-1.amazonaws.com");
|
||||
}
|
||||
|
||||
public void testRegionSettingBackcompat() {
|
||||
Settings settings = Settings.builder()
|
||||
.put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
|
||||
.build();
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
|
||||
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
|
||||
"s3-eu-west-1.amazonaws.com");
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
|
||||
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
|
||||
"s3.eu-central-1.amazonaws.com");
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
|
||||
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
|
||||
settings, "repository.endpoint");
|
||||
assertWarnings("[" + InternalAwsS3Service.REGION_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + S3Repository.Repository.REGION_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
|
||||
}
|
||||
|
||||
public void testRegionWithAwsAndS3Settings() {
|
||||
public void testRegionAndEndpointSettingBackcompatPrecedence() {
|
||||
Settings settings = Settings.builder()
|
||||
.put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
|
||||
.put(InternalAwsS3Service.CLOUD_S3.REGION_SETTING.getKey(), randomFrom("us-west", "us-west-1"))
|
||||
.build();
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
|
||||
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
|
||||
"s3-us-west-1.amazonaws.com");
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
|
||||
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
|
||||
"s3.eu-central-1.amazonaws.com");
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
|
||||
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
|
||||
settings, "repository.endpoint");
|
||||
assertWarnings("[" + InternalAwsS3Service.REGION_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + S3Repository.Repository.REGION_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + InternalAwsS3Service.CLOUD_S3.REGION_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
|
||||
}
|
||||
|
||||
public void testInvalidRegion() {
|
||||
Settings settings = Settings.builder()
|
||||
.put(InternalAwsS3Service.REGION_SETTING.getKey(), "does-not-exist")
|
||||
.put("s3.client.default.region", "does-not-exist")
|
||||
.build();
|
||||
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, null);
|
||||
assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, null);
|
||||
});
|
||||
assertThat(e.getMessage(), containsString("No automatic endpoint could be derived from region"));
|
||||
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
|
||||
"s3.eu-central-1.amazonaws.com");
|
||||
launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
|
||||
settings, "repository.endpoint");
|
||||
}
|
||||
|
||||
protected void launchAWSEndpointTest(Settings singleRepositorySettings, Settings settings,
|
||||
String expectedEndpoint) {
|
||||
String region = S3Repository.getValue(singleRepositorySettings, settings,
|
||||
S3Repository.Repository.REGION_SETTING, S3Repository.Repositories.REGION_SETTING);
|
||||
String endpoint = S3Repository.getValue(singleRepositorySettings, settings,
|
||||
S3Repository.Repository.ENDPOINT_SETTING, S3Repository.Repositories.ENDPOINT_SETTING);
|
||||
|
||||
String foundEndpoint = InternalAwsS3Service.findEndpoint(logger, settings, endpoint, region);
|
||||
private void assertEndpoint(Settings repositorySettings, Settings settings,
|
||||
String expectedEndpoint) {
|
||||
String configName = InternalAwsS3Service.CLIENT_NAME.get(repositorySettings);
|
||||
String foundEndpoint = InternalAwsS3Service.findEndpoint(logger, repositorySettings, settings, configName);
|
||||
assertThat(foundEndpoint, is(expectedEndpoint));
|
||||
}
|
||||
|
||||
|
||||
@ -27,7 +27,7 @@ public class EnvironmentCredentialsTests extends ESTestCase {
|
||||
|
||||
public void test() {
|
||||
AWSCredentialsProvider provider =
|
||||
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY);
|
||||
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY, "default");
|
||||
// NOTE: env vars are setup by the test runner in gradle
|
||||
assertEquals("env_access", provider.getCredentials().getAWSAccessKeyId());
|
||||
assertEquals("env_secret", provider.getCredentials().getAWSSecretKey());
|
||||
|
||||
@ -26,7 +26,7 @@ import org.elasticsearch.test.ESTestCase;
|
||||
public class SyspropCredentialsTests extends ESTestCase {
|
||||
public void test() {
|
||||
AWSCredentialsProvider provider =
|
||||
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY);
|
||||
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY, "default");
|
||||
// NOTE: sys props are setup by the test runner in gradle
|
||||
assertEquals("sysprop_access", provider.getCredentials().getAWSAccessKeyId());
|
||||
assertEquals("sysprop_secret", provider.getCredentials().getAWSSecretKey());
|
||||
|
||||
@ -18,14 +18,14 @@
|
||||
*/
|
||||
package org.elasticsearch.cloud.aws;
|
||||
|
||||
import com.amazonaws.Protocol;
|
||||
import java.util.IdentityHashMap;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3Client;
|
||||
import org.elasticsearch.ElasticsearchException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin;
|
||||
|
||||
import java.util.IdentityHashMap;
|
||||
|
||||
public class TestAwsS3Service extends InternalAwsS3Service {
|
||||
public static class TestPlugin extends S3RepositoryPlugin {
|
||||
@Override
|
||||
@ -42,9 +42,9 @@ public class TestAwsS3Service extends InternalAwsS3Service {
|
||||
|
||||
|
||||
@Override
|
||||
public synchronized AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
|
||||
boolean useThrottleRetries, Boolean pathStyleAccess) {
|
||||
return cachedWrapper(super.client(repositorySettings, endpoint, protocol, region, maxRetries, useThrottleRetries, pathStyleAccess));
|
||||
public synchronized AmazonS3 client(Settings repositorySettings, Integer maxRetries,
|
||||
boolean useThrottleRetries, Boolean pathStyleAccess) {
|
||||
return cachedWrapper(super.client(repositorySettings, maxRetries, useThrottleRetries, pathStyleAccess));
|
||||
}
|
||||
|
||||
private AmazonS3 cachedWrapper(AmazonS3 client) {
|
||||
|
||||
@ -197,12 +197,7 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
|
||||
|
||||
Settings settings = internalCluster().getInstance(Settings.class);
|
||||
Settings bucket = settings.getByPrefix("repositories.s3.");
|
||||
AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(
|
||||
repositorySettings,
|
||||
null,
|
||||
S3Repository.Repositories.PROTOCOL_SETTING.get(settings),
|
||||
S3Repository.Repositories.REGION_SETTING.get(settings),
|
||||
null, randomBoolean(), null);
|
||||
AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(repositorySettings, null, randomBoolean(), null);
|
||||
|
||||
String bucketName = bucket.get("bucket");
|
||||
logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
|
||||
@ -467,16 +462,12 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
|
||||
settings.getByPrefix("repositories.s3.external-bucket.")
|
||||
};
|
||||
for (Settings bucket : buckets) {
|
||||
String endpoint = bucket.get("endpoint", S3Repository.Repositories.ENDPOINT_SETTING.get(settings));
|
||||
Protocol protocol = S3Repository.Repositories.PROTOCOL_SETTING.get(settings);
|
||||
String region = bucket.get("region", S3Repository.Repositories.REGION_SETTING.get(settings));
|
||||
String bucketName = bucket.get("bucket");
|
||||
|
||||
// We check that settings has been set in elasticsearch.yml integration test file
|
||||
// as described in README
|
||||
assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
|
||||
AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(Settings.EMPTY, endpoint, protocol, region, null,
|
||||
randomBoolean(), null);
|
||||
AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(Settings.EMPTY, null, randomBoolean(), null);
|
||||
try {
|
||||
ObjectListing prevListing = null;
|
||||
//From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
|
||||
@ -513,7 +504,7 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
|
||||
client.deleteObjects(multiObjectDeleteRequest);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to delete S3 repository [{}] in [{}]", bucketName, region), ex);
|
||||
logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to delete S3 repository [{}]", bucketName), ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -19,13 +19,13 @@
|
||||
|
||||
package org.elasticsearch.repositories.s3;
|
||||
|
||||
import com.amazonaws.Protocol;
|
||||
import java.io.IOException;
|
||||
|
||||
import com.amazonaws.services.s3.AbstractAmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import org.elasticsearch.cloud.aws.AwsS3Service;
|
||||
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
|
||||
import org.elasticsearch.common.component.AbstractLifecycleComponent;
|
||||
import org.elasticsearch.common.settings.MockSecureSettings;
|
||||
import org.elasticsearch.common.settings.SecureString;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.ByteSizeUnit;
|
||||
@ -35,8 +35,6 @@ import org.elasticsearch.repositories.RepositoryException;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.hamcrest.Matchers;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.elasticsearch.repositories.s3.S3Repository.Repositories;
|
||||
import static org.elasticsearch.repositories.s3.S3Repository.Repository;
|
||||
import static org.elasticsearch.repositories.s3.S3Repository.getValue;
|
||||
@ -62,19 +60,15 @@ public class S3RepositoryTests extends ESTestCase {
|
||||
@Override
|
||||
protected void doClose() {}
|
||||
@Override
|
||||
public AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
|
||||
public AmazonS3 client(Settings repositorySettings, Integer maxRetries,
|
||||
boolean useThrottleRetries, Boolean pathStyleAccess) {
|
||||
return new DummyS3Client();
|
||||
}
|
||||
}
|
||||
|
||||
public void testSettingsResolution() throws Exception {
|
||||
MockSecureSettings secureSettings1 = new MockSecureSettings();
|
||||
secureSettings1.setString(Repository.KEY_SETTING.getKey(), "key1");
|
||||
Settings localSettings = Settings.builder().setSecureSettings(secureSettings1).build();
|
||||
MockSecureSettings secureSettings2 = new MockSecureSettings();
|
||||
secureSettings2.setString(Repositories.KEY_SETTING.getKey(), "key2");
|
||||
Settings globalSettings = Settings.builder().setSecureSettings(secureSettings2).build();
|
||||
Settings localSettings = Settings.builder().put(Repository.KEY_SETTING.getKey(), "key1").build();
|
||||
Settings globalSettings = Settings.builder().put(Repositories.KEY_SETTING.getKey(), "key2").build();
|
||||
|
||||
assertEquals(new SecureString("key1".toCharArray()),
|
||||
getValue(localSettings, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
|
||||
@ -84,6 +78,8 @@ public class S3RepositoryTests extends ESTestCase {
|
||||
getValue(Settings.EMPTY, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
|
||||
assertEquals(new SecureString("".toCharArray()),
|
||||
getValue(Settings.EMPTY, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING));
|
||||
assertWarnings("[" + Repository.KEY_SETTING.getKey() + "] setting was deprecated",
|
||||
"[" + Repositories.KEY_SETTING.getKey() + "] setting was deprecated");
|
||||
}
|
||||
|
||||
public void testInvalidChunkBufferSizeSettings() throws IOException {
|
||||
|
||||
@ -306,12 +306,12 @@ public abstract class ESTestCase extends LuceneTestCase {
|
||||
}
|
||||
try {
|
||||
final List<String> actualWarnings = threadContext.getResponseHeaders().get(DeprecationLogger.WARNING_HEADER);
|
||||
assertEquals("Expected " + expectedWarnings.length + " warnings but found " + actualWarnings.size() + "\nExpected: "
|
||||
+ Arrays.asList(expectedWarnings) + "\nActual: " + actualWarnings,
|
||||
expectedWarnings.length, actualWarnings.size());
|
||||
for (String msg : expectedWarnings) {
|
||||
assertThat(actualWarnings, hasItem(containsString(msg)));
|
||||
}
|
||||
assertEquals("Expected " + expectedWarnings.length + " warnings but found " + actualWarnings.size() + "\nExpected: "
|
||||
+ Arrays.asList(expectedWarnings) + "\nActual: " + actualWarnings,
|
||||
expectedWarnings.length, actualWarnings.size());
|
||||
} finally {
|
||||
resetDeprecationLogger();
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user