mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-17 10:25:15 +00:00
S3 Repository: Deprecate remaining repositories.s3.* settings (#24144)
Most of these settings should always be pulled from the repository settings. A couple were leftover that should be moved to client settings. The path style access setting should be removed altogether. This commit adds deprecations for all of these existing settings, as well as adding new client specific settings for max retries and throttling. relates #24143
This commit is contained in:
Ryan Ernst
GitHub
parent
3c845727f8
commit
51b33f1fd5
@ -75,12 +75,6 @@ class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Se
|
||||
Strings.collectionToDelimitedString(clientsSettings.keySet(), ","));
|
||||
}
|
||||
|
||||
Integer maxRetries = getValue(repositorySettings, settings,
|
||||
S3Repository.Repository.MAX_RETRIES_SETTING,
|
||||
S3Repository.Repositories.MAX_RETRIES_SETTING);
|
||||
boolean useThrottleRetries = getValue(repositorySettings, settings,
|
||||
S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING,
|
||||
S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING);
|
||||
// If the user defined a path style access setting, we rely on it,
|
||||
// otherwise we use the default value set by the SDK
|
||||
Boolean pathStyleAccess = null;
|
||||
@ -91,12 +85,11 @@ class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Se
|
||||
S3Repository.Repositories.PATH_STYLE_ACCESS_SETTING);
|
||||
}
|
||||
|
||||
logger.debug("creating S3 client with client_name [{}], endpoint [{}], max_retries [{}], " +
|
||||
"use_throttle_retries [{}], path_style_access [{}]",
|
||||
clientName, clientSettings.endpoint, maxRetries, useThrottleRetries, pathStyleAccess);
|
||||
logger.debug("creating S3 client with client_name [{}], endpoint [{}], path_style_access [{}]",
|
||||
clientName, clientSettings.endpoint, pathStyleAccess);
|
||||
|
||||
AWSCredentialsProvider credentials = buildCredentials(logger, clientSettings);
|
||||
ClientConfiguration configuration = buildConfiguration(clientSettings, maxRetries, useThrottleRetries);
|
||||
ClientConfiguration configuration = buildConfiguration(clientSettings, repositorySettings);
|
||||
|
||||
client = new AmazonS3Client(credentials, configuration);
|
||||
|
||||
@ -113,7 +106,7 @@ class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Se
|
||||
}
|
||||
|
||||
// pkg private for tests
|
||||
static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings, Integer maxRetries, boolean useThrottleRetries) {
|
||||
static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings, Settings repositorySettings) {
|
||||
ClientConfiguration clientConfiguration = new ClientConfiguration();
|
||||
// the response metadata cache is only there for diagnostics purposes,
|
||||
// but can force objects from every response to the old generation.
|
||||
@ -128,10 +121,13 @@ class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Se
|
||||
clientConfiguration.setProxyPassword(clientSettings.proxyPassword);
|
||||
}
|
||||
|
||||
Integer maxRetries = getRepoValue(repositorySettings, S3Repository.Repository.MAX_RETRIES_SETTING, clientSettings.maxRetries);
|
||||
if (maxRetries != null) {
|
||||
// If not explicitly set, default to 3 with exponential backoff policy
|
||||
clientConfiguration.setMaxErrorRetry(maxRetries);
|
||||
}
|
||||
boolean useThrottleRetries = getRepoValue(repositorySettings,
|
||||
S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING, clientSettings.throttleRetries);
|
||||
clientConfiguration.setUseThrottleRetries(useThrottleRetries);
|
||||
clientConfiguration.setSocketTimeout(clientSettings.readTimeoutMillis);
|
||||
|
||||
@ -149,6 +145,14 @@ class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Se
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns the value for a given setting from the repository, or returns the fallback value. */
|
||||
private static <T> T getRepoValue(Settings repositorySettings, Setting<T> repositorySetting, T fallback) {
|
||||
if (repositorySetting.exists(repositorySettings)) {
|
||||
return repositorySetting.get(repositorySettings);
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doStart() throws ElasticsearchException {
|
||||
}
|
||||
|
||||
@ -31,6 +31,7 @@ import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import org.elasticsearch.common.settings.SecureSetting;
|
||||
import org.elasticsearch.common.settings.SecureString;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Setting.Property;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
|
||||
@ -52,20 +53,19 @@ class S3ClientSettings {
|
||||
|
||||
/** An override for the s3 endpoint to connect to. */
|
||||
static final Setting.AffixSetting<String> ENDPOINT_SETTING = Setting.affixKeySetting(PREFIX, "endpoint",
|
||||
key -> new Setting<>(key, "", s -> s.toLowerCase(Locale.ROOT),
|
||||
Setting.Property.NodeScope));
|
||||
key -> new Setting<>(key, "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope));
|
||||
|
||||
/** The protocol to use to connect to s3. */
|
||||
static final Setting.AffixSetting<Protocol> PROTOCOL_SETTING = Setting.affixKeySetting(PREFIX, "protocol",
|
||||
key -> new Setting<>(key, "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Setting.Property.NodeScope));
|
||||
key -> new Setting<>(key, "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope));
|
||||
|
||||
/** The host name of a proxy to connect to s3 through. */
|
||||
static final Setting.AffixSetting<String> PROXY_HOST_SETTING = Setting.affixKeySetting(PREFIX, "proxy.host",
|
||||
key -> Setting.simpleString(key, Setting.Property.NodeScope));
|
||||
key -> Setting.simpleString(key, Property.NodeScope));
|
||||
|
||||
/** The port of a proxy to connect to s3 through. */
|
||||
static final Setting.AffixSetting<Integer> PROXY_PORT_SETTING = Setting.affixKeySetting(PREFIX, "proxy.port",
|
||||
key -> Setting.intSetting(key, 80, 0, 1<<16, Setting.Property.NodeScope));
|
||||
key -> Setting.intSetting(key, 80, 0, 1<<16, Property.NodeScope));
|
||||
|
||||
/** The username of a proxy to connect to s3 through. */
|
||||
static final Setting.AffixSetting<SecureString> PROXY_USERNAME_SETTING = Setting.affixKeySetting(PREFIX, "proxy.username",
|
||||
@ -77,8 +77,15 @@ class S3ClientSettings {
|
||||
|
||||
/** The socket timeout for connecting to s3. */
|
||||
static final Setting.AffixSetting<TimeValue> READ_TIMEOUT_SETTING = Setting.affixKeySetting(PREFIX, "read_timeout",
|
||||
key -> Setting.timeSetting(key, TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT),
|
||||
Setting.Property.NodeScope));
|
||||
key -> Setting.timeSetting(key, TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope));
|
||||
|
||||
/** The number of retries to use when an s3 request fails. */
|
||||
static final Setting.AffixSetting<Integer> MAX_RETRIES_SETTING = Setting.affixKeySetting(PREFIX, "max_retries",
|
||||
key -> Setting.intSetting(key, S3Repository.Repositories.MAX_RETRIES_SETTING, 0, Property.NodeScope));
|
||||
|
||||
/** Whether retries should be throttled (ie use backoff). */
|
||||
static final Setting.AffixSetting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.affixKeySetting(PREFIX, "use_throttle_retries",
|
||||
key -> Setting.boolSetting(key, S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING, Property.NodeScope));
|
||||
|
||||
/** Credentials to authenticate with s3. */
|
||||
final BasicAWSCredentials credentials;
|
||||
@ -106,9 +113,15 @@ class S3ClientSettings {
|
||||
/** The read timeout for the s3 client. */
|
||||
final int readTimeoutMillis;
|
||||
|
||||
/** The number of retries to use for the s3 client. */
|
||||
final int maxRetries;
|
||||
|
||||
/** Whether the s3 client should use an exponential backoff retry policy. */
|
||||
final boolean throttleRetries;
|
||||
|
||||
private S3ClientSettings(BasicAWSCredentials credentials, String endpoint, Protocol protocol,
|
||||
String proxyHost, int proxyPort, String proxyUsername,
|
||||
String proxyPassword, int readTimeoutMillis) {
|
||||
String proxyHost, int proxyPort, String proxyUsername, String proxyPassword,
|
||||
int readTimeoutMillis, int maxRetries, boolean throttleRetries) {
|
||||
this.credentials = credentials;
|
||||
this.endpoint = endpoint;
|
||||
this.protocol = protocol;
|
||||
@ -117,6 +130,8 @@ class S3ClientSettings {
|
||||
this.proxyUsername = proxyUsername;
|
||||
this.proxyPassword = proxyPassword;
|
||||
this.readTimeoutMillis = readTimeoutMillis;
|
||||
this.maxRetries = maxRetries;
|
||||
this.throttleRetries = throttleRetries;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -163,7 +178,9 @@ class S3ClientSettings {
|
||||
getConfigValue(settings, clientName, PROXY_PORT_SETTING),
|
||||
proxyUsername.toString(),
|
||||
proxyPassword.toString(),
|
||||
(int)getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()
|
||||
(int)getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis(),
|
||||
getConfigValue(settings, clientName, MAX_RETRIES_SETTING),
|
||||
getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -61,13 +61,13 @@ class S3Repository extends BlobStoreRepository {
|
||||
/**
|
||||
* repositories.s3.bucket: The name of the bucket to be used for snapshots.
|
||||
*/
|
||||
Setting<String> BUCKET_SETTING = Setting.simpleString("repositories.s3.bucket", Property.NodeScope);
|
||||
Setting<String> BUCKET_SETTING = Setting.simpleString("repositories.s3.bucket", Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* repositories.s3.server_side_encryption: When set to true files are encrypted on server side using AES256 algorithm.
|
||||
* Defaults to false.
|
||||
*/
|
||||
Setting<Boolean> SERVER_SIDE_ENCRYPTION_SETTING =
|
||||
Setting.boolSetting("repositories.s3.server_side_encryption", false, Property.NodeScope);
|
||||
Setting.boolSetting("repositories.s3.server_side_encryption", false, Property.NodeScope, Property.Deprecated);
|
||||
|
||||
/**
|
||||
* Default is to use 100MB (S3 defaults) for heaps above 2GB and 5% of
|
||||
@ -89,41 +89,41 @@ class S3Repository extends BlobStoreRepository {
|
||||
*/
|
||||
Setting<ByteSizeValue> BUFFER_SIZE_SETTING =
|
||||
Setting.byteSizeSetting("repositories.s3.buffer_size", DEFAULT_BUFFER_SIZE,
|
||||
new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope);
|
||||
new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* repositories.s3.max_retries: Number of retries in case of S3 errors. Defaults to 3.
|
||||
*/
|
||||
Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope);
|
||||
Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* repositories.s3.use_throttle_retries: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
|
||||
*/
|
||||
Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("repositories.s3.use_throttle_retries",
|
||||
ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
|
||||
ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g.
|
||||
*/
|
||||
Setting<ByteSizeValue> CHUNK_SIZE_SETTING =
|
||||
Setting.byteSizeSetting("repositories.s3.chunk_size", new ByteSizeValue(1, ByteSizeUnit.GB),
|
||||
new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope);
|
||||
new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* repositories.s3.compress: When set to true metadata files are stored in compressed format. This setting doesn’t affect index
|
||||
* files that are already compressed by default. Defaults to false.
|
||||
*/
|
||||
Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("repositories.s3.compress", false, Property.NodeScope);
|
||||
Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("repositories.s3.compress", false, Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* repositories.s3.storage_class: Sets the S3 storage class type for the backup files. Values may be standard, reduced_redundancy,
|
||||
* standard_ia. Defaults to standard.
|
||||
*/
|
||||
Setting<String> STORAGE_CLASS_SETTING = Setting.simpleString("repositories.s3.storage_class", Property.NodeScope);
|
||||
Setting<String> STORAGE_CLASS_SETTING = Setting.simpleString("repositories.s3.storage_class", Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* repositories.s3.canned_acl: The S3 repository supports all S3 canned ACLs : private, public-read, public-read-write,
|
||||
* authenticated-read, log-delivery-write, bucket-owner-read, bucket-owner-full-control. Defaults to private.
|
||||
*/
|
||||
Setting<String> CANNED_ACL_SETTING = Setting.simpleString("repositories.s3.canned_acl", Property.NodeScope);
|
||||
Setting<String> CANNED_ACL_SETTING = Setting.simpleString("repositories.s3.canned_acl", Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* repositories.s3.base_path: Specifies the path within bucket to repository data. Defaults to root directory.
|
||||
*/
|
||||
Setting<String> BASE_PATH_SETTING = Setting.simpleString("repositories.s3.base_path", Property.NodeScope);
|
||||
Setting<String> BASE_PATH_SETTING = Setting.simpleString("repositories.s3.base_path", Property.NodeScope, Property.Deprecated);
|
||||
/**
|
||||
* repositories.s3.path_style_access: When set to true configures the client to use path-style access for all requests.
|
||||
Amazon S3 supports virtual-hosted-style and path-style access in all Regions. The path-style syntax, however,
|
||||
@ -132,7 +132,8 @@ class S3Repository extends BlobStoreRepository {
|
||||
in path-style access) and the bucket being accessed (some buckets are not valid DNS names). Setting this flag
|
||||
will result in path-style access being used for all requests.
|
||||
*/
|
||||
Setting<Boolean> PATH_STYLE_ACCESS_SETTING = Setting.boolSetting("repositories.s3.path_style_access", false, Property.NodeScope);
|
||||
Setting<Boolean> PATH_STYLE_ACCESS_SETTING = Setting.boolSetting("repositories.s3.path_style_access", false,
|
||||
Property.NodeScope, Property.Deprecated);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -160,13 +161,13 @@ class S3Repository extends BlobStoreRepository {
|
||||
* max_retries
|
||||
* @see Repositories#MAX_RETRIES_SETTING
|
||||
*/
|
||||
Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3);
|
||||
Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3, Property.Deprecated);
|
||||
/**
|
||||
* use_throttle_retries
|
||||
* @see Repositories#USE_THROTTLE_RETRIES_SETTING
|
||||
*/
|
||||
Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("use_throttle_retries",
|
||||
ClientConfiguration.DEFAULT_THROTTLE_RETRIES);
|
||||
ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.Deprecated);
|
||||
/**
|
||||
* chunk_size
|
||||
* @see Repositories#CHUNK_SIZE_SETTING
|
||||
@ -198,7 +199,7 @@ class S3Repository extends BlobStoreRepository {
|
||||
* path_style_access
|
||||
* @see Repositories#PATH_STYLE_ACCESS_SETTING
|
||||
*/
|
||||
Setting<Boolean> PATH_STYLE_ACCESS_SETTING = Setting.boolSetting("path_style_access", false);
|
||||
Setting<Boolean> PATH_STYLE_ACCESS_SETTING = Setting.boolSetting("path_style_access", false, Property.Deprecated);
|
||||
}
|
||||
|
||||
private final S3BlobStore blobStore;
|
||||
|
||||
@ -90,6 +90,8 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin {
|
||||
S3ClientSettings.PROXY_USERNAME_SETTING,
|
||||
S3ClientSettings.PROXY_PASSWORD_SETTING,
|
||||
S3ClientSettings.READ_TIMEOUT_SETTING,
|
||||
S3ClientSettings.MAX_RETRIES_SETTING,
|
||||
S3ClientSettings.USE_THROTTLE_RETRIES_SETTING,
|
||||
|
||||
// Register S3 repositories settings: repositories.s3
|
||||
S3Repository.Repositories.BUCKET_SETTING,
|
||||
|
||||
@ -24,6 +24,7 @@ import com.amazonaws.Protocol;
|
||||
import com.amazonaws.auth.AWSCredentials;
|
||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||
import org.elasticsearch.common.settings.MockSecureSettings;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
@ -86,21 +87,70 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
"aws_proxy_password", 3, false, 10000);
|
||||
}
|
||||
|
||||
public void testGlobalMaxRetries() {
|
||||
public void testGlobalMaxRetriesBackcompat() {
|
||||
Settings settings = Settings.builder()
|
||||
.put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10)
|
||||
.build();
|
||||
launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, null, -1, null,
|
||||
null, 10, false, 50000);
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
|
||||
S3Repository.Repositories.MAX_RETRIES_SETTING
|
||||
});
|
||||
}
|
||||
|
||||
public void testRepositoryMaxRetries() {
|
||||
Settings repositorySettings = generateRepositorySettings(20);
|
||||
Settings settings = Settings.builder()
|
||||
.put("s3.client.default.max_retries", 5)
|
||||
.build();
|
||||
launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, null, -1, null,
|
||||
null, 5, false, 50000);
|
||||
}
|
||||
|
||||
public void testRepositoryMaxRetriesBackcompat() {
|
||||
Settings repositorySettings = Settings.builder()
|
||||
.put(S3Repository.Repository.MAX_RETRIES_SETTING.getKey(), 20).build();
|
||||
Settings settings = Settings.builder()
|
||||
.put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10)
|
||||
.build();
|
||||
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null,
|
||||
null, 20, false, 50000);
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
|
||||
S3Repository.Repositories.MAX_RETRIES_SETTING,
|
||||
S3Repository.Repository.MAX_RETRIES_SETTING
|
||||
});
|
||||
}
|
||||
|
||||
public void testGlobalThrottleRetriesBackcompat() {
|
||||
Settings settings = Settings.builder()
|
||||
.put(S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING.getKey(), true)
|
||||
.build();
|
||||
launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, null, -1, null,
|
||||
null, 3, true, 50000);
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
|
||||
S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING
|
||||
});
|
||||
}
|
||||
|
||||
public void testRepositoryThrottleRetries() {
|
||||
Settings settings = Settings.builder()
|
||||
.put("s3.client.default.use_throttle_retries", true)
|
||||
.build();
|
||||
launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, null, -1, null,
|
||||
null, 3, true, 50000);
|
||||
}
|
||||
|
||||
public void testRepositoryThrottleRetriesBackcompat() {
|
||||
Settings repositorySettings = Settings.builder()
|
||||
.put(S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING.getKey(), true).build();
|
||||
Settings settings = Settings.builder()
|
||||
.put(S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING.getKey(), false)
|
||||
.build();
|
||||
launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null,
|
||||
null, 3, true, 50000);
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
|
||||
S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING,
|
||||
S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING
|
||||
});
|
||||
}
|
||||
|
||||
private void launchAWSConfigurationTest(Settings settings,
|
||||
@ -113,13 +163,9 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
Integer expectedMaxRetries,
|
||||
boolean expectedUseThrottleRetries,
|
||||
int expectedReadTimeout) {
|
||||
Integer maxRetries = S3Repository.getValue(singleRepositorySettings, settings,
|
||||
S3Repository.Repository.MAX_RETRIES_SETTING, S3Repository.Repositories.MAX_RETRIES_SETTING);
|
||||
Boolean useThrottleRetries = S3Repository.getValue(singleRepositorySettings, settings,
|
||||
S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING, S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING);
|
||||
|
||||
S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default");
|
||||
ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(clientSettings, maxRetries, useThrottleRetries);
|
||||
ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(clientSettings, singleRepositorySettings);
|
||||
|
||||
assertThat(configuration.getResponseMetadataCacheSize(), is(0));
|
||||
assertThat(configuration.getProtocol(), is(expectedProtocol));
|
||||
@ -132,14 +178,6 @@ public class AwsS3ServiceImplTests extends ESTestCase {
|
||||
assertThat(configuration.getSocketTimeout(), is(expectedReadTimeout));
|
||||
}
|
||||
|
||||
private static Settings generateRepositorySettings(Integer maxRetries) {
|
||||
Settings.Builder builder = Settings.builder();
|
||||
if (maxRetries != null) {
|
||||
builder.put(S3Repository.Repository.MAX_RETRIES_SETTING.getKey(), maxRetries);
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
public void testEndpointSetting() {
|
||||
Settings settings = Settings.builder()
|
||||
.put("s3.client.default.endpoint", "s3.endpoint")
|
||||
|
||||
@ -108,7 +108,8 @@ public class S3RepositoryTests extends ESTestCase {
|
||||
Settings settings = Settings.builder().put(Repositories.BASE_PATH_SETTING.getKey(), "/foo/bar").build();
|
||||
s3repo = new S3Repository(metadata, settings, NamedXContentRegistry.EMPTY, new DummyS3Service());
|
||||
assertEquals("foo/bar/", s3repo.basePath().buildAsString()); // make sure leading `/` is removed and trailing is added
|
||||
assertWarnings("S3 repository base_path" +
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[] { Repositories.BASE_PATH_SETTING },
|
||||
"S3 repository base_path" +
|
||||
" trimming the leading `/`, and leading `/` will not be supported for the S3 repository in future releases");
|
||||
}
|
||||
|
||||
|
||||
@ -327,6 +327,7 @@ public abstract class ESTestCase extends LuceneTestCase {
|
||||
}
|
||||
try {
|
||||
final List<String> actualWarnings = threadContext.getResponseHeaders().get("Warning");
|
||||
assertNotNull(actualWarnings);
|
||||
final Set<String> actualWarningValues =
|
||||
actualWarnings.stream().map(DeprecationLogger::extractWarningValueFromWarningHeader).collect(Collectors.toSet());
|
||||
for (String msg : expectedWarnings) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user