From 2176d09c373861e2e8f83b46d2a9521621391faa Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 8 Jul 2019 08:10:01 +0200 Subject: [PATCH] Provide an Option to Use Path-Style-Access with S3 Repo (#41966) (#44046) * Provide an Option to Use Path-Style-Access with S3 Repo * As discussed, added the option to use path style access back again and deprecated it. * Defaulted to `false` * Added warning to docs * Closes #41816 --- docs/plugins/repository-s3.asciidoc | 18 ++++++++++++++---- docs/reference/migration/migrate_7_4.asciidoc | 18 ++++++++++++++++++ .../repositories/s3/S3ClientSettings.java | 18 +++++++++++++++--- .../repositories/s3/S3RepositoryPlugin.java | 1 + .../repositories/s3/S3Service.java | 7 ++++--- .../repositories/s3/S3ClientSettingsTests.java | 7 +++++++ 6 files changed, 59 insertions(+), 10 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 50674b62654..5711dc8d953 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -145,6 +145,20 @@ settings belong in the `elasticsearch.yml` file. Whether retries should be throttled (i.e. should back off). Must be `true` or `false`. Defaults to `true`. +`path_style_access`:: + + Whether to force the use of the path style access pattern. If `true`, the + path style access pattern will be used. If `false`, the access pattern will + be automatically determined by the AWS Java SDK (See + https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#setPathStyleAccessEnabled-java.lang.Boolean-[AWS + documentation] for details). Defaults to `false`. + +[[repository-s3-path-style-deprecation]] +NOTE: In versions `7.0`, `7.1`, `7.2` and `7.3` all bucket operations used the +https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/[now-deprecated] +path style access pattern. If your deployment requires the path style access +pattern then you should set this setting to `true` when upgrading. + [float] [[repository-s3-compatible-services]] ===== S3-compatible services @@ -381,10 +395,6 @@ bucket, in this example, named "foo". The bucket needs to exist to register a repository for snapshots. If you did not create the bucket then the repository registration will fail. -Note: Starting in version 7.0, all bucket operations are using the path style -access pattern. In previous versions the decision to use virtual hosted style or -path style access was made by the AWS Java SDK. - [[repository-s3-aws-vpc]] [float] ==== AWS VPC Bandwidth Settings diff --git a/docs/reference/migration/migrate_7_4.asciidoc b/docs/reference/migration/migrate_7_4.asciidoc index 041b1db281c..92f04767aff 100644 --- a/docs/reference/migration/migrate_7_4.asciidoc +++ b/docs/reference/migration/migrate_7_4.asciidoc @@ -38,3 +38,21 @@ If a document doesn't have a value for a vector field (dense_vector or sparse_vector) on which a vector function is executed, an error will be thrown. +[float] +[[breaking_74_snapshots_changes]] +=== Snapshot and Restore changes + +[float] +==== The S3 repository plugin uses the DNS style access pattern by default + +Starting in version 7.4 the `repository-s3` plugin does not use the +now-deprecated path-style access pattern by default. In versions 7.0, 7.1, 7.2 +and 7.3 the `repository-s3` plugin always used the path-style access pattern. +This is a breaking change for deployments that only support path-style access +but which are recognized as supporting DNS-style access by the AWS SDK. If your +deployment only supports path-style access and is affected by this change then +you must configure the S3 client setting `path_style_access` to `true`. This +breaking change was made necessary by +https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/[AWS's +announcement] that the path-style access pattern is deprecated and will be +unsupported on buckets created after September 30th 2020. diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index ea45fbaf93d..ae2bd2e905b 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -95,6 +95,10 @@ final class S3ClientSettings { static final Setting.AffixSetting USE_THROTTLE_RETRIES_SETTING = Setting.affixKeySetting(PREFIX, "use_throttle_retries", key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope)); + /** Whether the s3 client should use path style access. */ + static final Setting.AffixSetting USE_PATH_STYLE_ACCESS = Setting.affixKeySetting(PREFIX, "path_style_access", + key -> Setting.boolSetting(key, false, Property.NodeScope)); + /** Credentials to authenticate with s3. */ final S3BasicCredentials credentials; @@ -127,9 +131,13 @@ final class S3ClientSettings { /** Whether the s3 client should use an exponential backoff retry policy. */ final boolean throttleRetries; + /** Whether the s3 client should use path style access. */ + final boolean pathStyleAccess; + private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, - int readTimeoutMillis, int maxRetries, boolean throttleRetries) { + int readTimeoutMillis, int maxRetries, boolean throttleRetries, + boolean pathStyleAccess) { this.credentials = credentials; this.endpoint = endpoint; this.protocol = protocol; @@ -140,6 +148,7 @@ final class S3ClientSettings { this.readTimeoutMillis = readTimeoutMillis; this.maxRetries = maxRetries; this.throttleRetries = throttleRetries; + this.pathStyleAccess = pathStyleAccess; } /** @@ -162,6 +171,7 @@ final class S3ClientSettings { getRepoSettingOrDefault(READ_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(readTimeoutMillis)).millis()); final int newMaxRetries = getRepoSettingOrDefault(MAX_RETRIES_SETTING, normalizedSettings, maxRetries); final boolean newThrottleRetries = getRepoSettingOrDefault(USE_THROTTLE_RETRIES_SETTING, normalizedSettings, throttleRetries); + final boolean usePathStyleAccess = getRepoSettingOrDefault(USE_PATH_STYLE_ACCESS, normalizedSettings, pathStyleAccess); final S3BasicCredentials newCredentials; if (checkDeprecatedCredentials(repoSettings)) { newCredentials = loadDeprecatedCredentials(repoSettings); @@ -183,7 +193,8 @@ final class S3ClientSettings { proxyPassword, newReadTimeoutMillis, newMaxRetries, - newThrottleRetries + newThrottleRetries, + usePathStyleAccess ); } @@ -270,7 +281,8 @@ final class S3ClientSettings { proxyPassword.toString(), Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), getConfigValue(settings, clientName, MAX_RETRIES_SETTING), - getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING) + getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING), + getConfigValue(settings, clientName, USE_PATH_STYLE_ACCESS) ); } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index bb044771e60..118197902f6 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -105,6 +105,7 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo S3ClientSettings.READ_TIMEOUT_SETTING, S3ClientSettings.MAX_RETRIES_SETTING, S3ClientSettings.USE_THROTTLE_RETRIES_SETTING, + S3ClientSettings.USE_PATH_STYLE_ACCESS, S3Repository.ACCESS_KEY_SETTING, S3Repository.SECRET_KEY_SETTING); } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index b0c8a619813..b3210dc5b30 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -152,9 +152,10 @@ class S3Service implements Closeable { // // We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too) // so this change removes that usage of a deprecated API. - builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null)) - .enablePathStyleAccess(); - + builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null)); + if (clientSettings.pathStyleAccess) { + builder.enablePathStyleAccess(); + } return builder.build(); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java index 53740672df3..312d9649aa3 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -144,4 +144,11 @@ public class S3ClientSettingsTests extends ESTestCase { assertThat(credentials.getSessionToken(), is("session_token")); } } + + public void testPathStyleAccessCanBeSet() { + final Map settings = S3ClientSettings.load( + Settings.builder().put("s3.client.other.path_style_access", true).build()); + assertThat(settings.get("default").pathStyleAccess, is(false)); + assertThat(settings.get("other").pathStyleAccess, is(true)); + } }