From a6a138905dfffbf0b0ed3401af9769c633735f97 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 22 Feb 2018 15:40:20 +0100 Subject: [PATCH] Use client settings in repository-gcs (#28575) Similarly to what has been done for s3 and azure, this commit removes the repository settings `application_name` and `connect/read_timeout` in favor of client settings. It introduce a GoogleCloudStorageClientSettings class (similar to S3ClientSettings) and a bunch of unit tests for that, it aligns the documentation to be more coherent with the S3 one, it documents the connect/read timeouts that were not documented at all and also adds a new client setting that allows to define a custom endpoint. --- docs/plugins/repository-gcs.asciidoc | 104 +++++++-- docs/plugins/repository-s3.asciidoc | 2 +- .../migration/migrate_7_0/plugins.asciidoc | 6 + .../gcs/GoogleCloudStorageClientSettings.java | 173 +++++++++++++++ .../gcs/GoogleCloudStoragePlugin.java | 37 ++-- .../gcs/GoogleCloudStorageRepository.java | 32 +-- .../gcs/GoogleCloudStorageService.java | 204 ++++++++---------- ...eCloudStorageBlobStoreRepositoryTests.java | 23 +- ...GoogleCloudStorageClientSettingsTests.java | 197 +++++++++++++++++ .../gcs/GoogleCloudStorageServiceTests.java | 49 ++--- .../src/test/resources/dummy-account.json | 12 -- 11 files changed, 594 insertions(+), 245 deletions(-) create mode 100644 plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java create mode 100644 plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java delete mode 100644 plugins/repository-gcs/src/test/resources/dummy-account.json diff --git a/docs/plugins/repository-gcs.asciidoc b/docs/plugins/repository-gcs.asciidoc index 22186a8e079..a51200fb7fe 100644 --- a/docs/plugins/repository-gcs.asciidoc +++ b/docs/plugins/repository-gcs.asciidoc @@ -116,23 +116,15 @@ PUT _snapshot/my_gcs_repository // CONSOLE // TEST[skip:we don't have gcs setup while testing this] -[[repository-gcs-bucket-permission]] -===== Set Bucket Permission +[[repository-gcs-client]] +==== Client Settings -The service account used to access the bucket must have the "Writer" access to the bucket: +The client used to connect to Google Cloud Storage has a number of settings available. +Client setting names are of the form `gcs.client.CLIENT_NAME.SETTING_NAME` and specified +inside `elasticsearch.yml`. The default client name looked up by a `gcs` repository is +called `default`, but can be customized with the repository setting `client`. -1. Connect to the https://console.cloud.google.com/[Google Cloud Platform Console] -2. Select your project -3. Got to the https://console.cloud.google.com/storage/browser[Storage Browser] -4. Select the bucket and "Edit bucket permission" -5. The service account must be configured as a "User" with "Writer" access - - -[[repository-gcs-repository]] -==== Create a Repository - -Once everything is installed and every node is started, you can create a new repository that -uses Google Cloud Storage to store snapshots: +For example: [source,js] ---- @@ -140,13 +132,74 @@ PUT _snapshot/my_gcs_repository { "type": "gcs", "settings": { - "bucket": "my_bucket" + "bucket": "my_bucket", + "client": "my_alternate_client" } } ---- // CONSOLE // TEST[skip:we don't have gcs setup while testing this] +Some settings are sensitive and must be stored in the +{ref}/secure-settings.html[elasticsearch keystore]. This is the case for the service account file: + +[source,sh] +---- +bin/elasticsearch-keystore add-file gcs.client.default.credentials_file +---- + +The following are the available client settings. Those that must be stored in the keystore +are marked as `Secure`. + +`credentials_file`:: + + The service account file that is used to authenticate to the Google Cloud Storage service. (Secure) + +`endpoint`:: + + The Google Cloud Storage service endpoint to connect to. This will be automatically + determined by the Google Cloud Storage client but can be specified explicitly. + +`connect_timeout`:: + + The timeout to establish a connection to the Google Cloud Storage service. The value should + specify the unit. For example, a value of `5s` specifies a 5 second timeout. The value of `-1` + corresponds to an infinite timeout. The default value is 20 seconds. + +`read_timeout`:: + + The timeout to read data from an established connection. The value should + specify the unit. For example, a value of `5s` specifies a 5 second timeout. The value of `-1` + corresponds to an infinite timeout. The default value is 20 seconds. + +`application_name`:: + + Name used by the client when it uses the Google Cloud Storage service. Setting + a custom name can be useful to authenticate your cluster when requests + statistics are logged in the Google Cloud Platform. Default to `repository-gcs` + +[[repository-gcs-repository]] +==== Repository Settings + +The `gcs` repository type supports a number of settings to customize how data +is stored in Google Cloud Storage. + +These can be specified when creating the repository. For example: + +[source,js] +---- +PUT _snapshot/my_gcs_repository +{ + "type": "gcs", + "settings": { + "bucket": "my_other_bucket", + "base_path": "dev" + } +} +---- +// CONSOLE +// TEST[skip:we don't have gcs set up while testing this] + The following settings are supported: `bucket`:: @@ -155,8 +208,8 @@ The following settings are supported: `client`:: - The client congfiguration to use. This controls which credentials are used to connect - to Compute Engine. + The name of the client to use to connect to Google Cloud Storage. + Defaults to `default`. `base_path`:: @@ -177,6 +230,15 @@ The following settings are supported: `application_name`:: - Name used by the plugin when it uses the Google Cloud JSON API. Setting - a custom name can be useful to authenticate your cluster when requests - statistics are logged in the Google Cloud Platform. Default to `repository-gcs` + deprecated[7.0.0, This setting is now defined in the <>] + +[[repository-gcs-bucket-permission]] +===== Recommended Bucket Permission + +The service account used to access the bucket must have the "Writer" access to the bucket: + +1. Connect to the https://console.cloud.google.com/[Google Cloud Platform Console] +2. Select your project +3. Got to the https://console.cloud.google.com/storage/browser[Storage Browser] +4. Select the bucket and "Edit bucket permission" +5. The service account must be configured as a "User" with "Writer" access diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index eb0828e96c6..bff64ebdc91 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -36,7 +36,7 @@ PUT _snapshot/my_s3_repository The client used to connect to S3 has a number of settings available. Client setting names are of the form `s3.client.CLIENT_NAME.SETTING_NAME` and specified inside `elasticsearch.yml`. The -default client name looked up by an s3 repository is called `default`, but can be customized +default client name looked up by a `s3` repository is called `default`, but can be customized with the repository setting `client`. For example: [source,js] diff --git a/docs/reference/migration/migrate_7_0/plugins.asciidoc b/docs/reference/migration/migrate_7_0/plugins.asciidoc index 6bc9edec0da..365a2c5a39f 100644 --- a/docs/reference/migration/migrate_7_0/plugins.asciidoc +++ b/docs/reference/migration/migrate_7_0/plugins.asciidoc @@ -12,3 +12,9 @@ You must set it per azure client instead. Like `azure.client.default.timeout: 10 See {plugins}/repository-azure-usage.html#repository-azure-repository-settings[Azure Repository settings]. +==== Google Cloud Storage Repository plugin + +* The repository settings `application_name`, `connect_timeout` and `read_timeout` have been removed and +must now be specified in the client settings instead. + +See {plugins}/repository-gcs-client.html#repository-gcs-client[Google Cloud Storage Client Settings]. diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java new file mode 100644 index 00000000000..03295c18c8a --- /dev/null +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java @@ -0,0 +1,173 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.repositories.gcs; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.services.storage.StorageScopes; +import org.elasticsearch.common.settings.SecureSetting; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.common.settings.Setting.timeSetting; + +/** + * Container for Google Cloud Storage clients settings. + */ +public class GoogleCloudStorageClientSettings { + + private static final String PREFIX = "gcs.client."; + + /** A json Service Account file loaded from secure settings. */ + static final Setting.AffixSetting CREDENTIALS_FILE_SETTING = Setting.affixKeySetting(PREFIX, "credentials_file", + key -> SecureSetting.secureFile(key, null)); + + /** An override for the Storage endpoint to connect to. */ + static final Setting.AffixSetting ENDPOINT_SETTING = Setting.affixKeySetting(PREFIX, "endpoint", + key -> new Setting<>(key, "", s -> s, Setting.Property.NodeScope)); + + /** + * The timeout to establish a connection. A value of {@code -1} corresponds to an infinite timeout. A value of {@code 0} + * corresponds to the default timeout of the Google Cloud Storage Java Library. + */ + static final Setting.AffixSetting CONNECT_TIMEOUT_SETTING = Setting.affixKeySetting(PREFIX, "connect_timeout", + key -> timeSetting(key, TimeValue.ZERO, TimeValue.MINUS_ONE, Setting.Property.NodeScope)); + + /** + * The timeout to read data from an established connection. A value of {@code -1} corresponds to an infinite timeout. A value of + * {@code 0} corresponds to the default timeout of the Google Cloud Storage Java Library. + */ + static final Setting.AffixSetting READ_TIMEOUT_SETTING = Setting.affixKeySetting(PREFIX, "read_timeout", + key -> timeSetting(key, TimeValue.ZERO, TimeValue.MINUS_ONE, Setting.Property.NodeScope)); + + /** Name used by the client when it uses the Google Cloud JSON API. **/ + static final Setting.AffixSetting APPLICATION_NAME_SETTING = Setting.affixKeySetting(PREFIX, "application_name", + key -> new Setting<>(key, "repository-gcs", s -> s, Setting.Property.NodeScope)); + + /** The credentials used by the client to connect to the Storage endpoint **/ + private final GoogleCredential credential; + + /** The Storage root URL the client should talk to, or empty string to use the default. **/ + private final String endpoint; + + /** The timeout to establish a connection **/ + private final TimeValue connectTimeout; + + /** The timeout to read data from an established connection **/ + private final TimeValue readTimeout; + + /** The Storage client application name **/ + private final String applicationName; + + GoogleCloudStorageClientSettings(final GoogleCredential credential, + final String endpoint, + final TimeValue connectTimeout, + final TimeValue readTimeout, + final String applicationName) { + this.credential = credential; + this.endpoint = endpoint; + this.connectTimeout = connectTimeout; + this.readTimeout = readTimeout; + this.applicationName = applicationName; + } + + public GoogleCredential getCredential() { + return credential; + } + + public String getEndpoint() { + return endpoint; + } + + public TimeValue getConnectTimeout() { + return connectTimeout; + } + + public TimeValue getReadTimeout() { + return readTimeout; + } + + public String getApplicationName() { + return applicationName; + } + + public static Map load(final Settings settings) { + final Map clients = new HashMap<>(); + for (String clientName: settings.getGroups(PREFIX).keySet()) { + clients.put(clientName, getClientSettings(settings, clientName)); + } + if (clients.containsKey("default") == false) { + // this won't find any settings under the default client, + // but it will pull all the fallback static settings + clients.put("default", getClientSettings(settings, "default")); + } + return Collections.unmodifiableMap(clients); + } + + static GoogleCloudStorageClientSettings getClientSettings(final Settings settings, final String clientName) { + return new GoogleCloudStorageClientSettings( + loadCredential(settings, clientName), + getConfigValue(settings, clientName, ENDPOINT_SETTING), + getConfigValue(settings, clientName, CONNECT_TIMEOUT_SETTING), + getConfigValue(settings, clientName, READ_TIMEOUT_SETTING), + getConfigValue(settings, clientName, APPLICATION_NAME_SETTING) + ); + } + + /** + * Loads the service account file corresponding to a given client name. If no file is defined for the client, + * a {@code null} credential is returned. + * + * @param settings the {@link Settings} + * @param clientName the client name + * + * @return the {@link GoogleCredential} to use for the given client, {@code null} if no service account is defined. + */ + static GoogleCredential loadCredential(final Settings settings, final String clientName) { + try { + if (CREDENTIALS_FILE_SETTING.getConcreteSettingForNamespace(clientName).exists(settings) == false) { + // explicitly returning null here so that the default credential + // can be loaded later when creating the Storage client + return null; + } + try (InputStream credStream = CREDENTIALS_FILE_SETTING.getConcreteSettingForNamespace(clientName).get(settings)) { + GoogleCredential credential = GoogleCredential.fromStream(credStream); + if (credential.createScopedRequired()) { + credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)); + } + return credential; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static T getConfigValue(final Settings settings, final String clientName, final Setting.AffixSetting clientSetting) { + Setting concreteSetting = clientSetting.getConcreteSettingForNamespace(clientName); + return concreteSetting.get(settings); + } +} diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java index fa43ab1bc3f..ef24cd959e5 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java @@ -19,15 +19,8 @@ package org.elasticsearch.repositories.gcs; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.Collections; -import java.util.List; -import java.util.Map; - import com.google.api.client.auth.oauth2.TokenRequest; import com.google.api.client.auth.oauth2.TokenResponse; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; @@ -48,13 +41,16 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.Repository; -import org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository; -import org.elasticsearch.repositories.gcs.GoogleCloudStorageService; + +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; public class GoogleCloudStoragePlugin extends Plugin implements RepositoryPlugin { - public static final String NAME = "repository-gcs"; - static { /* * Google HTTP client changes access levels because its silly and we @@ -112,15 +108,19 @@ public class GoogleCloudStoragePlugin extends Plugin implements RepositoryPlugin }); } - private final Map credentials; + private final Map clientsSettings; - public GoogleCloudStoragePlugin(Settings settings) { - credentials = GoogleCloudStorageService.loadClientCredentials(settings); + public GoogleCloudStoragePlugin(final Settings settings) { + clientsSettings = GoogleCloudStorageClientSettings.load(settings); + } + + protected Map getClientsSettings() { + return clientsSettings; } // overridable for tests protected GoogleCloudStorageService createStorageService(Environment environment) { - return new GoogleCloudStorageService.InternalGoogleCloudStorageService(environment, credentials); + return new GoogleCloudStorageService(environment, clientsSettings); } @Override @@ -131,6 +131,11 @@ public class GoogleCloudStoragePlugin extends Plugin implements RepositoryPlugin @Override public List> getSettings() { - return Collections.singletonList(GoogleCloudStorageService.CREDENTIALS_FILE_SETTING); + return Arrays.asList( + GoogleCloudStorageClientSettings.CREDENTIALS_FILE_SETTING, + GoogleCloudStorageClientSettings.ENDPOINT_SETTING, + GoogleCloudStorageClientSettings.CONNECT_TIMEOUT_SETTING, + GoogleCloudStorageClientSettings.READ_TIMEOUT_SETTING, + GoogleCloudStorageClientSettings.APPLICATION_NAME_SETTING); } } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java index d435f001448..e193b8238b8 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -39,7 +39,6 @@ import static org.elasticsearch.common.settings.Setting.Property; import static org.elasticsearch.common.settings.Setting.boolSetting; import static org.elasticsearch.common.settings.Setting.byteSizeSetting; import static org.elasticsearch.common.settings.Setting.simpleString; -import static org.elasticsearch.common.settings.Setting.timeSetting; import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; class GoogleCloudStorageRepository extends BlobStoreRepository { @@ -50,8 +49,6 @@ class GoogleCloudStorageRepository extends BlobStoreRepository { static final String TYPE = "gcs"; - static final TimeValue NO_TIMEOUT = timeValueMillis(-1); - static final Setting BUCKET = simpleString("bucket", Property.NodeScope, Property.Dynamic); static final Setting BASE_PATH = @@ -60,13 +57,7 @@ class GoogleCloudStorageRepository extends BlobStoreRepository { boolSetting("compress", false, Property.NodeScope, Property.Dynamic); static final Setting CHUNK_SIZE = byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, Property.NodeScope, Property.Dynamic); - static final Setting APPLICATION_NAME = - new Setting<>("application_name", GoogleCloudStoragePlugin.NAME, Function.identity(), Property.NodeScope, Property.Dynamic); static final Setting CLIENT_NAME = new Setting<>("client", "default", Function.identity()); - static final Setting HTTP_READ_TIMEOUT = - timeSetting("http.read_timeout", NO_TIMEOUT, Property.NodeScope, Property.Dynamic); - static final Setting HTTP_CONNECT_TIMEOUT = - timeSetting("http.connect_timeout", NO_TIMEOUT, Property.NodeScope, Property.Dynamic); private final ByteSizeValue chunkSize; private final boolean compress; @@ -79,9 +70,7 @@ class GoogleCloudStorageRepository extends BlobStoreRepository { super(metadata, environment.settings(), namedXContentRegistry); String bucket = getSetting(BUCKET, metadata); - String application = getSetting(APPLICATION_NAME, metadata); String clientName = CLIENT_NAME.get(metadata.settings()); - String basePath = BASE_PATH.get(metadata.settings()); if (Strings.hasLength(basePath)) { BlobPath path = new BlobPath(); @@ -93,29 +82,12 @@ class GoogleCloudStorageRepository extends BlobStoreRepository { this.basePath = BlobPath.cleanPath(); } - TimeValue connectTimeout = null; - TimeValue readTimeout = null; - - TimeValue timeout = HTTP_CONNECT_TIMEOUT.get(metadata.settings()); - if ((timeout != null) && (timeout.millis() != NO_TIMEOUT.millis())) { - connectTimeout = timeout; - } - - timeout = HTTP_READ_TIMEOUT.get(metadata.settings()); - if ((timeout != null) && (timeout.millis() != NO_TIMEOUT.millis())) { - readTimeout = timeout; - } - this.compress = getSetting(COMPRESS, metadata); this.chunkSize = getSetting(CHUNK_SIZE, metadata); - logger.debug("using bucket [{}], base_path [{}], chunk_size [{}], compress [{}], application [{}]", - bucket, basePath, chunkSize, compress, application); + logger.debug("using bucket [{}], base_path [{}], chunk_size [{}], compress [{}]", bucket, basePath, chunkSize, compress); - TimeValue finalConnectTimeout = connectTimeout; - TimeValue finalReadTimeout = readTimeout; - Storage client = SocketAccess.doPrivilegedIOException(() -> - storageService.createClient(clientName, application, finalConnectTimeout, finalReadTimeout)); + Storage client = SocketAccess.doPrivilegedIOException(() -> storageService.createClient(clientName)); this.blobStore = new GoogleCloudStorageBlobStore(settings, bucket, client); } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java index 7625ae78450..bccc5e0ffdc 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java @@ -25,156 +25,122 @@ import com.google.api.client.http.HttpBackOffIOExceptionHandler; import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; import com.google.api.client.http.HttpUnsuccessfulResponseHandler; -import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.client.util.ExponentialBackOff; import com.google.api.services.storage.Storage; -import com.google.api.services.storage.StorageScopes; -import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.settings.SecureSetting; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.env.Environment; import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import java.util.Set; -interface GoogleCloudStorageService { +public class GoogleCloudStorageService extends AbstractComponent { - /** A json credentials file loaded from secure settings. */ - Setting.AffixSetting CREDENTIALS_FILE_SETTING = Setting.affixKeySetting("gcs.client.", "credentials_file", - key -> SecureSetting.secureFile(key, null)); + /** Clients settings identified by client name. */ + private final Map clientsSettings; + + public GoogleCloudStorageService(final Environment environment, final Map clientsSettings) { + super(environment.settings()); + this.clientsSettings = clientsSettings; + } /** * Creates a client that can be used to manage Google Cloud Storage objects. * - * @param clientName name of client settings to use from secure settings - * @param application name of the application - * @param connectTimeout connection timeout for HTTP requests - * @param readTimeout read timeout for HTTP requests - * @return a Client instance that can be used to manage objects + * @param clientName name of client settings to use from secure settings + * @return a Client instance that can be used to manage Storage objects */ - Storage createClient(String clientName, String application, - TimeValue connectTimeout, TimeValue readTimeout) throws Exception; + public Storage createClient(final String clientName) throws Exception { + final GoogleCloudStorageClientSettings clientSettings = clientsSettings.get(clientName); + if (clientSettings == null) { + throw new IllegalArgumentException("Unknown client name [" + clientName + "]. Existing client configs: " + + Strings.collectionToDelimitedString(clientsSettings.keySet(), ",")); + } + + HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + HttpRequestInitializer requestInitializer = createRequestInitializer(clientSettings); + + Storage.Builder storage = new Storage.Builder(transport, JacksonFactory.getDefaultInstance(), requestInitializer); + if (Strings.hasLength(clientSettings.getApplicationName())) { + storage.setApplicationName(clientSettings.getApplicationName()); + } + if (Strings.hasLength(clientSettings.getEndpoint())) { + storage.setRootUrl(clientSettings.getEndpoint()); + } + return storage.build(); + } + + static HttpRequestInitializer createRequestInitializer(final GoogleCloudStorageClientSettings settings) throws IOException { + GoogleCredential credential = settings.getCredential(); + if (credential == null) { + credential = GoogleCredential.getApplicationDefault(); + } + return new DefaultHttpRequestInitializer(credential, toTimeout(settings.getConnectTimeout()), toTimeout(settings.getReadTimeout())); + } + + /** Converts timeout values from the settings to a timeout value for the Google Cloud SDK **/ + static Integer toTimeout(final TimeValue timeout) { + // Null or zero in settings means the default timeout + if (timeout == null || TimeValue.ZERO.equals(timeout)) { + return null; + } + // -1 means infinite timeout + if (TimeValue.MINUS_ONE.equals(timeout)) { + // 0 is the infinite timeout expected by Google Cloud SDK + return 0; + } + return Math.toIntExact(timeout.getMillis()); + } /** - * Default implementation + * HTTP request initializer that set timeouts and backoff handler while deferring authentication to GoogleCredential. + * See https://cloud.google.com/storage/transfer/create-client#retry */ - class InternalGoogleCloudStorageService extends AbstractComponent implements GoogleCloudStorageService { + static class DefaultHttpRequestInitializer implements HttpRequestInitializer { - /** Credentials identified by client name. */ - private final Map credentials; + private final Integer connectTimeout; + private final Integer readTimeout; + private final GoogleCredential credential; - InternalGoogleCloudStorageService(Environment environment, Map credentials) { - super(environment.settings()); - this.credentials = credentials; + DefaultHttpRequestInitializer(GoogleCredential credential, Integer connectTimeoutMillis, Integer readTimeoutMillis) { + this.credential = credential; + this.connectTimeout = connectTimeoutMillis; + this.readTimeout = readTimeoutMillis; } @Override - public Storage createClient(String clientName, String application, - TimeValue connectTimeout, TimeValue readTimeout) throws Exception { - try { - GoogleCredential credential = getCredential(clientName); - NetHttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - - Storage.Builder storage = new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(), - new DefaultHttpRequestInitializer(credential, connectTimeout, readTimeout)); - storage.setApplicationName(application); - - logger.debug("initializing client with service account [{}/{}]", - credential.getServiceAccountId(), credential.getServiceAccountUser()); - return storage.build(); - } catch (IOException e) { - throw new ElasticsearchException("Error when loading Google Cloud Storage credentials file", e); + public void initialize(HttpRequest request) { + if (connectTimeout != null) { + request.setConnectTimeout(connectTimeout); } - } - - // pkg private for tests - GoogleCredential getCredential(String clientName) throws IOException { - GoogleCredential cred = credentials.get(clientName); - if (cred != null) { - return cred; - } - return getDefaultCredential(); - } - - // pkg private for tests - GoogleCredential getDefaultCredential() throws IOException { - return GoogleCredential.getApplicationDefault(); - } - - /** - * HTTP request initializer that set timeouts and backoff handler while deferring authentication to GoogleCredential. - * See https://cloud.google.com/storage/transfer/create-client#retry - */ - class DefaultHttpRequestInitializer implements HttpRequestInitializer { - - private final TimeValue connectTimeout; - private final TimeValue readTimeout; - private final GoogleCredential credential; - - DefaultHttpRequestInitializer(GoogleCredential credential, TimeValue connectTimeout, TimeValue readTimeout) { - this.credential = credential; - this.connectTimeout = connectTimeout; - this.readTimeout = readTimeout; + if (readTimeout != null) { + request.setReadTimeout(readTimeout); } - @Override - public void initialize(HttpRequest request) throws IOException { - if (connectTimeout != null) { - request.setConnectTimeout((int) connectTimeout.millis()); - } - if (readTimeout != null) { - request.setReadTimeout((int) readTimeout.millis()); + request.setIOExceptionHandler(new HttpBackOffIOExceptionHandler(newBackOff())); + request.setInterceptor(credential); + + final HttpUnsuccessfulResponseHandler handler = new HttpBackOffUnsuccessfulResponseHandler(newBackOff()); + request.setUnsuccessfulResponseHandler((req, resp, supportsRetry) -> { + // Let the credential handle the response. If it failed, we rely on our backoff handler + return credential.handleResponse(req, resp, supportsRetry) || handler.handleResponse(req, resp, supportsRetry); } + ); + } - request.setIOExceptionHandler(new HttpBackOffIOExceptionHandler(newBackOff())); - request.setInterceptor(credential); - - final HttpUnsuccessfulResponseHandler handler = new HttpBackOffUnsuccessfulResponseHandler(newBackOff()); - request.setUnsuccessfulResponseHandler((req, resp, supportsRetry) -> { - // Let the credential handle the response. If it failed, we rely on our backoff handler - return credential.handleResponse(req, resp, supportsRetry) || handler.handleResponse(req, resp, supportsRetry); - } - ); - } - - private ExponentialBackOff newBackOff() { - return new ExponentialBackOff.Builder() - .setInitialIntervalMillis(100) - .setMaxIntervalMillis(6000) - .setMaxElapsedTimeMillis(900000) - .setMultiplier(1.5) - .setRandomizationFactor(0.5) - .build(); - } + private ExponentialBackOff newBackOff() { + return new ExponentialBackOff.Builder() + .setInitialIntervalMillis(100) + .setMaxIntervalMillis(6000) + .setMaxElapsedTimeMillis(900000) + .setMultiplier(1.5) + .setRandomizationFactor(0.5) + .build(); } } - /** Load all secure credentials from the settings. */ - static Map loadClientCredentials(Settings settings) { - Map credentials = new HashMap<>(); - Iterable> iterable = CREDENTIALS_FILE_SETTING.getAllConcreteSettings(settings)::iterator; - for (Setting concreteSetting : iterable) { - try (InputStream credStream = concreteSetting.get(settings)) { - GoogleCredential credential = GoogleCredential.fromStream(credStream); - if (credential.createScopedRequired()) { - credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)); - } - credentials.put(CREDENTIALS_FILE_SETTING.getNamespace(concreteSetting), credential); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - return credentials; - } } diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index dbad40ec083..ec166ff867f 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase; @@ -32,8 +31,9 @@ import org.junit.BeforeClass; import java.net.SocketPermission; import java.security.AccessController; -import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -48,7 +48,7 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepos @Override protected Collection> nodePlugins() { - return Arrays.asList(MockGoogleCloudStoragePlugin.class); + return Collections.singletonList(MockGoogleCloudStoragePlugin.class); } @Override @@ -58,7 +58,6 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepos .setSettings(Settings.builder() .put("bucket", BUCKET) .put("base_path", GoogleCloudStorageBlobStoreRepositoryTests.class.getSimpleName()) - .put("service_account", "_default_") .put("compress", randomBoolean()) .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); } @@ -69,19 +68,23 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepos } public static class MockGoogleCloudStoragePlugin extends GoogleCloudStoragePlugin { - public MockGoogleCloudStoragePlugin() { - super(Settings.EMPTY); + public MockGoogleCloudStoragePlugin(final Settings settings) { + super(settings); } @Override protected GoogleCloudStorageService createStorageService(Environment environment) { - return new MockGoogleCloudStorageService(); + return new MockGoogleCloudStorageService(environment, getClientsSettings()); } } - public static class MockGoogleCloudStorageService implements GoogleCloudStorageService { + public static class MockGoogleCloudStorageService extends GoogleCloudStorageService { + + MockGoogleCloudStorageService(Environment environment, Map clientsSettings) { + super(environment, clientsSettings); + } + @Override - public Storage createClient(String accountName, String application, - TimeValue connectTimeout, TimeValue readTimeout) throws Exception { + public Storage createClient(String clientName) { // The actual impl might open a connection. So check we have permission when this call is made. AccessController.checkPermission(new SocketPermission("*", "connect")); return storage.get(); diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java new file mode 100644 index 00000000000..badd86cd8a2 --- /dev/null +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java @@ -0,0 +1,197 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.repositories.gcs; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.services.storage.StorageScopes; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; + +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.APPLICATION_NAME_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.CONNECT_TIMEOUT_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.CREDENTIALS_FILE_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.ENDPOINT_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.READ_TIMEOUT_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.getClientSettings; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.loadCredential; + +public class GoogleCloudStorageClientSettingsTests extends ESTestCase { + + public void testLoadWithEmptySettings() { + Map clientsSettings = GoogleCloudStorageClientSettings.load(Settings.EMPTY); + assertEquals(1, clientsSettings.size()); + assertNotNull(clientsSettings.get("default")); + } + + public void testLoad() throws Exception { + final int nbClients = randomIntBetween(1, 5); + final Tuple, Settings> randomClients = randomClients(nbClients); + final Map expectedClientsSettings = randomClients.v1(); + + Map actualClientsSettings = GoogleCloudStorageClientSettings.load(randomClients.v2()); + assertEquals(expectedClientsSettings.size(), actualClientsSettings.size()); + + for (String clientName : expectedClientsSettings.keySet()) { + GoogleCloudStorageClientSettings actualClientSettings = actualClientsSettings.get(clientName); + assertNotNull(actualClientSettings); + GoogleCloudStorageClientSettings expectedClientSettings = expectedClientsSettings.get(clientName); + assertNotNull(expectedClientSettings); + + assertGoogleCredential(expectedClientSettings.getCredential(), actualClientSettings.getCredential()); + assertEquals(expectedClientSettings.getEndpoint(), actualClientSettings.getEndpoint()); + assertEquals(expectedClientSettings.getConnectTimeout(), actualClientSettings.getConnectTimeout()); + assertEquals(expectedClientSettings.getReadTimeout(), actualClientSettings.getReadTimeout()); + assertEquals(expectedClientSettings.getApplicationName(), actualClientSettings.getApplicationName()); + } + } + + public void testLoadCredential() throws Exception { + Tuple, Settings> randomClient = randomClients(1); + GoogleCloudStorageClientSettings expectedClientSettings = randomClient.v1().values().iterator().next(); + String clientName = randomClient.v1().keySet().iterator().next(); + + assertGoogleCredential(expectedClientSettings.getCredential(), loadCredential(randomClient.v2(), clientName)); + } + + /** Generates a given number of GoogleCloudStorageClientSettings along with the Settings to build them from **/ + private Tuple, Settings> randomClients(final int nbClients) throws Exception { + final Map expectedClients = new HashMap<>(); + expectedClients.put("default", getClientSettings(Settings.EMPTY, "default")); + + final Settings.Builder settings = Settings.builder(); + final MockSecureSettings secureSettings = new MockSecureSettings(); + + for (int i = 0; i < nbClients; i++) { + String clientName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + + GoogleCloudStorageClientSettings clientSettings = randomClient(clientName, settings, secureSettings); + expectedClients.put(clientName, clientSettings); + } + + if (randomBoolean()) { + GoogleCloudStorageClientSettings clientSettings = randomClient("default", settings, secureSettings); + expectedClients.put("default", clientSettings); + } + + return Tuple.tuple(expectedClients, settings.setSecureSettings(secureSettings).build()); + } + + /** Generates a random GoogleCloudStorageClientSettings along with the Settings to build it **/ + private static GoogleCloudStorageClientSettings randomClient(final String clientName, + final Settings.Builder settings, + final MockSecureSettings secureSettings) throws Exception { + + Tuple credentials = randomCredential(clientName); + GoogleCredential credential = credentials.v1(); + secureSettings.setFile(CREDENTIALS_FILE_SETTING.getConcreteSettingForNamespace(clientName).getKey(), credentials.v2()); + + String endpoint; + if (randomBoolean()) { + endpoint = randomAlphaOfLength(5); + settings.put(ENDPOINT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint); + } else { + endpoint = ENDPOINT_SETTING.getDefault(Settings.EMPTY); + } + + TimeValue connectTimeout; + if (randomBoolean()) { + connectTimeout = randomTimeout(); + settings.put(CONNECT_TIMEOUT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), connectTimeout.getStringRep()); + } else { + connectTimeout = CONNECT_TIMEOUT_SETTING.getDefault(Settings.EMPTY); + } + + TimeValue readTimeout; + if (randomBoolean()) { + readTimeout = randomTimeout(); + settings.put(READ_TIMEOUT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), readTimeout.getStringRep()); + } else { + readTimeout = READ_TIMEOUT_SETTING.getDefault(Settings.EMPTY); + } + + String applicationName; + if (randomBoolean()) { + applicationName = randomAlphaOfLength(5); + settings.put(APPLICATION_NAME_SETTING.getConcreteSettingForNamespace(clientName).getKey(), applicationName); + } else { + applicationName = APPLICATION_NAME_SETTING.getDefault(Settings.EMPTY); + } + + return new GoogleCloudStorageClientSettings(credential, endpoint, connectTimeout, readTimeout, applicationName); + } + + /** Generates a random GoogleCredential along with its corresponding Service Account file provided as a byte array **/ + private static Tuple randomCredential(final String clientName) throws Exception { + KeyPair keyPair = KeyPairGenerator.getInstance("RSA").generateKeyPair(); + + GoogleCredential.Builder credentialBuilder = new GoogleCredential.Builder(); + credentialBuilder.setServiceAccountId(clientName); + credentialBuilder.setServiceAccountProjectId("project_id_" + clientName); + credentialBuilder.setServiceAccountScopes(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)); + credentialBuilder.setServiceAccountPrivateKey(keyPair.getPrivate()); + credentialBuilder.setServiceAccountPrivateKeyId("private_key_id_" + clientName); + + String encodedPrivateKey = Base64.getEncoder().encodeToString(keyPair.getPrivate().getEncoded()); + String serviceAccount = "{\"type\":\"service_account\"," + + "\"project_id\":\"project_id_" + clientName + "\"," + + "\"private_key_id\":\"private_key_id_" + clientName + "\"," + + "\"private_key\":\"-----BEGIN PRIVATE KEY-----\\n" + + encodedPrivateKey + + "\\n-----END PRIVATE KEY-----\\n\"," + + "\"client_email\":\"" + clientName + "\"," + + "\"client_id\":\"id_" + clientName + "\"," + + "\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\"," + + "\"token_uri\":\"https://accounts.google.com/o/oauth2/token\"," + + "\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\"," + + "\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/" + + clientName + + "%40appspot.gserviceaccount.com\"}"; + + return Tuple.tuple(credentialBuilder.build(), serviceAccount.getBytes(StandardCharsets.UTF_8)); + } + + private static TimeValue randomTimeout() { + return randomFrom(TimeValue.MINUS_ONE, TimeValue.ZERO, TimeValue.parseTimeValue(randomPositiveTimeValue(), "test")); + } + + private static void assertGoogleCredential(final GoogleCredential expected, final GoogleCredential actual) { + if (expected != null) { + assertEquals(expected.getServiceAccountUser(), actual.getServiceAccountUser()); + assertEquals(expected.getServiceAccountId(), actual.getServiceAccountId()); + assertEquals(expected.getServiceAccountProjectId(), actual.getServiceAccountProjectId()); + assertEquals(expected.getServiceAccountScopesAsString(), actual.getServiceAccountScopesAsString()); + assertEquals(expected.getServiceAccountPrivateKey(), actual.getServiceAccountPrivateKey()); + assertEquals(expected.getServiceAccountPrivateKeyId(), actual.getServiceAccountPrivateKeyId()); + } else { + assertNull(actual); + } + } +} diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java index 07bd6974c65..44897819fd9 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java @@ -31,17 +31,10 @@ import com.google.api.client.testing.http.MockHttpTransport; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.TestEnvironment; -import org.elasticsearch.repositories.gcs.GoogleCloudStorageService.InternalGoogleCloudStorageService; import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import java.io.InputStream; -import java.util.Collections; -import java.util.Map; -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonMap; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Mockito.mock; @@ -51,34 +44,8 @@ import static org.mockito.Mockito.when; public class GoogleCloudStorageServiceTests extends ESTestCase { - private InputStream getDummyCredentialStream() throws IOException { - return GoogleCloudStorageServiceTests.class.getResourceAsStream("/dummy-account.json"); - } - - public void testDefaultCredential() throws Exception { - Environment env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); - GoogleCredential cred = GoogleCredential.fromStream(getDummyCredentialStream()); - InternalGoogleCloudStorageService service = new InternalGoogleCloudStorageService(env, Collections.emptyMap()) { - @Override - GoogleCredential getDefaultCredential() throws IOException { - return cred; - } - }; - assertSame(cred, service.getCredential("default")); - - service.new DefaultHttpRequestInitializer(cred, null, null); - } - - public void testClientCredential() throws Exception { - GoogleCredential cred = GoogleCredential.fromStream(getDummyCredentialStream()); - Map credentials = singletonMap("clientname", cred); - Environment env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); - InternalGoogleCloudStorageService service = new InternalGoogleCloudStorageService(env, credentials); - assertSame(cred, service.getCredential("clientname")); - } - /** - * Test that the {@link InternalGoogleCloudStorageService.DefaultHttpRequestInitializer} attaches new instances + * Test that the {@link GoogleCloudStorageService.DefaultHttpRequestInitializer} attaches new instances * of {@link HttpIOExceptionHandler} and {@link HttpUnsuccessfulResponseHandler} for every HTTP requests. */ public void testDefaultHttpRequestInitializer() throws IOException { @@ -90,9 +57,13 @@ public class GoogleCloudStorageServiceTests extends ESTestCase { final TimeValue readTimeout = TimeValue.timeValueSeconds(randomIntBetween(1, 120)); final TimeValue connectTimeout = TimeValue.timeValueSeconds(randomIntBetween(1, 120)); + final String endpoint = randomBoolean() ? randomAlphaOfLength(10) : null; + final String applicationName = randomBoolean() ? randomAlphaOfLength(10) : null; - final InternalGoogleCloudStorageService service = new InternalGoogleCloudStorageService(environment, emptyMap()); - final HttpRequestInitializer initializer = service.new DefaultHttpRequestInitializer(credential, connectTimeout, readTimeout); + final GoogleCloudStorageClientSettings clientSettings = + new GoogleCloudStorageClientSettings(credential, endpoint, connectTimeout, readTimeout, applicationName); + + final HttpRequestInitializer initializer = GoogleCloudStorageService.createRequestInitializer(clientSettings); final HttpRequestFactory requestFactory = new MockHttpTransport().createRequestFactory(initializer); final HttpRequest request1 = requestFactory.buildGetRequest(new GenericUrl()); @@ -117,4 +88,10 @@ public class GoogleCloudStorageServiceTests extends ESTestCase { request2.getUnsuccessfulResponseHandler().handleResponse(null, null, false); verify(credential, times(2)).handleResponse(any(HttpRequest.class), any(HttpResponse.class), anyBoolean()); } + + public void testToTimeout() { + assertNull(GoogleCloudStorageService.toTimeout(null)); + assertNull(GoogleCloudStorageService.toTimeout(TimeValue.ZERO)); + assertEquals(0, GoogleCloudStorageService.toTimeout(TimeValue.MINUS_ONE).intValue()); + } } diff --git a/plugins/repository-gcs/src/test/resources/dummy-account.json b/plugins/repository-gcs/src/test/resources/dummy-account.json deleted file mode 100644 index e282b6db0e3..00000000000 --- a/plugins/repository-gcs/src/test/resources/dummy-account.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "service_account", - "project_id": "some-project-name", - "private_key_id": "c7cefcb7c72a2880ecce49cb9d1095de5a61aff0", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDa+7r0RE1YykXC\n+d+DXlN3Dg3aL1YOfYuhy5PF/Vi0FFQHyXuPtAvkVHZD2NxMDZq2DxTu3AVLh1UE\nt2hMrWjDQDuArPl8FezpyYQwde04Qlx1YpQ1xUjTaFWd0hrOZEfsxY00h3ilxR3G\nJsofR3PZBKYI11VGruNemCgjiJg5hcoJDxLXUgcfpKJaiPeHutczCeZ1RANQwQF1\n/dPXhqbtWiaS/iu5so64P54TsrVX5DcXmbGr6hQAReIcI6cjA8QhSu6QBtdvEPhv\n27uTuSu4XRtTh3djVGFzFV9pamGZeGELkTiHVSDI8IkQ32s8yuP5Zys/4bFJk7nn\niqJpe0/DAgMBAAECggEAEexQnPWKLx4/H3o8JRBvXGs2DwmYzY7RAukaqzXVMMgJ\nKKoBBv4Biyquk1cIkOD8LLKHUBWKCWiGOOCaFMyMqo5zUFDYCqPwxCHOQ/ki9VvZ\nHXJ4Fv6Su1rqxwQPVZ03ldWFfSspYMgFa9Z47J54iOasgES/og1mZrOldWMUsoBu\nCKf0fH+vIsxWPwmRtyxKCMwqenqdc22nGGLhmpm8tuw1eQp6XtTXagqkPtAVMMga\nmgC0EGqhZA/IklGW1JuGWELjXVMgS/tLIPq+hYsmY14y6Ie032YoSMWkz6Z5p7i0\n/JwCzVZNO1mD0MwVj7nDmokXOpoyM7Qcbx8r1E4Q4QKBgQDxqAZ6D+A671mCNU0J\n6Qzc3cOZq7MBj4y7M/2qPXHC7i/DdbmnM7PPPriaBBch2nX7jZRlRmVDBsmrC4OG\n3m5+HAx7YPVbefwe6h5ki5O9wg1pLcgYY9uvgLSlD85lVZKAzO7QK2W5zfM19kPD\nSckIa+U7DKFbwKhtCsxcP6ARJwKBgQDn+zAPHepGP2Zf78pOLlVXcQqVA1uOu+bW\nrG4G+lPrytB0C4QdwWdBV3Lcqmnw/ae5PkQBs0dCbtWG8+MT8gA6k5kleflaZrAY\ngdUJIUP6J7ocWYxVTfqGFyFF1n5VT8/jbVucaT7izBZfZvlGyf7Vz7ewQzgWQWlK\nCQ0qstV2BQKBgQCajAQAYlDcQCC1dlMbqHDye91RVQ65S84MF1b+Xid4LA5d6dde\nyGERhKJY1Y7ZtrZHt6cVEe1G7XtiKY3nXi+59URCT6L66svEFaR0VxOYgxdCkeXr\nO0nPNvfQrIgqJIz6VJXSij6XktAdTa7OoUyxVxeWKSC05kSQ4BwMTyCWdwKBgQCW\noqlmZ4qE6w5TJaY8diG8kg7JDFEbsjAHHhikN1DfP+d0MzYrDDc8WsifOZlpf4y1\n4RTP9dZD8Sx+YUgG35H+d3FuwHGGnj+i6kunjg5SFhHn7s4NZoFTKRnV+541T4oy\nqARg4IaRRu0QLhGYQfpUZHlm339AFGGGTbJbE51A8QKBgQDTEN5O+3bRG3Fa1J6z\nU9PMrjjs6l8xhXFso10YEYG5KRnfhzCFujyWNiLE6WrlUL8invVBaCxsZr51GDgA\nhyEEdm4kXCRrv4JyhOvIuGxNcAIiQK/e91UQEM6u1t6hUI1rE7ZOyJQzBxj9hFlV\n7OvhBlHXQUtAOdq0XLHr9GzdSA==\n-----END PRIVATE KEY-----\n", - "client_email": "some-project-name@appspot.gserviceaccount.com", - "client_id": "123456789101112130594", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://accounts.google.com/o/oauth2/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/some-project-name%40appspot.gserviceaccount.com" -}