From 3bee2d319565298a58ea07849372701d72a75db5 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Thu, 28 Jan 2016 14:53:32 +0100 Subject: [PATCH] Migrate Azure settings to new settings infrastructure With this commit we migrate all Azure related settings to the new settings infrastructure. --- .../cloud/azure/AzureDiscoveryModule.java | 34 +++++----- .../azure/management/AzureComputeService.java | 25 +++---- .../management/AzureComputeServiceImpl.java | 23 ++----- .../AzureComputeSettingsFilter.java | 16 ++--- .../AbstractAzureComputeServiceTestCase.java | 8 +-- .../discovery/azure/AzureSimpleTests.java | 6 +- .../azure/AzureTwoStartedNodesTests.java | 4 +- .../cloud/azure/blobstore/AzureBlobStore.java | 18 +++-- .../azure/storage/AzureStorageService.java | 20 +++--- .../storage/AzureStorageServiceImpl.java | 9 +-- .../azure/storage/AzureStorageSettings.java | 50 +++++++------- .../storage/AzureStorageSettingsFilter.java | 2 +- .../repositories/azure/AzureRepository.java | 65 ++++++++----------- ...bstractAzureRepositoryServiceTestCase.java | 2 +- ...est.java => AzureStorageServiceTests.java} | 12 ++-- ...a => AzureStorageSettingsFilterTests.java} | 2 +- ...est.java => AzureSettingsParserTests.java} | 2 +- .../azure/AzureSnapshotRestoreTests.java | 46 ++++++------- 18 files changed, 155 insertions(+), 189 deletions(-) rename plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/{AzureStorageServiceTest.java => AzureStorageServiceTests.java} (93%) rename plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/{AzureStorageSettingsFilterTest.java => AzureStorageSettingsFilterTests.java} (97%) rename plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/{AzureSettingsParserTest.java => AzureSettingsParserTests.java} (98%) diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java index d48eed9e507..5a9b9f5f412 100644 --- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java +++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.discovery.azure.AzureDiscovery; @@ -79,29 +80,24 @@ public class AzureDiscoveryModule extends AbstractModule { return false; } - if (isPropertyMissing(settings, Management.SUBSCRIPTION_ID) || - isPropertyMissing(settings, Management.SERVICE_NAME) || - isPropertyMissing(settings, Management.KEYSTORE_PATH) || - isPropertyMissing(settings, Management.KEYSTORE_PASSWORD) - ) { - logger.debug("one or more azure discovery settings are missing. " + + if (isDefined(settings, Management.SUBSCRIPTION_ID_SETTING) && + isDefined(settings, Management.SERVICE_NAME_SETTING) && + isDefined(settings, Management.KEYSTORE_PATH_SETTING) && + isDefined(settings, Management.KEYSTORE_PASSWORD_SETTING)) { + logger.trace("All required properties for Azure discovery are set!"); + return true; + } else { + logger.debug("One or more Azure discovery settings are missing. " + "Check elasticsearch.yml file. Should have [{}], [{}], [{}] and [{}].", - Management.SUBSCRIPTION_ID, - Management.SERVICE_NAME, - Management.KEYSTORE_PATH, - Management.KEYSTORE_PASSWORD); + Management.SUBSCRIPTION_ID_SETTING.getKey(), + Management.SERVICE_NAME_SETTING.getKey(), + Management.KEYSTORE_PATH_SETTING.getKey(), + Management.KEYSTORE_PASSWORD_SETTING.getKey()); return false; } - - logger.trace("all required properties for azure discovery are set!"); - - return true; } - public static boolean isPropertyMissing(Settings settings, String name) throws ElasticsearchException { - if (!Strings.hasText(settings.get(name))) { - return true; - } - return false; + private static boolean isDefined(Settings settings, Setting property) throws ElasticsearchException { + return (property.exists(settings) && Strings.hasText(property.get(settings))); } } diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java index de2343d9d87..0c665c138b8 100644 --- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java +++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java @@ -19,31 +19,25 @@ package org.elasticsearch.cloud.azure.management; +import com.microsoft.windowsazure.core.utils.KeyStoreType; import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDetailedResponse; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.azure.AzureUnicastHostsProvider; -import java.util.Locale; - -/** - * - */ public interface AzureComputeService { - static public final class Management { - public static final String API_IMPLEMENTATION = "cloud.azure.management.api.impl"; - - public static final String SUBSCRIPTION_ID = "cloud.azure.management.subscription.id"; - public static final String SERVICE_NAME = "cloud.azure.management.cloud.service.name"; + final class Management { + public static final Setting SUBSCRIPTION_ID_SETTING = Setting.simpleString("cloud.azure.management.subscription.id", false, Setting.Scope.CLUSTER); + public static final Setting SERVICE_NAME_SETTING = Setting.simpleString("cloud.azure.management.cloud.service.name", false, Setting.Scope.CLUSTER); // Keystore settings - public static final String KEYSTORE_PATH = "cloud.azure.management.keystore.path"; - public static final String KEYSTORE_PASSWORD = "cloud.azure.management.keystore.password"; - public static final String KEYSTORE_TYPE = "cloud.azure.management.keystore.type"; + public static final Setting KEYSTORE_PATH_SETTING = Setting.simpleString("cloud.azure.management.keystore.path", false, Setting.Scope.CLUSTER); + public static final Setting KEYSTORE_PASSWORD_SETTING = Setting.simpleString("cloud.azure.management.keystore.password", false, Setting.Scope.CLUSTER); + public static final Setting KEYSTORE_TYPE_SETTING = new Setting<>("cloud.azure.management.keystore.type", KeyStoreType.pkcs12.name(), KeyStoreType::fromString, false, Setting.Scope.CLUSTER); } - static public final class Discovery { + final class Discovery { public static final Setting REFRESH_SETTING = Setting.positiveTimeSetting("discovery.azure.refresh_interval", TimeValue.timeValueSeconds(0), false, Setting.Scope.CLUSTER); public static final Setting HOST_TYPE_SETTING = new Setting<>("discovery.azure.host.type", @@ -53,5 +47,6 @@ public interface AzureComputeService { public static final String DEPLOYMENT_NAME = "discovery.azure.deployment.name"; public static final String DEPLOYMENT_SLOT = "discovery.azure.deployment.slot"; } - public HostedServiceGetDetailedResponse getServiceDetails(); + + HostedServiceGetDetailedResponse getServiceDetails(); } diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java index 39221ee6904..04b4f32ea92 100644 --- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java +++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java @@ -36,11 +36,6 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.KEYSTORE_PASSWORD; -import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.KEYSTORE_PATH; -import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.KEYSTORE_TYPE; -import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.SUBSCRIPTION_ID; - /** * */ @@ -57,20 +52,12 @@ public class AzureComputeServiceImpl extends AbstractLifecycleComponent start one node"); @@ -53,7 +53,7 @@ public class AzureSimpleTests extends AbstractAzureComputeServiceTestCase { public void testOneNodeShouldRunUsingPublicIp() { Settings.Builder settings = Settings.settingsBuilder() - .put(Management.SERVICE_NAME, "dummy") + .put(Management.SERVICE_NAME_SETTING.getKey(), "dummy") .put(Discovery.HOST_TYPE_SETTING.getKey(), "public_ip"); logger.info("--> start one node"); @@ -66,7 +66,7 @@ public class AzureSimpleTests extends AbstractAzureComputeServiceTestCase { public void testOneNodeShouldRunUsingWrongSettings() { Settings.Builder settings = Settings.settingsBuilder() - .put(Management.SERVICE_NAME, "dummy") + .put(Management.SERVICE_NAME_SETTING.getKey(), "dummy") .put(Discovery.HOST_TYPE_SETTING.getKey(), "do_not_exist"); logger.info("--> start one node"); diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTests.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTests.java index 880c05ed121..bb15ad050f0 100644 --- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTests.java +++ b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTests.java @@ -41,7 +41,7 @@ public class AzureTwoStartedNodesTests extends AbstractAzureComputeServiceTestCa @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/11533") public void testTwoNodesShouldRunUsingPrivateIp() { Settings.Builder settings = Settings.settingsBuilder() - .put(Management.SERVICE_NAME, "dummy") + .put(Management.SERVICE_NAME_SETTING.getKey(), "dummy") .put(Discovery.HOST_TYPE_SETTING.getKey(), "private_ip"); logger.info("--> start first node"); @@ -59,7 +59,7 @@ public class AzureTwoStartedNodesTests extends AbstractAzureComputeServiceTestCa @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/11533") public void testTwoNodesShouldRunUsingPublicIp() { Settings.Builder settings = Settings.settingsBuilder() - .put(Management.SERVICE_NAME, "dummy") + .put(Management.SERVICE_NAME_SETTING.getKey(), "dummy") .put(Discovery.HOST_TYPE_SETTING.getKey(), "public_ip"); logger.info("--> start first node"); diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java index cf97008249f..85baf00b909 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java @@ -23,6 +23,7 @@ import com.microsoft.azure.storage.LocationMode; import com.microsoft.azure.storage.StorageException; import org.elasticsearch.cloud.azure.storage.AzureStorageService; import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.BlobPath; @@ -32,7 +33,6 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.repositories.RepositoryName; import org.elasticsearch.repositories.RepositorySettings; -import org.elasticsearch.repositories.azure.AzureRepository.Defaults; import java.io.InputStream; import java.io.OutputStream; @@ -40,7 +40,7 @@ import java.net.URISyntaxException; import java.util.Locale; import java.util.Map; -import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepositorySettings; +import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getValue; import static org.elasticsearch.repositories.azure.AzureRepository.Repository; public class AzureBlobStore extends AbstractComponent implements BlobStore { @@ -57,17 +57,15 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore { AzureStorageService client) throws URISyntaxException, StorageException { super(settings); this.client = client.start(); - this.container = getRepositorySettings(repositorySettings, Repository.CONTAINER, Storage.CONTAINER, Defaults.CONTAINER); + this.container = getValue(repositorySettings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING); this.repositoryName = name.getName(); + this.accountName = getValue(repositorySettings, Repository.ACCOUNT_SETTING, Storage.ACCOUNT_SETTING); - // NOTE: null account means to use the first one specified in config - this.accountName = getRepositorySettings(repositorySettings, Repository.ACCOUNT, Storage.ACCOUNT, null); - - String modeStr = getRepositorySettings(repositorySettings, Repository.LOCATION_MODE, Storage.LOCATION_MODE, null); - if (modeStr == null) { - this.locMode = LocationMode.PRIMARY_ONLY; - } else { + String modeStr = getValue(repositorySettings, Repository.LOCATION_MODE_SETTING, Storage.LOCATION_MODE_SETTING); + if (Strings.hasLength(modeStr)) { this.locMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT)); + } else { + this.locMode = LocationMode.PRIMARY_ONLY; } } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java index c154f78eeb5..657c292db31 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java @@ -23,11 +23,15 @@ import com.microsoft.azure.storage.LocationMode; import com.microsoft.azure.storage.StorageException; import org.elasticsearch.common.blobstore.BlobMetaData; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; import java.io.InputStream; import java.io.OutputStream; import java.net.URISyntaxException; import java.util.Map; +import java.util.function.Function; /** * Azure Storage Service interface @@ -37,15 +41,13 @@ public interface AzureStorageService { final class Storage { public static final String PREFIX = "cloud.azure.storage."; - - public static final String TIMEOUT = "cloud.azure.storage.timeout"; - - public static final String ACCOUNT = "repositories.azure.account"; - public static final String LOCATION_MODE = "repositories.azure.location_mode"; - public static final String CONTAINER = "repositories.azure.container"; - public static final String BASE_PATH = "repositories.azure.base_path"; - public static final String CHUNK_SIZE = "repositories.azure.chunk_size"; - public static final String COMPRESS = "repositories.azure.compress"; + public static final Setting TIMEOUT_SETTING = Setting.timeSetting("cloud.azure.storage.timeout", TimeValue.timeValueMinutes(5), false, Setting.Scope.CLUSTER); + public static final Setting ACCOUNT_SETTING = Setting.simpleString("repositories.azure.account", false, Setting.Scope.CLUSTER); + public static final Setting CONTAINER_SETTING = Setting.simpleString("repositories.azure.container", false, Setting.Scope.CLUSTER); + public static final Setting BASE_PATH_SETTING = Setting.simpleString("repositories.azure.base_path", false, Setting.Scope.CLUSTER); + public static final Setting LOCATION_MODE_SETTING = Setting.simpleString("repositories.azure.location_mode", false, Setting.Scope.CLUSTER); + public static final Setting CHUNK_SIZE_SETTING = Setting.byteSizeSetting("repositories.azure.chunk_size", new ByteSizeValue(-1), false, Setting.Scope.CLUSTER); + public static final Setting COMPRESS_SETTING = Setting.boolSetting("repositories.azure.compress", false, false, Setting.Scope.CLUSTER); } boolean doesContainerExist(String account, LocationMode mode, String container); diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java index 3cecd810a95..74ba008c8ec 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java @@ -28,6 +28,7 @@ import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlockBlob; import com.microsoft.azure.storage.blob.ListBlobItem; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; import org.elasticsearch.common.collect.MapBuilder; @@ -41,7 +42,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; -import java.util.Hashtable; +import java.util.HashMap; import java.util.Map; public class AzureStorageServiceImpl extends AbstractLifecycleComponent @@ -60,7 +61,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent(); + this.clients = new HashMap<>(); } void createClient(AzureStorageSettings azureStorageSettings) { @@ -94,13 +95,13 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent secondaryStorage = new HashMap<>(); - TimeValue globalTimeout = settings.getAsTime(Storage.TIMEOUT, TimeValue.timeValueMinutes(5)); + TimeValue globalTimeout = Storage.TIMEOUT_SETTING.get(settings); Settings storageSettings = settings.getByPrefix(Storage.PREFIX); if (storageSettings != null) { @@ -124,27 +124,23 @@ public class AzureStorageSettings { return Tuple.tuple(primaryStorage, secondaryStorage); } - public static String getRepositorySettings(RepositorySettings repositorySettings, - String repositorySettingName, - String repositoriesSettingName, - String defaultValue) { - return repositorySettings.settings().get(repositorySettingName, - repositorySettings.globalSettings().get(repositoriesSettingName, defaultValue)); + public static T getValue(RepositorySettings repositorySettings, + Setting repositorySetting, + Setting repositoriesSetting) { + if (repositorySetting.exists(repositorySettings.settings())) { + return repositorySetting.get(repositorySettings.settings()); + } else { + return repositoriesSetting.get(repositorySettings.globalSettings()); + } } - public static ByteSizeValue getRepositorySettingsAsBytesSize(RepositorySettings repositorySettings, - String repositorySettingName, - String repositoriesSettingName, - ByteSizeValue defaultValue) { - return repositorySettings.settings().getAsBytesSize(repositorySettingName, - repositorySettings.globalSettings().getAsBytesSize(repositoriesSettingName, defaultValue)); - } - - public static Boolean getRepositorySettingsAsBoolean(RepositorySettings repositorySettings, - String repositorySettingName, - String repositoriesSettingName, - Boolean defaultValue) { - return repositorySettings.settings().getAsBoolean(repositorySettingName, - repositorySettings.globalSettings().getAsBoolean(repositoriesSettingName, defaultValue)); + public static Setting getEffectiveSetting(RepositorySettings repositorySettings, + Setting repositorySetting, + Setting repositoriesSetting) { + if (repositorySetting.exists(repositorySettings.settings())) { + return repositorySetting; + } else { + return repositoriesSetting; + } } } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilter.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilter.java index 2c4e7957af3..76ac68bd436 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilter.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilter.java @@ -33,6 +33,6 @@ public class AzureStorageSettingsFilter extends AbstractComponent { // Cloud storage API settings needed to be hidden settingsFilter.addFilter(Storage.PREFIX + "*.account"); settingsFilter.addFilter(Storage.PREFIX + "*.key"); - settingsFilter.addFilter(Storage.ACCOUNT); + settingsFilter.addFilter(Storage.ACCOUNT_SETTING.getKey()); } } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index a3abf9b4adf..f2773bccbbd 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -29,6 +29,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.snapshots.IndexShardRepository; @@ -42,10 +44,10 @@ import java.io.IOException; import java.net.URISyntaxException; import java.util.List; import java.util.Locale; +import java.util.function.Function; -import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepositorySettings; -import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepositorySettingsAsBoolean; -import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepositorySettingsAsBytesSize; +import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getEffectiveSetting; +import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getValue; /** * Azure file system implementation of the BlobStoreRepository @@ -60,31 +62,23 @@ import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepo */ public class AzureRepository extends BlobStoreRepository { + private static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB); + public final static String TYPE = "azure"; - static public final class Defaults { - public static final String CONTAINER = "elasticsearch-snapshots"; - public static final ByteSizeValue CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB); - public static final Boolean COMPRESS = false; - } - - - static public final class Repository { - public static final String ACCOUNT = "account"; - public static final String LOCATION_MODE = "location_mode"; - public static final String CONTAINER = "container"; - public static final String CHUNK_SIZE = "chunk_size"; - public static final String COMPRESS = "compress"; - public static final String BASE_PATH = "base_path"; + public static final class Repository { + public static final Setting ACCOUNT_SETTING = Setting.simpleString("account", false, Setting.Scope.CLUSTER); + public static final Setting CONTAINER_SETTING = new Setting<>("container", "elasticsearch-snapshots", Function.identity(), false, Setting.Scope.CLUSTER); + public static final Setting BASE_PATH_SETTING = Setting.simpleString("base_path", false, Setting.Scope.CLUSTER); + public static final Setting LOCATION_MODE_SETTING = Setting.simpleString("location_mode", false, Setting.Scope.CLUSTER); + public static final Setting CHUNK_SIZE_SETTING = Setting.byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, false, Setting.Scope.CLUSTER); + public static final Setting COMPRESS_SETTING = Setting.boolSetting("compress", false, false, Setting.Scope.CLUSTER); } private final AzureBlobStore blobStore; - private final BlobPath basePath; - - private ByteSizeValue chunkSize; - - private boolean compress; + private final ByteSizeValue chunkSize; + private final boolean compress; private final boolean readonly; @Inject @@ -93,30 +87,27 @@ public class AzureRepository extends BlobStoreRepository { AzureBlobStore azureBlobStore) throws IOException, URISyntaxException, StorageException { super(name.getName(), repositorySettings, indexShardRepository); - String container = getRepositorySettings(repositorySettings, Repository.CONTAINER, Storage.CONTAINER, Defaults.CONTAINER); + String container = getValue(repositorySettings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING); this.blobStore = azureBlobStore; - this.chunkSize = getRepositorySettingsAsBytesSize(repositorySettings, Repository.CHUNK_SIZE, Storage.CHUNK_SIZE, Defaults.CHUNK_SIZE); - - if (this.chunkSize.getMb() > 64) { - logger.warn("azure repository does not support yet size > 64mb. Fall back to 64mb."); - this.chunkSize = new ByteSizeValue(64, ByteSizeUnit.MB); + ByteSizeValue configuredChunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING); + if (configuredChunkSize.getMb() > MAX_CHUNK_SIZE.getMb()) { + Setting setting = getEffectiveSetting(repositorySettings, Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING); + throw new SettingsException("[" + setting.getKey() + "] must not exceed [" + MAX_CHUNK_SIZE + "] but is set to [" + configuredChunkSize + "]."); + } else { + this.chunkSize = configuredChunkSize; } - this.compress = getRepositorySettingsAsBoolean(repositorySettings, Repository.COMPRESS, Storage.COMPRESS, Defaults.COMPRESS); - String modeStr = getRepositorySettings(repositorySettings, Repository.LOCATION_MODE, Storage.LOCATION_MODE, null); - if (modeStr != null) { + this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Storage.COMPRESS_SETTING); + String modeStr = getValue(repositorySettings, Repository.LOCATION_MODE_SETTING, Storage.LOCATION_MODE_SETTING); + if (Strings.hasLength(modeStr)) { LocationMode locationMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT)); - if (locationMode == LocationMode.SECONDARY_ONLY) { - readonly = true; - } else { - readonly = false; - } + readonly = locationMode == LocationMode.SECONDARY_ONLY; } else { readonly = false; } - String basePath = getRepositorySettings(repositorySettings, Repository.BASE_PATH, Storage.BASE_PATH, null); + String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Storage.BASE_PATH_SETTING); if (Strings.hasLength(basePath)) { // Remove starting / if any diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java index 73aa7e3921c..d8bca609ce3 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java @@ -80,7 +80,7 @@ public abstract class AbstractAzureRepositoryServiceTestCase extends AbstractAzu @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.settingsBuilder() - .put(Storage.CONTAINER, "snapshots"); + .put(Storage.CONTAINER_SETTING.getKey(), "snapshots"); return builder.build(); } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTest.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java similarity index 93% rename from plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTest.java rename to plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java index 93683d9d014..5fc4937ea92 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTest.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java @@ -28,7 +28,7 @@ import java.net.URI; import static org.hamcrest.Matchers.is; -public class AzureStorageServiceTest extends ESTestCase { +public class AzureStorageServiceTests extends ESTestCase { final static Settings settings = Settings.builder() .put("cloud.azure.storage.azure1.account", "myaccount1") .put("cloud.azure.storage.azure1.key", "mykey1") @@ -120,24 +120,24 @@ public class AzureStorageServiceTest extends ESTestCase { public void testGetSelectedClientGlobalTimeout() { Settings timeoutSettings = Settings.builder() .put(settings) - .put("cloud.azure.storage.timeout", "10s") + .put(AzureStorageService.Storage.TIMEOUT_SETTING.getKey(), "10s") .build(); AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings); azureStorageService.doStart(); CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY); - assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(10 * 1000)); + assertThat(client1.getDefaultRequestOptions().getMaximumExecutionTimeInMs(), is(10 * 1000)); CloudBlobClient client3 = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY); - assertThat(client3.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(30 * 1000)); + assertThat(client3.getDefaultRequestOptions().getMaximumExecutionTimeInMs(), is(30 * 1000)); } public void testGetSelectedClientDefaultTimeout() { AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings); azureStorageService.doStart(); CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY); - assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(5 * 60 * 1000)); + assertThat(client1.getDefaultRequestOptions().getMaximumExecutionTimeInMs(), is(5 * 60 * 1000)); CloudBlobClient client3 = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY); - assertThat(client3.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(30 * 1000)); + assertThat(client3.getDefaultRequestOptions().getMaximumExecutionTimeInMs(), is(30 * 1000)); } /** diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTest.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java similarity index 97% rename from plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTest.java rename to plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java index eaaf9c224d8..6e36b27cebe 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTest.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java @@ -31,7 +31,7 @@ import java.io.IOException; import static org.hamcrest.Matchers.contains; -public class AzureStorageSettingsFilterTest extends ESTestCase { +public class AzureStorageSettingsFilterTests extends ESTestCase { final static Settings settings = Settings.builder() .put("cloud.azure.storage.azure1.account", "myaccount1") .put("cloud.azure.storage.azure1.key", "mykey1") diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTest.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java similarity index 98% rename from plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTest.java rename to plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java index aec8506ca6d..5347be09da0 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTest.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java @@ -31,7 +31,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class AzureSettingsParserTest extends LuceneTestCase { +public class AzureSettingsParserTests extends LuceneTestCase { public void testParseTwoSettingsExplicitDefault() { Settings settings = Settings.builder() diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java index 1818a5e6252..7f6f3106fc3 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java @@ -106,9 +106,9 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa logger.info("--> creating azure repository with path [{}]", getRepositoryPath()); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") .setType("azure").setSettings(Settings.settingsBuilder() - .put(Repository.CONTAINER, getContainerName()) - .put(Repository.BASE_PATH, getRepositoryPath()) - .put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) + .put(Repository.CONTAINER_SETTING.getKey(), getContainerName()) + .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath()) + .put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) ).get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); @@ -197,9 +197,9 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa logger.info("creating Azure repository with path [{}]", getRepositoryPath()); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository(repositoryName) .setType("azure").setSettings(Settings.settingsBuilder() - .put(Repository.CONTAINER, getContainerName()) - .put(Repository.BASE_PATH, getRepositoryPath()) - .put(Repository.BASE_PATH, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) + .put(Repository.CONTAINER_SETTING.getKey(), getContainerName()) + .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath()) + .put(Repository.BASE_PATH_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) ).get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); @@ -237,16 +237,16 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa logger.info("--> creating azure repository with path [{}]", getRepositoryPath()); PutRepositoryResponse putRepositoryResponse1 = client.admin().cluster().preparePutRepository("test-repo1") .setType("azure").setSettings(Settings.settingsBuilder() - .put(Repository.CONTAINER, getContainerName().concat("-1")) - .put(Repository.BASE_PATH, getRepositoryPath()) - .put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) + .put(Repository.CONTAINER_SETTING.getKey(), getContainerName().concat("-1")) + .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath()) + .put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) ).get(); assertThat(putRepositoryResponse1.isAcknowledged(), equalTo(true)); PutRepositoryResponse putRepositoryResponse2 = client.admin().cluster().preparePutRepository("test-repo2") .setType("azure").setSettings(Settings.settingsBuilder() - .put(Repository.CONTAINER, getContainerName().concat("-2")) - .put(Repository.BASE_PATH, getRepositoryPath()) - .put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) + .put(Repository.CONTAINER_SETTING.getKey(), getContainerName().concat("-2")) + .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath()) + .put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) ).get(); assertThat(putRepositoryResponse2.isAcknowledged(), equalTo(true)); @@ -316,7 +316,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa logger.info("--> creating azure repository without any path"); PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure") .setSettings(Settings.settingsBuilder() - .put(Repository.CONTAINER, getContainerName()) + .put(Repository.CONTAINER_SETTING.getKey(), getContainerName()) ).get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); @@ -337,8 +337,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa logger.info("--> creating azure repository path [{}]", getRepositoryPath()); putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure") .setSettings(Settings.settingsBuilder() - .put(Repository.CONTAINER, getContainerName()) - .put(Repository.BASE_PATH, getRepositoryPath()) + .put(Repository.CONTAINER_SETTING.getKey(), getContainerName()) + .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath()) ).get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); @@ -363,7 +363,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa logger.info("--> creating azure repository without any path"); PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure") .setSettings(Settings.settingsBuilder() - .put(Repository.CONTAINER, getContainerName()) + .put(Repository.CONTAINER_SETTING.getKey(), getContainerName()) ).get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); @@ -414,9 +414,9 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa try { PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo") .setType("azure").setSettings(Settings.settingsBuilder() - .put(Repository.CONTAINER, container) - .put(Repository.BASE_PATH, getRepositoryPath()) - .put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) + .put(Repository.CONTAINER_SETTING.getKey(), container) + .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath()) + .put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) ).get(); client().admin().cluster().prepareDeleteRepository("test-repo").get(); try { @@ -444,9 +444,9 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa logger.info("--> creating azure repository with path [{}]", getRepositoryPath()); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") .setType("azure").setSettings(Settings.settingsBuilder() - .put(Repository.CONTAINER, getContainerName()) - .put(Repository.BASE_PATH, getRepositoryPath()) - .put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) + .put(Repository.CONTAINER_SETTING.getKey(), getContainerName()) + .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath()) + .put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) ).get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); @@ -492,7 +492,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa try { client.preparePutRepository("test-repo").setType("azure") .setSettings(Settings.settingsBuilder() - .put(Repository.CONTAINER, container) + .put(Repository.CONTAINER_SETTING.getKey(), container) ).get(); fail("we should get a RepositoryVerificationException"); } catch (RepositoryVerificationException e) {