honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Migrate Azure settings to new settings infrastructure 

With this commit we migrate all Azure related settings to
the new settings infrastructure.
This commit is contained in:
Daniel Mitterdorfer 2016-01-28 14:53:32 +01:00
parent 6f12048cda
commit 3bee2d3195
18 changed files with 155 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.discovery.DiscoveryModule;
import org.elasticsearch.discovery.azure.AzureDiscovery;
@ -79,29 +80,24 @@ public class AzureDiscoveryModule extends AbstractModule {
return false;
}
if (isPropertyMissing(settings, Management.SUBSCRIPTION_ID) ||
isPropertyMissing(settings, Management.SERVICE_NAME) ||
isPropertyMissing(settings, Management.KEYSTORE_PATH) ||
isPropertyMissing(settings, Management.KEYSTORE_PASSWORD)
) {
logger.debug("one or more azure discovery settings are missing. " +
if (isDefined(settings, Management.SUBSCRIPTION_ID_SETTING) &&
isDefined(settings, Management.SERVICE_NAME_SETTING) &&
isDefined(settings, Management.KEYSTORE_PATH_SETTING) &&
isDefined(settings, Management.KEYSTORE_PASSWORD_SETTING)) {
logger.trace("All required properties for Azure discovery are set!");
return true;
} else {
logger.debug("One or more Azure discovery settings are missing. " +
"Check elasticsearch.yml file. Should have [{}], [{}], [{}] and [{}].",
Management.SUBSCRIPTION_ID,
Management.SERVICE_NAME,
Management.KEYSTORE_PATH,
Management.KEYSTORE_PASSWORD);
Management.SUBSCRIPTION_ID_SETTING.getKey(),
Management.SERVICE_NAME_SETTING.getKey(),
Management.KEYSTORE_PATH_SETTING.getKey(),
Management.KEYSTORE_PASSWORD_SETTING.getKey());
return false;
}
logger.trace("all required properties for azure discovery are set!");
return true;
}
public static boolean isPropertyMissing(Settings settings, String name) throws ElasticsearchException {
if (!Strings.hasText(settings.get(name))) {
return true;
}
return false;
private static boolean isDefined(Settings settings, Setting<String> property) throws ElasticsearchException {
return (property.exists(settings) && Strings.hasText(property.get(settings)));
}
}

25
plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java
View File

@ -19,31 +19,25 @@
package org.elasticsearch.cloud.azure.management;
import com.microsoft.windowsazure.core.utils.KeyStoreType;
import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDetailedResponse;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.discovery.azure.AzureUnicastHostsProvider;
import java.util.Locale;
/**
*
*/
public interface AzureComputeService {
static public final class Management {
public static final String API_IMPLEMENTATION = "cloud.azure.management.api.impl";
public static final String SUBSCRIPTION_ID = "cloud.azure.management.subscription.id";
public static final String SERVICE_NAME = "cloud.azure.management.cloud.service.name";
final class Management {
public static final Setting<String> SUBSCRIPTION_ID_SETTING = Setting.simpleString("cloud.azure.management.subscription.id", false, Setting.Scope.CLUSTER);
public static final Setting<String> SERVICE_NAME_SETTING = Setting.simpleString("cloud.azure.management.cloud.service.name", false, Setting.Scope.CLUSTER);
// Keystore settings
public static final String KEYSTORE_PATH = "cloud.azure.management.keystore.path";
public static final String KEYSTORE_PASSWORD = "cloud.azure.management.keystore.password";
public static final String KEYSTORE_TYPE = "cloud.azure.management.keystore.type";
public static final Setting<String> KEYSTORE_PATH_SETTING = Setting.simpleString("cloud.azure.management.keystore.path", false, Setting.Scope.CLUSTER);
public static final Setting<String> KEYSTORE_PASSWORD_SETTING = Setting.simpleString("cloud.azure.management.keystore.password", false, Setting.Scope.CLUSTER);
public static final Setting<KeyStoreType> KEYSTORE_TYPE_SETTING = new Setting<>("cloud.azure.management.keystore.type", KeyStoreType.pkcs12.name(), KeyStoreType::fromString, false, Setting.Scope.CLUSTER);
}
static public final class Discovery {
final class Discovery {
public static final Setting<TimeValue> REFRESH_SETTING = Setting.positiveTimeSetting("discovery.azure.refresh_interval", TimeValue.timeValueSeconds(0), false, Setting.Scope.CLUSTER);
public static final Setting<AzureUnicastHostsProvider.HostType> HOST_TYPE_SETTING = new Setting<>("discovery.azure.host.type",
@ -53,5 +47,6 @@ public interface AzureComputeService {
public static final String DEPLOYMENT_NAME = "discovery.azure.deployment.name";
public static final String DEPLOYMENT_SLOT = "discovery.azure.deployment.slot";
}
public HostedServiceGetDetailedResponse getServiceDetails();
HostedServiceGetDetailedResponse getServiceDetails();
}

23
plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java
View File

@ -36,11 +36,6 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.KEYSTORE_PASSWORD;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.KEYSTORE_PATH;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.KEYSTORE_TYPE;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.SUBSCRIPTION_ID;
/**
*
*/
@ -57,20 +52,12 @@ public class AzureComputeServiceImpl extends AbstractLifecycleComponent<AzureCom
@Inject
public AzureComputeServiceImpl(Settings settings) {
super(settings);
String subscriptionId = settings.get(SUBSCRIPTION_ID);
String subscriptionId = Management.SUBSCRIPTION_ID_SETTING.get(settings);
serviceName = settings.get(Management.SERVICE_NAME);
String keystorePath = settings.get(KEYSTORE_PATH);
String keystorePassword = settings.get(KEYSTORE_PASSWORD);
String strKeyStoreType = settings.get(KEYSTORE_TYPE, KeyStoreType.pkcs12.name());
KeyStoreType tmpKeyStoreType = KeyStoreType.pkcs12;
try {
tmpKeyStoreType = KeyStoreType.fromString(strKeyStoreType);
} catch (Exception e) {
logger.warn("wrong value for [{}]: [{}]. falling back to [{}]...", KEYSTORE_TYPE,
strKeyStoreType, KeyStoreType.pkcs12.name());
}
KeyStoreType keystoreType = tmpKeyStoreType;
serviceName = Management.SERVICE_NAME_SETTING.get(settings);
String keystorePath = Management.KEYSTORE_PATH_SETTING.get(settings);
String keystorePassword = Management.KEYSTORE_PASSWORD_SETTING.get(settings);
KeyStoreType keystoreType = Management.KEYSTORE_TYPE_SETTING.get(settings);
// Check that we have all needed properties
Configuration configuration;

16
plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeSettingsFilter.java
View File

@ -24,10 +24,10 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.KEYSTORE_PASSWORD;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.KEYSTORE_PATH;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.KEYSTORE_TYPE;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.SUBSCRIPTION_ID;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.KEYSTORE_PASSWORD_SETTING;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.KEYSTORE_PATH_SETTING;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.KEYSTORE_TYPE_SETTING;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.SUBSCRIPTION_ID_SETTING;
public class AzureComputeSettingsFilter extends AbstractComponent {
@ -35,9 +35,9 @@ public class AzureComputeSettingsFilter extends AbstractComponent {
public AzureComputeSettingsFilter(Settings settings, SettingsFilter settingsFilter) {
super(settings);
// Cloud management API settings we need to hide
settingsFilter.addFilter(KEYSTORE_PATH);
settingsFilter.addFilter(KEYSTORE_PASSWORD);
settingsFilter.addFilter(KEYSTORE_TYPE);
settingsFilter.addFilter(SUBSCRIPTION_ID);
settingsFilter.addFilter(KEYSTORE_PATH_SETTING.getKey());
settingsFilter.addFilter(KEYSTORE_PASSWORD_SETTING.getKey());
settingsFilter.addFilter(KEYSTORE_TYPE_SETTING.getKey());
settingsFilter.addFilter(SUBSCRIPTION_ID_SETTING.getKey());
}
}

8
plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureComputeServiceTestCase.java
View File

@ -48,11 +48,11 @@ public abstract class AbstractAzureComputeServiceTestCase extends ESIntegTestCas
.put(Node.NODE_MODE_SETTING.getKey(), "network");
// We add a fake subscription_id to start mock compute service
builder.put(Management.SUBSCRIPTION_ID, "fake")
builder.put(Management.SUBSCRIPTION_ID_SETTING.getKey(), "fake")
.put(Discovery.REFRESH_SETTING.getKey(), "5s")
.put(Management.KEYSTORE_PATH, "dummy")
.put(Management.KEYSTORE_PASSWORD, "dummy")
.put(Management.SERVICE_NAME, "dummy");
.put(Management.KEYSTORE_PATH_SETTING.getKey(), "dummy")
.put(Management.KEYSTORE_PASSWORD_SETTING.getKey(), "dummy")
.put(Management.SERVICE_NAME_SETTING.getKey(), "dummy");
return builder.build();
}

6
plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureSimpleTests.java
View File

@ -40,7 +40,7 @@ public class AzureSimpleTests extends AbstractAzureComputeServiceTestCase {
public void testOneNodeDhouldRunUsingPrivateIp() {
Settings.Builder settings = Settings.settingsBuilder()
.put(Management.SERVICE_NAME, "dummy")
.put(Management.SERVICE_NAME_SETTING.getKey(), "dummy")
.put(Discovery.HOST_TYPE_SETTING.getKey(), "private_ip");
logger.info("--> start one node");
@ -53,7 +53,7 @@ public class AzureSimpleTests extends AbstractAzureComputeServiceTestCase {
public void testOneNodeShouldRunUsingPublicIp() {
Settings.Builder settings = Settings.settingsBuilder()
.put(Management.SERVICE_NAME, "dummy")
.put(Management.SERVICE_NAME_SETTING.getKey(), "dummy")
.put(Discovery.HOST_TYPE_SETTING.getKey(), "public_ip");
logger.info("--> start one node");
@ -66,7 +66,7 @@ public class AzureSimpleTests extends AbstractAzureComputeServiceTestCase {
public void testOneNodeShouldRunUsingWrongSettings() {
Settings.Builder settings = Settings.settingsBuilder()
.put(Management.SERVICE_NAME, "dummy")
.put(Management.SERVICE_NAME_SETTING.getKey(), "dummy")
.put(Discovery.HOST_TYPE_SETTING.getKey(), "do_not_exist");
logger.info("--> start one node");

4
plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTests.java
View File

@ -41,7 +41,7 @@ public class AzureTwoStartedNodesTests extends AbstractAzureComputeServiceTestCa
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/11533")
public void testTwoNodesShouldRunUsingPrivateIp() {
Settings.Builder settings = Settings.settingsBuilder()
.put(Management.SERVICE_NAME, "dummy")
.put(Management.SERVICE_NAME_SETTING.getKey(), "dummy")
.put(Discovery.HOST_TYPE_SETTING.getKey(), "private_ip");
logger.info("--> start first node");
@ -59,7 +59,7 @@ public class AzureTwoStartedNodesTests extends AbstractAzureComputeServiceTestCa
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/11533")
public void testTwoNodesShouldRunUsingPublicIp() {
Settings.Builder settings = Settings.settingsBuilder()
.put(Management.SERVICE_NAME, "dummy")
.put(Management.SERVICE_NAME_SETTING.getKey(), "dummy")
.put(Discovery.HOST_TYPE_SETTING.getKey(), "public_ip");
logger.info("--> start first node");

18
plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
View File

@ -23,6 +23,7 @@ import com.microsoft.azure.storage.LocationMode;
import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.BlobPath;
@ -32,7 +33,6 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.RepositoryName;
import org.elasticsearch.repositories.RepositorySettings;
import org.elasticsearch.repositories.azure.AzureRepository.Defaults;
import java.io.InputStream;
import java.io.OutputStream;
@ -40,7 +40,7 @@ import java.net.URISyntaxException;
import java.util.Locale;
import java.util.Map;
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepositorySettings;
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getValue;
import static org.elasticsearch.repositories.azure.AzureRepository.Repository;
public class AzureBlobStore extends AbstractComponent implements BlobStore {
@ -57,17 +57,15 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore {
AzureStorageService client) throws URISyntaxException, StorageException {
super(settings);
this.client = client.start();
this.container = getRepositorySettings(repositorySettings, Repository.CONTAINER, Storage.CONTAINER, Defaults.CONTAINER);
this.container = getValue(repositorySettings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING);
this.repositoryName = name.getName();
this.accountName = getValue(repositorySettings, Repository.ACCOUNT_SETTING, Storage.ACCOUNT_SETTING);
// NOTE: null account means to use the first one specified in config
this.accountName = getRepositorySettings(repositorySettings, Repository.ACCOUNT, Storage.ACCOUNT, null);
String modeStr = getRepositorySettings(repositorySettings, Repository.LOCATION_MODE, Storage.LOCATION_MODE, null);
if (modeStr == null) {
this.locMode = LocationMode.PRIMARY_ONLY;
} else {
String modeStr = getValue(repositorySettings, Repository.LOCATION_MODE_SETTING, Storage.LOCATION_MODE_SETTING);
if (Strings.hasLength(modeStr)) {
this.locMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT));
} else {
this.locMode = LocationMode.PRIMARY_ONLY;
}
}

20
plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
View File

@ -23,11 +23,15 @@ import com.microsoft.azure.storage.LocationMode;
import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URISyntaxException;
import java.util.Map;
import java.util.function.Function;
/**
* Azure Storage Service interface
@ -37,15 +41,13 @@ public interface AzureStorageService {
final class Storage {
public static final String PREFIX = "cloud.azure.storage.";
public static final String TIMEOUT = "cloud.azure.storage.timeout";
public static final String ACCOUNT = "repositories.azure.account";
public static final String LOCATION_MODE = "repositories.azure.location_mode";
public static final String CONTAINER = "repositories.azure.container";
public static final String BASE_PATH = "repositories.azure.base_path";
public static final String CHUNK_SIZE = "repositories.azure.chunk_size";
public static final String COMPRESS = "repositories.azure.compress";
public static final Setting<TimeValue> TIMEOUT_SETTING = Setting.timeSetting("cloud.azure.storage.timeout", TimeValue.timeValueMinutes(5), false, Setting.Scope.CLUSTER);
public static final Setting<String> ACCOUNT_SETTING = Setting.simpleString("repositories.azure.account", false, Setting.Scope.CLUSTER);
public static final Setting<String> CONTAINER_SETTING = Setting.simpleString("repositories.azure.container", false, Setting.Scope.CLUSTER);
public static final Setting<String> BASE_PATH_SETTING = Setting.simpleString("repositories.azure.base_path", false, Setting.Scope.CLUSTER);
public static final Setting<String> LOCATION_MODE_SETTING = Setting.simpleString("repositories.azure.location_mode", false, Setting.Scope.CLUSTER);
public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING = Setting.byteSizeSetting("repositories.azure.chunk_size", new ByteSizeValue(-1), false, Setting.Scope.CLUSTER);
public static final Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("repositories.azure.compress", false, false, Setting.Scope.CLUSTER);
}
boolean doesContainerExist(String account, LocationMode mode, String container);

9
plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
View File

@ -28,6 +28,7 @@ import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.azure.storage.blob.CloudBlockBlob;
import com.microsoft.azure.storage.blob.ListBlobItem;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
import org.elasticsearch.common.collect.MapBuilder;
@ -41,7 +42,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Hashtable;
import java.util.HashMap;
import java.util.Map;
public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureStorageServiceImpl>
@ -60,7 +61,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
this.primaryStorageSettings = storageSettings.v1();
this.secondariesStorageSettings = storageSettings.v2();
this.clients = new Hashtable<>();
this.clients = new HashMap<>();
}
void createClient(AzureStorageSettings azureStorageSettings) {
@ -94,13 +95,13 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
throw new IllegalArgumentException("No primary azure storage can be found. Check your elasticsearch.yml.");
}
if (account != null) {
if (Strings.hasLength(account)) {
azureStorageSettings = this.secondariesStorageSettings.get(account);
}
// if account is not secondary, it's the primary
if (azureStorageSettings == null) {
if (account == null || primaryStorageSettings.getName() == null || account.equals(primaryStorageSettings.getName())) {
if (Strings.hasLength(account) == false || primaryStorageSettings.getName() == null || account.equals(primaryStorageSettings.getName())) {
azureStorageSettings = primaryStorageSettings;
}
}

48
plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
View File

@ -23,8 +23,8 @@ import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.repositories.RepositorySettings;
@ -34,10 +34,10 @@ import java.util.Map;
public class AzureStorageSettings {
private static ESLogger logger = ESLoggerFactory.getLogger(AzureStorageSettings.class.getName());
private String name;
private String account;
private String key;
private TimeValue timeout;
private final String name;
private final String account;
private final String key;
private final TimeValue timeout;
public AzureStorageSettings(String name, String account, String key, TimeValue timeout) {
this.name = name;
@ -64,7 +64,7 @@ public class AzureStorageSettings {
@Override
public String toString() {
final StringBuffer sb = new StringBuffer("AzureStorageSettings{");
final StringBuilder sb = new StringBuilder("AzureStorageSettings{");
sb.append("name='").append(name).append('\'');
sb.append(", account='").append(account).append('\'');
sb.append(", key='").append(key).append('\'');
@ -82,7 +82,7 @@ public class AzureStorageSettings {
AzureStorageSettings primaryStorage = null;
Map<String, AzureStorageSettings> secondaryStorage = new HashMap<>();
TimeValue globalTimeout = settings.getAsTime(Storage.TIMEOUT, TimeValue.timeValueMinutes(5));
TimeValue globalTimeout = Storage.TIMEOUT_SETTING.get(settings);
Settings storageSettings = settings.getByPrefix(Storage.PREFIX);
if (storageSettings != null) {
@ -124,27 +124,23 @@ public class AzureStorageSettings {
return Tuple.tuple(primaryStorage, secondaryStorage);
}
public static String getRepositorySettings(RepositorySettings repositorySettings,
String repositorySettingName,
String repositoriesSettingName,
String defaultValue) {
return repositorySettings.settings().get(repositorySettingName,
repositorySettings.globalSettings().get(repositoriesSettingName, defaultValue));
public static <T> T getValue(RepositorySettings repositorySettings,
Setting<T> repositorySetting,
Setting<T> repositoriesSetting) {
if (repositorySetting.exists(repositorySettings.settings())) {
return repositorySetting.get(repositorySettings.settings());
} else {
return repositoriesSetting.get(repositorySettings.globalSettings());
}
}
public static ByteSizeValue getRepositorySettingsAsBytesSize(RepositorySettings repositorySettings,
String repositorySettingName,
String repositoriesSettingName,
ByteSizeValue defaultValue) {
return repositorySettings.settings().getAsBytesSize(repositorySettingName,
repositorySettings.globalSettings().getAsBytesSize(repositoriesSettingName, defaultValue));
public static <T> Setting<T> getEffectiveSetting(RepositorySettings repositorySettings,
Setting<T> repositorySetting,
Setting<T> repositoriesSetting) {
if (repositorySetting.exists(repositorySettings.settings())) {
return repositorySetting;
} else {
return repositoriesSetting;
}
public static Boolean getRepositorySettingsAsBoolean(RepositorySettings repositorySettings,
String repositorySettingName,
String repositoriesSettingName,
Boolean defaultValue) {
return repositorySettings.settings().getAsBoolean(repositorySettingName,
repositorySettings.globalSettings().getAsBoolean(repositoriesSettingName, defaultValue));
}
}

2
plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilter.java
View File

@ -33,6 +33,6 @@ public class AzureStorageSettingsFilter extends AbstractComponent {
// Cloud storage API settings needed to be hidden
settingsFilter.addFilter(Storage.PREFIX + "*.account");
settingsFilter.addFilter(Storage.PREFIX + "*.key");
settingsFilter.addFilter(Storage.ACCOUNT);
settingsFilter.addFilter(Storage.ACCOUNT_SETTING.getKey());
}
}

65
plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
View File

@ -29,6 +29,8 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.index.snapshots.IndexShardRepository;
@ -42,10 +44,10 @@ import java.io.IOException;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Locale;
import java.util.function.Function;
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepositorySettings;
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepositorySettingsAsBoolean;
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepositorySettingsAsBytesSize;
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getEffectiveSetting;
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getValue;
/**
* Azure file system implementation of the BlobStoreRepository
@ -60,31 +62,23 @@ import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepo
*/
public class AzureRepository extends BlobStoreRepository {
private static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB);
public final static String TYPE = "azure";
static public final class Defaults {
public static final String CONTAINER = "elasticsearch-snapshots";
public static final ByteSizeValue CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB);
public static final Boolean COMPRESS = false;
}
static public final class Repository {
public static final String ACCOUNT = "account";
public static final String LOCATION_MODE = "location_mode";
public static final String CONTAINER = "container";
public static final String CHUNK_SIZE = "chunk_size";
public static final String COMPRESS = "compress";
public static final String BASE_PATH = "base_path";
public static final class Repository {
public static final Setting<String> ACCOUNT_SETTING = Setting.simpleString("account", false, Setting.Scope.CLUSTER);
public static final Setting<String> CONTAINER_SETTING = new Setting<>("container", "elasticsearch-snapshots", Function.identity(), false, Setting.Scope.CLUSTER);
public static final Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", false, Setting.Scope.CLUSTER);
public static final Setting<String> LOCATION_MODE_SETTING = Setting.simpleString("location_mode", false, Setting.Scope.CLUSTER);
public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING = Setting.byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, false, Setting.Scope.CLUSTER);
public static final Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("compress", false, false, Setting.Scope.CLUSTER);
}
private final AzureBlobStore blobStore;
private final BlobPath basePath;
private ByteSizeValue chunkSize;
private boolean compress;
private final ByteSizeValue chunkSize;
private final boolean compress;
private final boolean readonly;
@Inject
@ -93,30 +87,27 @@ public class AzureRepository extends BlobStoreRepository {
AzureBlobStore azureBlobStore) throws IOException, URISyntaxException, StorageException {
super(name.getName(), repositorySettings, indexShardRepository);
String container = getRepositorySettings(repositorySettings, Repository.CONTAINER, Storage.CONTAINER, Defaults.CONTAINER);
String container = getValue(repositorySettings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING);
this.blobStore = azureBlobStore;
this.chunkSize = getRepositorySettingsAsBytesSize(repositorySettings, Repository.CHUNK_SIZE, Storage.CHUNK_SIZE, Defaults.CHUNK_SIZE);
if (this.chunkSize.getMb() > 64) {
logger.warn("azure repository does not support yet size > 64mb. Fall back to 64mb.");
this.chunkSize = new ByteSizeValue(64, ByteSizeUnit.MB);
ByteSizeValue configuredChunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING);
if (configuredChunkSize.getMb() > MAX_CHUNK_SIZE.getMb()) {
Setting<ByteSizeValue> setting = getEffectiveSetting(repositorySettings, Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING);
throw new SettingsException("[" + setting.getKey() + "] must not exceed [" + MAX_CHUNK_SIZE + "] but is set to [" + configuredChunkSize + "].");
} else {
this.chunkSize = configuredChunkSize;
}
this.compress = getRepositorySettingsAsBoolean(repositorySettings, Repository.COMPRESS, Storage.COMPRESS, Defaults.COMPRESS);
String modeStr = getRepositorySettings(repositorySettings, Repository.LOCATION_MODE, Storage.LOCATION_MODE, null);
if (modeStr != null) {
this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Storage.COMPRESS_SETTING);
String modeStr = getValue(repositorySettings, Repository.LOCATION_MODE_SETTING, Storage.LOCATION_MODE_SETTING);
if (Strings.hasLength(modeStr)) {
LocationMode locationMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT));
if (locationMode == LocationMode.SECONDARY_ONLY) {
readonly = true;
} else {
readonly = false;
}
readonly = locationMode == LocationMode.SECONDARY_ONLY;
} else {
readonly = false;
}
String basePath = getRepositorySettings(repositorySettings, Repository.BASE_PATH, Storage.BASE_PATH, null);
String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Storage.BASE_PATH_SETTING);
if (Strings.hasLength(basePath)) {
// Remove starting / if any

2
plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java
View File

@ -80,7 +80,7 @@ public abstract class AbstractAzureRepositoryServiceTestCase extends AbstractAzu
@Override
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder builder = Settings.settingsBuilder()
.put(Storage.CONTAINER, "snapshots");
.put(Storage.CONTAINER_SETTING.getKey(), "snapshots");
return builder.build();
}

12
plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTest.java → plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
View File

@ -28,7 +28,7 @@ import java.net.URI;
import static org.hamcrest.Matchers.is;
public class AzureStorageServiceTest extends ESTestCase {
public class AzureStorageServiceTests extends ESTestCase {
final static Settings settings = Settings.builder()
.put("cloud.azure.storage.azure1.account", "myaccount1")
.put("cloud.azure.storage.azure1.key", "mykey1")
@ -120,24 +120,24 @@ public class AzureStorageServiceTest extends ESTestCase {
public void testGetSelectedClientGlobalTimeout() {
Settings timeoutSettings = Settings.builder()
.put(settings)
.put("cloud.azure.storage.timeout", "10s")
.put(AzureStorageService.Storage.TIMEOUT_SETTING.getKey(), "10s")
.build();
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings);
azureStorageService.doStart();
CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY);
assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(10 * 1000));
assertThat(client1.getDefaultRequestOptions().getMaximumExecutionTimeInMs(), is(10 * 1000));
CloudBlobClient client3 = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY);
assertThat(client3.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(30 * 1000));
assertThat(client3.getDefaultRequestOptions().getMaximumExecutionTimeInMs(), is(30 * 1000));
}
public void testGetSelectedClientDefaultTimeout() {
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings);
azureStorageService.doStart();
CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY);
assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(5 * 60 * 1000));
assertThat(client1.getDefaultRequestOptions().getMaximumExecutionTimeInMs(), is(5 * 60 * 1000));
CloudBlobClient client3 = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY);
assertThat(client3.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(30 * 1000));
assertThat(client3.getDefaultRequestOptions().getMaximumExecutionTimeInMs(), is(30 * 1000));
}
/**

2
plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTest.java → plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
View File

@ -31,7 +31,7 @@ import java.io.IOException;
import static org.hamcrest.Matchers.contains;
public class AzureStorageSettingsFilterTest extends ESTestCase {
public class AzureStorageSettingsFilterTests extends ESTestCase {
final static Settings settings = Settings.builder()
.put("cloud.azure.storage.azure1.account", "myaccount1")
.put("cloud.azure.storage.azure1.key", "mykey1")

2
plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTest.java → plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java
View File

@ -31,7 +31,7 @@ import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
public class AzureSettingsParserTest extends LuceneTestCase {
public class AzureSettingsParserTests extends LuceneTestCase {
public void testParseTwoSettingsExplicitDefault() {
Settings settings = Settings.builder()

46
plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
View File

@ -106,9 +106,9 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
logger.info("--> creating azure repository with path [{}]", getRepositoryPath());
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("azure").setSettings(Settings.settingsBuilder()
.put(Repository.CONTAINER, getContainerName())
.put(Repository.BASE_PATH, getRepositoryPath())
.put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
.put(Repository.CONTAINER_SETTING.getKey(), getContainerName())
.put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
.put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@ -197,9 +197,9 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
logger.info("creating Azure repository with path [{}]", getRepositoryPath());
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository(repositoryName)
.setType("azure").setSettings(Settings.settingsBuilder()
.put(Repository.CONTAINER, getContainerName())
.put(Repository.BASE_PATH, getRepositoryPath())
.put(Repository.BASE_PATH, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
.put(Repository.CONTAINER_SETTING.getKey(), getContainerName())
.put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
.put(Repository.BASE_PATH_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@ -237,16 +237,16 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
logger.info("--> creating azure repository with path [{}]", getRepositoryPath());
PutRepositoryResponse putRepositoryResponse1 = client.admin().cluster().preparePutRepository("test-repo1")
.setType("azure").setSettings(Settings.settingsBuilder()
.put(Repository.CONTAINER, getContainerName().concat("-1"))
.put(Repository.BASE_PATH, getRepositoryPath())
.put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
.put(Repository.CONTAINER_SETTING.getKey(), getContainerName().concat("-1"))
.put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
.put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
).get();
assertThat(putRepositoryResponse1.isAcknowledged(), equalTo(true));
PutRepositoryResponse putRepositoryResponse2 = client.admin().cluster().preparePutRepository("test-repo2")
.setType("azure").setSettings(Settings.settingsBuilder()
.put(Repository.CONTAINER, getContainerName().concat("-2"))
.put(Repository.BASE_PATH, getRepositoryPath())
.put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
.put(Repository.CONTAINER_SETTING.getKey(), getContainerName().concat("-2"))
.put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
.put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
).get();
assertThat(putRepositoryResponse2.isAcknowledged(), equalTo(true));
@ -316,7 +316,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
logger.info("--> creating azure repository without any path");
PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure")
.setSettings(Settings.settingsBuilder()
.put(Repository.CONTAINER, getContainerName())
.put(Repository.CONTAINER_SETTING.getKey(), getContainerName())
).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@ -337,8 +337,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
logger.info("--> creating azure repository path [{}]", getRepositoryPath());
putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure")
.setSettings(Settings.settingsBuilder()
.put(Repository.CONTAINER, getContainerName())
.put(Repository.BASE_PATH, getRepositoryPath())
.put(Repository.CONTAINER_SETTING.getKey(), getContainerName())
.put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@ -363,7 +363,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
logger.info("--> creating azure repository without any path");
PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure")
.setSettings(Settings.settingsBuilder()
.put(Repository.CONTAINER, getContainerName())
.put(Repository.CONTAINER_SETTING.getKey(), getContainerName())
).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@ -414,9 +414,9 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
try {
PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
.setType("azure").setSettings(Settings.settingsBuilder()
.put(Repository.CONTAINER, container)
.put(Repository.BASE_PATH, getRepositoryPath())
.put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
.put(Repository.CONTAINER_SETTING.getKey(), container)
.put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
.put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
).get();
client().admin().cluster().prepareDeleteRepository("test-repo").get();
try {
@ -444,9 +444,9 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
logger.info("--> creating azure repository with path [{}]", getRepositoryPath());
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("azure").setSettings(Settings.settingsBuilder()
.put(Repository.CONTAINER, getContainerName())
.put(Repository.BASE_PATH, getRepositoryPath())
.put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
.put(Repository.CONTAINER_SETTING.getKey(), getContainerName())
.put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
.put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@ -492,7 +492,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
try {
client.preparePutRepository("test-repo").setType("azure")
.setSettings(Settings.settingsBuilder()
.put(Repository.CONTAINER, container)
.put(Repository.CONTAINER_SETTING.getKey(), container)
).get();
fail("we should get a RepositoryVerificationException");
} catch (RepositoryVerificationException e) {