Remove azure deprecated settings (#26099)
Follow up for #23405. We remove azure deprecated settings in 7.0: * The legacy azure settings which where starting with `cloud.azure.storage.` prefix have been removed. This includes `account`, `key`, `default` and `timeout`. You need to use settings which are starting with `azure.client.` prefix instead. * Global timeout setting `cloud.azure.storage.timeout` has been removed. You must set it per azure client instead. Like `azure.client.default.timeout: 10s` for example.
This commit is contained in:
David Pilato
GitHub
parent
0e57a416f1
commit
b01b1c2a58
|
|
@ -46,7 +46,7 @@ before retrying after a first timeout or failure. The maximum backoff period is
|
|||
|
||||
[source,yaml]
|
||||
----
|
||||
cloud.azure.storage.timeout: 10s
|
||||
azure.client.default.timeout: 10s
|
||||
azure.client.default.max_retries: 7
|
||||
azure.client.secondary.timeout: 30s
|
||||
----
|
||||
|
|
|
|||
|
|
@ -29,9 +29,11 @@ way to reindex old indices is to use the `reindex` API.
|
|||
* <<breaking_70_indices_changes>>
|
||||
* <<breaking_70_mappings_changes>>
|
||||
* <<breaking_70_search_changes>>
|
||||
* <<breaking_70_plugins_changes>>
|
||||
|
||||
include::migrate_7_0/aggregations.asciidoc[]
|
||||
include::migrate_7_0/cluster.asciidoc[]
|
||||
include::migrate_7_0/indices.asciidoc[]
|
||||
include::migrate_7_0/mappings.asciidoc[]
|
||||
include::migrate_7_0/search.asciidoc[]
|
||||
include::migrate_7_0/plugins.asciidoc[]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
[[breaking_70_plugins_changes]]
|
||||
=== Plugins changes
|
||||
|
||||
==== Azure Repository plugin
|
||||
|
||||
* The legacy azure settings which where starting with `cloud.azure.storage.` prefix have been removed.
|
||||
This includes `account`, `key`, `default` and `timeout`.
|
||||
You need to use settings which are starting with `azure.client.` prefix instead.
|
||||
|
||||
* Global timeout setting `cloud.azure.storage.timeout` has been removed.
|
||||
You must set it per azure client instead. Like `azure.client.default.timeout: 10s` for example.
|
||||
|
||||
See {plugins}/repository-azure-usage.html#repository-azure-repository-settings[Azure Repository settings].
|
||||
|
||||
|
|
@ -43,8 +43,6 @@ thirdPartyAudit.excludes = [
|
|||
]
|
||||
|
||||
integTestCluster {
|
||||
setting 'cloud.azure.storage.my_account_test.account', 'cloudazureresource'
|
||||
setting 'cloud.azure.storage.my_account_test.key', 'abcdefgh'
|
||||
keystoreSetting 'azure.client.default.account', 'cloudazureresource'
|
||||
keystoreSetting 'azure.client.default.key', 'abcdefgh'
|
||||
keystoreSetting 'azure.client.secondary.account', 'cloudazureresource'
|
||||
|
|
|
|||
|
|
@ -22,12 +22,8 @@ package org.elasticsearch.cloud.azure.storage;
|
|||
import com.microsoft.azure.storage.LocationMode;
|
||||
import com.microsoft.azure.storage.StorageException;
|
||||
import org.elasticsearch.common.blobstore.BlobMetaData;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Setting.Property;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.ByteSizeUnit;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
|
@ -44,22 +40,6 @@ public interface AzureStorageService {
|
|||
ByteSizeValue MIN_CHUNK_SIZE = new ByteSizeValue(1, ByteSizeUnit.BYTES);
|
||||
ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB);
|
||||
|
||||
final class Storage {
|
||||
@Deprecated
|
||||
public static final String PREFIX = "cloud.azure.storage.";
|
||||
|
||||
@Deprecated
|
||||
public static final Setting<Settings> STORAGE_ACCOUNTS = Setting.groupSetting(Storage.PREFIX, Setting.Property.NodeScope);
|
||||
|
||||
/**
|
||||
* Azure timeout (defaults to -1 minute)
|
||||
* @deprecated We don't want to support global timeout settings anymore
|
||||
*/
|
||||
@Deprecated
|
||||
static final Setting<TimeValue> TIMEOUT_SETTING =
|
||||
Setting.timeSetting("cloud.azure.storage.timeout", TimeValue.timeValueMinutes(-1), Property.NodeScope, Property.Deprecated);
|
||||
}
|
||||
|
||||
boolean doesContainerExist(String account, LocationMode mode, String container);
|
||||
|
||||
void removeContainer(String account, LocationMode mode, String container) throws URISyntaxException, StorageException;
|
||||
|
|
|
|||
|
|
@ -33,11 +33,9 @@ import com.microsoft.azure.storage.blob.ListBlobItem;
|
|||
import org.apache.logging.log4j.message.ParameterizedMessage;
|
||||
import org.apache.logging.log4j.util.Supplier;
|
||||
import org.elasticsearch.cloud.azure.blobstore.util.SocketAccess;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.blobstore.BlobMetaData;
|
||||
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
|
||||
import org.elasticsearch.common.collect.MapBuilder;
|
||||
import org.elasticsearch.common.collect.Tuple;
|
||||
import org.elasticsearch.common.component.AbstractComponent;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.repositories.RepositoryException;
|
||||
|
|
@ -53,40 +51,19 @@ import java.util.Map;
|
|||
public class AzureStorageServiceImpl extends AbstractComponent implements AzureStorageService {
|
||||
|
||||
final Map<String, AzureStorageSettings> storageSettings;
|
||||
final Map<String, AzureStorageSettings> deprecatedStorageSettings;
|
||||
|
||||
final Map<String, CloudBlobClient> clients;
|
||||
final Map<String, CloudBlobClient> clients = new HashMap<>();
|
||||
|
||||
public AzureStorageServiceImpl(Settings settings, Map<String, AzureStorageSettings> regularStorageSettings) {
|
||||
public AzureStorageServiceImpl(Settings settings, Map<String, AzureStorageSettings> storageSettings) {
|
||||
super(settings);
|
||||
|
||||
if (regularStorageSettings.isEmpty()) {
|
||||
this.storageSettings = new HashMap<>();
|
||||
// We have deprecated settings so we need to migrate them to the new implementation
|
||||
Tuple<AzureStorageSettings, Map<String, AzureStorageSettings>> storageSettingsMapTuple = AzureStorageSettings.loadLegacy(settings);
|
||||
deprecatedStorageSettings = storageSettingsMapTuple.v2();
|
||||
if (storageSettingsMapTuple.v1() != null) {
|
||||
if (storageSettingsMapTuple.v1().getName().equals("default") == false) {
|
||||
// We add the primary configuration to the list of all settings with its deprecated name in case someone is
|
||||
// forcing a specific configuration name when creating the repository instance
|
||||
deprecatedStorageSettings.put(storageSettingsMapTuple.v1().getName(), storageSettingsMapTuple.v1());
|
||||
}
|
||||
// We add the primary configuration to the list of all settings as the "default" one
|
||||
deprecatedStorageSettings.put("default", storageSettingsMapTuple.v1());
|
||||
} else {
|
||||
// If someone did not register any settings or deprecated settings, they
|
||||
// basically can't use the plugin
|
||||
this.storageSettings = storageSettings;
|
||||
|
||||
if (storageSettings.isEmpty()) {
|
||||
// If someone did not register any settings, they basically can't use the plugin
|
||||
throw new IllegalArgumentException("If you want to use an azure repository, you need to define a client configuration.");
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
this.storageSettings = regularStorageSettings;
|
||||
this.deprecatedStorageSettings = new HashMap<>();
|
||||
}
|
||||
|
||||
this.clients = new HashMap<>();
|
||||
|
||||
logger.debug("starting azure storage client instance");
|
||||
|
||||
// We register all regular azure clients
|
||||
|
|
@ -94,12 +71,6 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
|
|||
logger.debug("registering regular client for account [{}]", azureStorageSettingsEntry.getKey());
|
||||
createClient(azureStorageSettingsEntry.getValue());
|
||||
}
|
||||
|
||||
// We register all deprecated azure clients
|
||||
for (Map.Entry<String, AzureStorageSettings> azureStorageSettingsEntry : this.deprecatedStorageSettings.entrySet()) {
|
||||
logger.debug("registering deprecated client for account [{}]", azureStorageSettingsEntry.getKey());
|
||||
createClient(azureStorageSettingsEntry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
void createClient(AzureStorageSettings azureStorageSettings) {
|
||||
|
|
@ -125,31 +96,21 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
|
|||
}
|
||||
}
|
||||
|
||||
CloudBlobClient getSelectedClient(String account, LocationMode mode) {
|
||||
logger.trace("selecting a client for account [{}], mode [{}]", account, mode.name());
|
||||
AzureStorageSettings azureStorageSettings = this.storageSettings.get(account);
|
||||
CloudBlobClient getSelectedClient(String clientName, LocationMode mode) {
|
||||
logger.trace("selecting a client named [{}], mode [{}]", clientName, mode.name());
|
||||
AzureStorageSettings azureStorageSettings = this.storageSettings.get(clientName);
|
||||
if (azureStorageSettings == null) {
|
||||
// We can't find a client that has been registered using regular settings so we try deprecated client
|
||||
azureStorageSettings = this.deprecatedStorageSettings.get(account);
|
||||
if (azureStorageSettings == null) {
|
||||
// We did not get an account. That's bad.
|
||||
if (Strings.hasLength(account)) {
|
||||
throw new IllegalArgumentException("Can not find named azure client [" + account +
|
||||
"]. Check your elasticsearch.yml.");
|
||||
}
|
||||
throw new IllegalArgumentException("Can not find primary/secondary client using deprecated settings. " +
|
||||
"Check your elasticsearch.yml.");
|
||||
}
|
||||
throw new IllegalArgumentException("Can not find named azure client [" + clientName + "]. Check your settings.");
|
||||
}
|
||||
|
||||
CloudBlobClient client = this.clients.get(azureStorageSettings.getAccount());
|
||||
|
||||
if (client == null) {
|
||||
throw new IllegalArgumentException("Can not find an azure client for account [" + azureStorageSettings.getAccount() + "]");
|
||||
throw new IllegalArgumentException("Can not find an azure client named [" + azureStorageSettings.getAccount() + "]");
|
||||
}
|
||||
|
||||
// NOTE: for now, just set the location mode in case it is different;
|
||||
// only one mode per storage account can be active at a time
|
||||
// only one mode per storage clientName can be active at a time
|
||||
client.getDefaultRequestOptions().setLocationMode(mode);
|
||||
|
||||
// Set timeout option if the user sets cloud.azure.storage.timeout or cloud.azure.storage.xxx.timeout (it's negative by default)
|
||||
|
|
|
|||
|
|
@ -20,26 +20,19 @@
|
|||
package org.elasticsearch.cloud.azure.storage;
|
||||
|
||||
import com.microsoft.azure.storage.RetryPolicy;
|
||||
import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
|
||||
import org.elasticsearch.common.collect.Tuple;
|
||||
import org.elasticsearch.common.settings.SecureSetting;
|
||||
import org.elasticsearch.common.settings.SecureString;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Setting.AffixSetting;
|
||||
import org.elasticsearch.common.settings.Setting.Property;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.settings.SettingsException;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage.STORAGE_ACCOUNTS;
|
||||
|
||||
public final class AzureStorageSettings {
|
||||
// prefix for azure client settings
|
||||
private static final String PREFIX = "azure.client.";
|
||||
|
|
@ -64,56 +57,20 @@ public final class AzureStorageSettings {
|
|||
key -> SecureSetting.secureString(key, null));
|
||||
|
||||
public static final AffixSetting<TimeValue> TIMEOUT_SETTING = Setting.affixKeySetting(PREFIX, "timeout",
|
||||
(key) -> Setting.timeSetting(key, Storage.TIMEOUT_SETTING, Property.NodeScope));
|
||||
(key) -> Setting.timeSetting(key, TimeValue.timeValueMinutes(-1), Property.NodeScope));
|
||||
|
||||
|
||||
@Deprecated
|
||||
public static final Setting<TimeValue> DEPRECATED_TIMEOUT_SETTING = Setting.affixKeySetting(Storage.PREFIX, "timeout",
|
||||
(key) -> Setting.timeSetting(key, Storage.TIMEOUT_SETTING, Property.NodeScope, Property.Deprecated));
|
||||
@Deprecated
|
||||
public static final Setting<String> DEPRECATED_ACCOUNT_SETTING = Setting.affixKeySetting(Storage.PREFIX, "account",
|
||||
(key) -> Setting.simpleString(key, Property.NodeScope, Property.Deprecated));
|
||||
@Deprecated
|
||||
public static final Setting<String> DEPRECATED_KEY_SETTING = Setting.affixKeySetting(Storage.PREFIX, "key",
|
||||
(key) -> Setting.simpleString(key, Property.NodeScope, Property.Deprecated));
|
||||
@Deprecated
|
||||
public static final Setting<Boolean> DEPRECATED_DEFAULT_SETTING = Setting.affixKeySetting(Storage.PREFIX, "default",
|
||||
(key) -> Setting.boolSetting(key, false, Property.NodeScope, Property.Deprecated));
|
||||
|
||||
|
||||
@Deprecated
|
||||
private final String name;
|
||||
private final String account;
|
||||
private final String key;
|
||||
private final TimeValue timeout;
|
||||
@Deprecated
|
||||
private final boolean activeByDefault;
|
||||
private final int maxRetries;
|
||||
|
||||
public AzureStorageSettings(String account, String key, TimeValue timeout, int maxRetries) {
|
||||
this.name = null;
|
||||
this.account = account;
|
||||
this.key = key;
|
||||
this.timeout = timeout;
|
||||
this.activeByDefault = false;
|
||||
this.maxRetries = maxRetries;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public AzureStorageSettings(String name, String account, String key, TimeValue timeout, boolean activeByDefault, int maxRetries) {
|
||||
this.name = name;
|
||||
this.account = account;
|
||||
this.key = key;
|
||||
this.timeout = timeout;
|
||||
this.activeByDefault = activeByDefault;
|
||||
this.maxRetries = maxRetries;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public String getKey() {
|
||||
return key;
|
||||
}
|
||||
|
|
@ -126,11 +83,6 @@ public final class AzureStorageSettings {
|
|||
return timeout;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public Boolean isActiveByDefault() {
|
||||
return activeByDefault;
|
||||
}
|
||||
|
||||
public int getMaxRetries() {
|
||||
return maxRetries;
|
||||
}
|
||||
|
|
@ -138,27 +90,14 @@ public final class AzureStorageSettings {
|
|||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder sb = new StringBuilder("AzureStorageSettings{");
|
||||
sb.append("name='").append(name).append('\'');
|
||||
sb.append(", account='").append(account).append('\'');
|
||||
sb.append(", key='").append(key).append('\'');
|
||||
sb.append(", activeByDefault='").append(activeByDefault).append('\'');
|
||||
sb.append(", timeout=").append(timeout);
|
||||
sb.append(", maxRetries=").append(maxRetries);
|
||||
sb.append('}');
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses settings and read all legacy settings available under cloud.azure.storage.*
|
||||
* @param settings settings to parse
|
||||
* @return A tuple with v1 = primary storage and v2 = secondary storage
|
||||
*/
|
||||
@Deprecated
|
||||
public static Tuple<AzureStorageSettings, Map<String, AzureStorageSettings>> loadLegacy(Settings settings) {
|
||||
List<AzureStorageSettings> storageSettings = createStorageSettingsDeprecated(settings);
|
||||
return Tuple.tuple(getPrimary(storageSettings), getSecondaries(storageSettings));
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses settings and read all settings available under azure.client.*
|
||||
* @param settings settings to parse
|
||||
|
|
@ -192,25 +131,6 @@ public final class AzureStorageSettings {
|
|||
}
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
private static List<AzureStorageSettings> createStorageSettingsDeprecated(Settings settings) {
|
||||
// ignore global timeout which has the same prefix but does not belong to any group
|
||||
Settings groups = STORAGE_ACCOUNTS.get(settings.filter((k) -> k.equals(Storage.TIMEOUT_SETTING.getKey()) == false));
|
||||
List<AzureStorageSettings> storageSettings = new ArrayList<>();
|
||||
for (String groupName : groups.getAsGroups().keySet()) {
|
||||
storageSettings.add(
|
||||
new AzureStorageSettings(
|
||||
groupName,
|
||||
getValue(settings, groupName, DEPRECATED_ACCOUNT_SETTING),
|
||||
getValue(settings, groupName, DEPRECATED_KEY_SETTING),
|
||||
getValue(settings, groupName, DEPRECATED_TIMEOUT_SETTING),
|
||||
getValue(settings, groupName, DEPRECATED_DEFAULT_SETTING),
|
||||
getValue(settings, groupName, MAX_RETRIES_SETTING))
|
||||
);
|
||||
}
|
||||
return storageSettings;
|
||||
}
|
||||
|
||||
private static <T> T getConfigValue(Settings settings, String clientName,
|
||||
Setting.AffixSetting<T> clientSetting) {
|
||||
Setting<T> concreteSetting = clientSetting.getConcreteSettingForNamespace(clientName);
|
||||
|
|
@ -222,45 +142,4 @@ public final class AzureStorageSettings {
|
|||
String fullKey = k.toConcreteKey(groupName).toString();
|
||||
return setting.getConcreteSetting(fullKey).get(settings);
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
private static AzureStorageSettings getPrimary(List<AzureStorageSettings> settings) {
|
||||
if (settings.isEmpty()) {
|
||||
return null;
|
||||
} else if (settings.size() == 1) {
|
||||
// the only storage settings belong (implicitly) to the default primary storage
|
||||
AzureStorageSettings storage = settings.get(0);
|
||||
return new AzureStorageSettings(storage.getName(), storage.getAccount(), storage.getKey(), storage.getTimeout(), true,
|
||||
storage.getMaxRetries());
|
||||
} else {
|
||||
AzureStorageSettings primary = null;
|
||||
for (AzureStorageSettings setting : settings) {
|
||||
if (setting.isActiveByDefault()) {
|
||||
if (primary == null) {
|
||||
primary = setting;
|
||||
} else {
|
||||
throw new SettingsException("Multiple default Azure data stores configured: [" + primary.getName() + "] and [" + setting.getName() + "]");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (primary == null) {
|
||||
throw new SettingsException("No default Azure data store configured");
|
||||
}
|
||||
return primary;
|
||||
}
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
private static Map<String, AzureStorageSettings> getSecondaries(List<AzureStorageSettings> settings) {
|
||||
Map<String, AzureStorageSettings> secondaries = new HashMap<>();
|
||||
// when only one setting is defined, we don't have secondaries
|
||||
if (settings.size() > 1) {
|
||||
for (AzureStorageSettings setting : settings) {
|
||||
if (setting.isActiveByDefault() == false) {
|
||||
secondaries.put(setting.getName(), setting);
|
||||
}
|
||||
}
|
||||
}
|
||||
return secondaries;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,16 +62,9 @@ public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin {
|
|||
@Override
|
||||
public List<Setting<?>> getSettings() {
|
||||
return Arrays.asList(
|
||||
AzureStorageService.Storage.STORAGE_ACCOUNTS,
|
||||
AzureStorageSettings.ACCOUNT_SETTING,
|
||||
AzureStorageSettings.KEY_SETTING,
|
||||
AzureStorageSettings.TIMEOUT_SETTING
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getSettingsFilter() {
|
||||
// Cloud storage API settings using a pattern needed to be hidden
|
||||
return Arrays.asList(AzureStorageService.Storage.PREFIX + "*.account", AzureStorageService.Storage.PREFIX + "*.key");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import com.microsoft.azure.storage.LocationMode;
|
|||
import com.microsoft.azure.storage.RetryExponentialRetry;
|
||||
import com.microsoft.azure.storage.blob.CloudBlobClient;
|
||||
import org.elasticsearch.common.settings.MockSecureSettings;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
|
|
@ -32,11 +31,6 @@ import java.net.URISyntaxException;
|
|||
import java.util.Map;
|
||||
|
||||
import static org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl.blobNameFromUri;
|
||||
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_ACCOUNT_SETTING;
|
||||
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_DEFAULT_SETTING;
|
||||
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_KEY_SETTING;
|
||||
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_TIMEOUT_SETTING;
|
||||
import static org.elasticsearch.repositories.azure.AzureSettingsParserTests.getConcreteSetting;
|
||||
import static org.hamcrest.Matchers.containsInAnyOrder;
|
||||
import static org.hamcrest.Matchers.instanceOf;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
|
|
@ -45,18 +39,6 @@ import static org.hamcrest.Matchers.nullValue;
|
|||
|
||||
public class AzureStorageServiceTests extends ESTestCase {
|
||||
|
||||
@Deprecated
|
||||
static final Settings deprecatedSettings = Settings.builder()
|
||||
.put("cloud.azure.storage.azure1.account", "myaccount1")
|
||||
.put("cloud.azure.storage.azure1.key", "mykey1")
|
||||
.put("cloud.azure.storage.azure1.default", true)
|
||||
.put("cloud.azure.storage.azure2.account", "myaccount2")
|
||||
.put("cloud.azure.storage.azure2.key", "mykey2")
|
||||
.put("cloud.azure.storage.azure3.account", "myaccount3")
|
||||
.put("cloud.azure.storage.azure3.key", "mykey3")
|
||||
.put("cloud.azure.storage.azure3.timeout", "30s")
|
||||
.build();
|
||||
|
||||
private MockSecureSettings buildSecureSettings() {
|
||||
MockSecureSettings secureSettings = new MockSecureSettings();
|
||||
secureSettings.setString("azure.client.azure1.account", "myaccount1");
|
||||
|
|
@ -102,23 +84,7 @@ public class AzureStorageServiceTests extends ESTestCase {
|
|||
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
|
||||
azureStorageService.getSelectedClient("azure4", LocationMode.PRIMARY_ONLY);
|
||||
});
|
||||
assertThat(e.getMessage(), is("Can not find named azure client [azure4]. Check your elasticsearch.yml."));
|
||||
}
|
||||
|
||||
public void testGetSelectedClientGlobalTimeout() {
|
||||
Settings timeoutSettings = Settings.builder()
|
||||
.setSecureSettings(buildSecureSettings())
|
||||
.put(AzureStorageService.Storage.TIMEOUT_SETTING.getKey(), "10s")
|
||||
.put("azure.client.azure3.timeout", "30s")
|
||||
.build();
|
||||
|
||||
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings);
|
||||
CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY);
|
||||
assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(10 * 1000));
|
||||
CloudBlobClient client3 = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY);
|
||||
assertThat(client3.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(30 * 1000));
|
||||
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[]{AzureStorageService.Storage.TIMEOUT_SETTING});
|
||||
assertThat(e.getMessage(), is("Can not find named azure client [azure4]. Check your settings."));
|
||||
}
|
||||
|
||||
public void testGetSelectedClientDefaultTimeout() {
|
||||
|
|
@ -170,7 +136,7 @@ public class AzureStorageServiceTests extends ESTestCase {
|
|||
@Override
|
||||
void createClient(AzureStorageSettings azureStorageSettings) {
|
||||
this.clients.put(azureStorageSettings.getAccount(),
|
||||
new CloudBlobClient(URI.create("https://" + azureStorageSettings.getName())));
|
||||
new CloudBlobClient(URI.create("https://" + azureStorageSettings.getAccount())));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -184,87 +150,4 @@ public class AzureStorageServiceTests extends ESTestCase {
|
|||
name = blobNameFromUri(new URI("https://127.0.0.1/container/path/to/myfile"));
|
||||
assertThat(name, is("path/to/myfile"));
|
||||
}
|
||||
|
||||
// Deprecated settings. We still test them until we remove definitely the deprecated settings
|
||||
|
||||
@Deprecated
|
||||
public void testGetSelectedClientWithNoSecondary() {
|
||||
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(Settings.builder()
|
||||
.put("cloud.azure.storage.azure1.account", "myaccount1")
|
||||
.put("cloud.azure.storage.azure1.key", "mykey1")
|
||||
.build());
|
||||
CloudBlobClient client = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY);
|
||||
assertThat(client.getEndpoint(), is(URI.create("https://azure1")));
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
|
||||
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1")
|
||||
});
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public void testGetDefaultClientWithNoSecondary() {
|
||||
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(Settings.builder()
|
||||
.put("cloud.azure.storage.azure1.account", "myaccount1")
|
||||
.put("cloud.azure.storage.azure1.key", "mykey1")
|
||||
.build());
|
||||
CloudBlobClient client = azureStorageService.getSelectedClient("default", LocationMode.PRIMARY_ONLY);
|
||||
assertThat(client.getEndpoint(), is(URI.create("https://azure1")));
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
|
||||
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1")
|
||||
});
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public void testGetSelectedClientPrimary() {
|
||||
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(deprecatedSettings);
|
||||
CloudBlobClient client = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY);
|
||||
assertThat(client.getEndpoint(), is(URI.create("https://azure1")));
|
||||
assertDeprecatedWarnings();
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public void testGetSelectedClientSecondary1() {
|
||||
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(deprecatedSettings);
|
||||
CloudBlobClient client = azureStorageService.getSelectedClient("azure2", LocationMode.PRIMARY_ONLY);
|
||||
assertThat(client.getEndpoint(), is(URI.create("https://azure2")));
|
||||
assertDeprecatedWarnings();
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public void testGetSelectedClientSecondary2() {
|
||||
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(deprecatedSettings);
|
||||
CloudBlobClient client = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY);
|
||||
assertThat(client.getEndpoint(), is(URI.create("https://azure3")));
|
||||
assertDeprecatedWarnings();
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public void testGetDefaultClientWithPrimaryAndSecondaries() {
|
||||
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(deprecatedSettings);
|
||||
CloudBlobClient client = azureStorageService.getSelectedClient("default", LocationMode.PRIMARY_ONLY);
|
||||
assertThat(client.getEndpoint(), is(URI.create("https://azure1")));
|
||||
assertDeprecatedWarnings();
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public void testGetSelectedClientDefault() {
|
||||
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(deprecatedSettings);
|
||||
CloudBlobClient client = azureStorageService.getSelectedClient("default", LocationMode.PRIMARY_ONLY);
|
||||
assertThat(client.getEndpoint(), is(URI.create("https://azure1")));
|
||||
assertDeprecatedWarnings();
|
||||
}
|
||||
|
||||
private void assertDeprecatedWarnings() {
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
|
||||
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_DEFAULT_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure2"),
|
||||
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure2"),
|
||||
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure3"),
|
||||
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure3"),
|
||||
getConcreteSetting(DEPRECATED_TIMEOUT_SETTING, "azure3")
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,72 +0,0 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.cloud.azure.storage;
|
||||
|
||||
import org.elasticsearch.common.inject.ModuleTestCase;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.settings.SettingsFilter;
|
||||
import org.elasticsearch.common.settings.SettingsModule;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin;
|
||||
import org.elasticsearch.rest.RestRequest;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.elasticsearch.test.rest.FakeRestRequest;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.hamcrest.Matchers.contains;
|
||||
|
||||
/**
|
||||
* TODO as we moved credentials to secure settings, we don't need anymore to keep this test in 7.x
|
||||
*/
|
||||
public class AzureStorageSettingsFilterTests extends ESTestCase {
|
||||
static final Settings settings = Settings.builder()
|
||||
.put("cloud.azure.storage.azure1.account", "myaccount1")
|
||||
.put("cloud.azure.storage.azure1.key", "mykey1")
|
||||
.put("cloud.azure.storage.azure1.default", true)
|
||||
.put("cloud.azure.storage.azure2.account", "myaccount2")
|
||||
.put("cloud.azure.storage.azure2.key", "mykey2")
|
||||
.put("cloud.azure.storage.azure3.account", "myaccount3")
|
||||
.put("cloud.azure.storage.azure3.key", "mykey3")
|
||||
.build();
|
||||
|
||||
public void testSettingsFiltering() throws IOException {
|
||||
AzureRepositoryPlugin p = new AzureRepositoryPlugin(settings);
|
||||
SettingsModule module = new SettingsModule(Settings.EMPTY, p.getSettings(), p.getSettingsFilter());
|
||||
SettingsFilter settingsFilter = ModuleTestCase.bindAndGetInstance(module, SettingsFilter.class);
|
||||
|
||||
// Test using direct filtering
|
||||
Settings filteredSettings = settingsFilter.filter(settings);
|
||||
assertThat(filteredSettings.getAsMap().keySet(), contains("cloud.azure.storage.azure1.default"));
|
||||
|
||||
// Test using toXContent filtering
|
||||
RestRequest request = new FakeRestRequest();
|
||||
settingsFilter.addFilterSettingParams(request);
|
||||
XContentBuilder xContentBuilder = XContentBuilder.builder(JsonXContent.jsonXContent);
|
||||
xContentBuilder.startObject();
|
||||
settings.toXContent(xContentBuilder, request);
|
||||
xContentBuilder.endObject();
|
||||
String filteredSettingsString = xContentBuilder.string();
|
||||
filteredSettings = Settings.builder().loadFromSource(filteredSettingsString, xContentBuilder.contentType()).build();
|
||||
assertThat(filteredSettings.getAsMap().keySet(), contains("cloud.azure.storage.azure1.default"));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,143 +0,0 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.repositories.azure;
|
||||
|
||||
import org.elasticsearch.cloud.azure.storage.AzureStorageSettings;
|
||||
import org.elasticsearch.common.collect.Tuple;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.settings.SettingsException;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_ACCOUNT_SETTING;
|
||||
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_DEFAULT_SETTING;
|
||||
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_KEY_SETTING;
|
||||
import static org.hamcrest.Matchers.hasSize;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
import static org.hamcrest.Matchers.notNullValue;
|
||||
import static org.hamcrest.Matchers.nullValue;
|
||||
|
||||
public class AzureSettingsParserTests extends ESTestCase {
|
||||
|
||||
public void testParseTwoSettingsExplicitDefault() {
|
||||
Settings settings = Settings.builder()
|
||||
.put("cloud.azure.storage.azure1.account", "myaccount1")
|
||||
.put("cloud.azure.storage.azure1.key", "mykey1")
|
||||
.put("cloud.azure.storage.azure1.default", true)
|
||||
.put("cloud.azure.storage.azure2.account", "myaccount2")
|
||||
.put("cloud.azure.storage.azure2.key", "mykey2")
|
||||
.build();
|
||||
|
||||
Tuple<AzureStorageSettings, Map<String, AzureStorageSettings>> tuple = AzureStorageSettings.loadLegacy(settings);
|
||||
assertThat(tuple.v1(), notNullValue());
|
||||
assertThat(tuple.v1().getAccount(), is("myaccount1"));
|
||||
assertThat(tuple.v1().getKey(), is("mykey1"));
|
||||
assertThat(tuple.v2().keySet(), hasSize(1));
|
||||
assertThat(tuple.v2().get("azure2"), notNullValue());
|
||||
assertThat(tuple.v2().get("azure2").getAccount(), is("myaccount2"));
|
||||
assertThat(tuple.v2().get("azure2").getKey(), is("mykey2"));
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
|
||||
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_DEFAULT_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure2"),
|
||||
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure2")
|
||||
});
|
||||
}
|
||||
|
||||
public void testParseUniqueSettings() {
|
||||
Settings settings = Settings.builder()
|
||||
.put("cloud.azure.storage.azure1.account", "myaccount1")
|
||||
.put("cloud.azure.storage.azure1.key", "mykey1")
|
||||
.build();
|
||||
|
||||
Tuple<AzureStorageSettings, Map<String, AzureStorageSettings>> tuple = AzureStorageSettings.loadLegacy(settings);
|
||||
assertThat(tuple.v1(), notNullValue());
|
||||
assertThat(tuple.v1().getAccount(), is("myaccount1"));
|
||||
assertThat(tuple.v1().getKey(), is("mykey1"));
|
||||
assertThat(tuple.v2().keySet(), hasSize(0));
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
|
||||
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1")
|
||||
});
|
||||
}
|
||||
|
||||
public void testParseTwoSettingsNoDefault() {
|
||||
Settings settings = Settings.builder()
|
||||
.put("cloud.azure.storage.azure1.account", "myaccount1")
|
||||
.put("cloud.azure.storage.azure1.key", "mykey1")
|
||||
.put("cloud.azure.storage.azure2.account", "myaccount2")
|
||||
.put("cloud.azure.storage.azure2.key", "mykey2")
|
||||
.build();
|
||||
|
||||
try {
|
||||
AzureStorageSettings.loadLegacy(settings);
|
||||
fail("Should have failed with a SettingsException (no default data store)");
|
||||
} catch (SettingsException ex) {
|
||||
assertEquals(ex.getMessage(), "No default Azure data store configured");
|
||||
}
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
|
||||
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure2"),
|
||||
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure2"),
|
||||
});
|
||||
}
|
||||
|
||||
public void testParseTwoSettingsTooManyDefaultSet() {
|
||||
Settings settings = Settings.builder()
|
||||
.put("cloud.azure.storage.azure1.account", "myaccount1")
|
||||
.put("cloud.azure.storage.azure1.key", "mykey1")
|
||||
.put("cloud.azure.storage.azure1.default", true)
|
||||
.put("cloud.azure.storage.azure2.account", "myaccount2")
|
||||
.put("cloud.azure.storage.azure2.key", "mykey2")
|
||||
.put("cloud.azure.storage.azure2.default", true)
|
||||
.build();
|
||||
|
||||
try {
|
||||
AzureStorageSettings.loadLegacy(settings);
|
||||
fail("Should have failed with a SettingsException (multiple default data stores)");
|
||||
} catch (SettingsException ex) {
|
||||
assertEquals(ex.getMessage(), "Multiple default Azure data stores configured: [azure1] and [azure2]");
|
||||
}
|
||||
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
|
||||
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_DEFAULT_SETTING, "azure1"),
|
||||
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure2"),
|
||||
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure2"),
|
||||
getConcreteSetting(DEPRECATED_DEFAULT_SETTING, "azure2")
|
||||
});
|
||||
}
|
||||
|
||||
public void testParseEmptySettings() {
|
||||
Tuple<AzureStorageSettings, Map<String, AzureStorageSettings>> tuple = AzureStorageSettings.loadLegacy(Settings.EMPTY);
|
||||
assertThat(tuple.v1(), nullValue());
|
||||
assertThat(tuple.v2().keySet(), hasSize(0));
|
||||
}
|
||||
|
||||
public static Setting<?> getConcreteSetting(Setting<?> setting, String groupName) {
|
||||
Setting.AffixKey k = (Setting.AffixKey) setting.getRawKey();
|
||||
String concreteKey = k.toConcreteKey(groupName).toString();
|
||||
return setting.getConcreteSetting(concreteKey);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue