honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-22 12:56:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add azure storage endpoint suffix #26432 (#26568)

Browse Source 
Allow specifying azure storage endpoint suffix for an azure client.
This commit is contained in:
kel 2017-09-21 13:26:19 +08:00 committed by Ryan Ernst
parent 06551a8549
commit 601be4f83e
5 changed files with 53 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/plugins/repository-azure.asciidoc
View File

@ -44,15 +44,18 @@ The initial backoff period is defined by Azure SDK as `30s`. Which means `30s` o
before retrying after a first timeout or failure. The maximum backoff period is defined by Azure SDK as
`90s`.
`endpoint_suffix` can be used to specify Azure endpoint suffix explicitly. Defaults to `core.windows.net`.
[source,yaml]
----
azure.client.default.timeout: 10s
azure.client.default.max_retries: 7
azure.client.default.endpoint_suffix: core.chinacloudapi.cn
azure.client.secondary.timeout: 30s
----
In this example, timeout will be `10s` per try for `default` with `7` retries before failing
and `30s` per try for `secondary` with `3` retries.
and endpoint suffix will be `core.chinacloudapi.cn` and `30s` per try for `secondary` with `3` retries.
[IMPORTANT]
.Supported Azure Storage Account types

1
plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
View File

@ -60,6 +60,7 @@ public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin {
return Arrays.asList(
AzureStorageSettings.ACCOUNT_SETTING,
AzureStorageSettings.KEY_SETTING,
AzureStorageSettings.ENDPOINT_SUFFIX_SETTING,
AzureStorageSettings.TIMEOUT_SETTING,
AzureStorageSettings.PROXY_TYPE_SETTING,
AzureStorageSettings.PROXY_HOST_SETTING,

8
plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageServiceImpl.java
View File

@ -79,14 +79,18 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
void createClient(AzureStorageSettings azureStorageSettings) {
try {
logger.trace("creating new Azure storage client using account [{}], key [{}]",
azureStorageSettings.getAccount(), azureStorageSettings.getKey());
logger.trace("creating new Azure storage client using account [{}], key [{}], endpoint suffix [{}]",
azureStorageSettings.getAccount(), azureStorageSettings.getKey(), azureStorageSettings.getEndpointSuffix());
String storageConnectionString =
"DefaultEndpointsProtocol=https;"
+ "AccountName=" + azureStorageSettings.getAccount() + ";"
+ "AccountKey=" + azureStorageSettings.getKey();
String endpointSuffix = azureStorageSettings.getEndpointSuffix();
if (endpointSuffix != null && !endpointSuffix.isEmpty()) {
storageConnectionString += ";EndpointSuffix=" + endpointSuffix;
}
// Retrieve storage account from connection-string.
CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);

17
plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java
View File

@ -52,6 +52,11 @@ public final class AzureStorageSettings {
private static final Setting<Integer> MAX_RETRIES_SETTING =
Setting.affixKeySetting(PREFIX, "max_retries",
(key) -> Setting.intSetting(key, RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT, Setting.Property.NodeScope));
/**
* Azure endpoint suffix. Default to core.windows.net (CloudStorageAccount.DEFAULT_DNS).
*/
public static final Setting<String> ENDPOINT_SUFFIX_SETTING = Setting.affixKeySetting(PREFIX, "endpoint_suffix",
key -> Setting.simpleString(key, Property.NodeScope));
/** Azure key */
public static final AffixSetting<SecureString> KEY_SETTING = Setting.affixKeySetting(PREFIX, "key",
@ -74,15 +79,17 @@ public final class AzureStorageSettings {
private final String account;
private final String key;
private final String endpointSuffix;
private final TimeValue timeout;
private final int maxRetries;
private final Proxy proxy;
public AzureStorageSettings(String account, String key, TimeValue timeout, int maxRetries, Proxy.Type proxyType, String proxyHost,
Integer proxyPort) {
public AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries,
Proxy.Type proxyType, String proxyHost, Integer proxyPort) {
this.account = account;
this.key = key;
this.endpointSuffix = endpointSuffix;
this.timeout = timeout;
this.maxRetries = maxRetries;
@ -114,6 +121,10 @@ public final class AzureStorageSettings {
return account;
}
public String getEndpointSuffix() {
return endpointSuffix;
}
public TimeValue getTimeout() {
return timeout;
}
@ -132,6 +143,7 @@ public final class AzureStorageSettings {
sb.append(", account='").append(account).append('\'');
sb.append(", key='").append(key).append('\'');
sb.append(", timeout=").append(timeout);
sb.append(", endpointSuffix='").append(endpointSuffix).append('\'');
sb.append(", maxRetries=").append(maxRetries);
sb.append(", proxy=").append(proxy);
sb.append('}');
@ -166,6 +178,7 @@ public final class AzureStorageSettings {
try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING);
SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) {
return new AzureStorageSettings(account.toString(), key.toString(),
getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING),
getValue(settings, clientName, TIMEOUT_SETTING),
getValue(settings, clientName, MAX_RETRIES_SETTING),
getValue(settings, clientName, PROXY_TYPE_SETTING),

28
plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java
View File

@ -22,6 +22,8 @@ package org.elasticsearch.repositories.azure;
import com.microsoft.azure.storage.LocationMode;
import com.microsoft.azure.storage.RetryExponentialRetry;
import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.azure.storage.core.Base64;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
@ -33,12 +35,15 @@ import java.net.Proxy;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import static org.elasticsearch.repositories.azure.AzureStorageServiceImpl.blobNameFromUri;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.isEmptyString;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
@ -69,10 +74,31 @@ public class AzureStorageServiceTests extends ESTestCase {
secureSettings.setString("azure.client.azure2.key", "mykey2");
secureSettings.setString("azure.client.azure3.account", "myaccount3");
secureSettings.setString("azure.client.azure3.key", "mykey3");
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
Settings settings = Settings.builder().setSecureSettings(secureSettings)
.put("azure.client.azure3.endpoint_suffix", "my_endpoint_suffix").build();
Map<String, AzureStorageSettings> loadedSettings = AzureStorageSettings.load(settings);
assertThat(loadedSettings.keySet(), containsInAnyOrder("azure1","azure2","azure3","default"));
assertThat(loadedSettings.get("azure1").getEndpointSuffix(), isEmptyString());
assertThat(loadedSettings.get("azure2").getEndpointSuffix(), isEmptyString());
assertThat(loadedSettings.get("azure3").getEndpointSuffix(), equalTo("my_endpoint_suffix"));
}
public void testCreateClientWithEndpointSuffix() {
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString("azure.client.azure1.account", "myaccount1");
secureSettings.setString("azure.client.azure1.key", Base64.encode("mykey1".getBytes(StandardCharsets.UTF_8)));
secureSettings.setString("azure.client.azure2.account", "myaccount2");
secureSettings.setString("azure.client.azure2.key", Base64.encode("mykey2".getBytes(StandardCharsets.UTF_8)));
Settings settings = Settings.builder().setSecureSettings(secureSettings)
.put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix").build();
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceImpl(settings, AzureStorageSettings.load(settings));
CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY);
assertThat(client1.getEndpoint().toString(), equalTo("https://myaccount1.blob.my_endpoint_suffix"));
CloudBlobClient client2 = azureStorageService.getSelectedClient("azure2", LocationMode.PRIMARY_ONLY);
assertThat(client2.getEndpoint().toString(), equalTo("https://myaccount2.blob.core.windows.net"));
}
public void testGetSelectedClientWithNoPrimaryAndSecondary() {