honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'pr/22728-azure-backoff'

This commit is contained in:
David Pilato 2017-04-03 10:54:03 +02:00
parent cad4fcd9c9 17be03e85e
commit 0951e20a87
4 changed files with 63 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/plugins/repository-azure.asciidoc
View File

@ -72,6 +72,12 @@ It's not set by default which means that elasticsearch is using the
http://azure.github.io/azure-storage-java/com/microsoft/azure/storage/RequestOptions.html#setTimeoutIntervalInMs(java.lang.Integer)[default value]
set by the azure client (known as 5 minutes).
`max_retries` can help to control the exponential backoff policy. It will fix the number of retries
in case of failures before considering the snapshot is failing. Defaults to `3` retries.
The initial backoff period is defined by Azure SDK as `30s`. Which means `30s` of wait time
before retrying after a first timeout or failure. The maximum backoff period is defined by Azure SDK as
`90s`.
[source,yaml]
----
cloud:
@ -82,13 +88,15 @@ cloud:
account: your_azure_storage_account1
key: your_azure_storage_key1
default: true
max_retries: 7
my_account2:
account: your_azure_storage_account2
key: your_azure_storage_key2
timeout: 30s
----
In this example, timeout will be 10s for `my_account1` and 30s for `my_account2`.
In this example, timeout will be `10s` per try for `my_account1` with `7` retries before failing
and `30s` per try for `my_account2` with `3` retries.
[[repository-azure-repository-settings]]
===== Repository settings

7
plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
View File

@ -21,6 +21,8 @@ package org.elasticsearch.cloud.azure.storage;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.LocationMode;
import com.microsoft.azure.storage.RetryExponentialRetry;
import com.microsoft.azure.storage.RetryPolicy;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.BlobProperties;
import com.microsoft.azure.storage.blob.CloudBlobClient;
@ -147,6 +149,11 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
"]. It can not be longer than 2,147,483,647ms.");
}
}
// We define a default exponential retry policy
client.getDefaultRequestOptions().setRetryPolicyFactory(
new RetryExponentialRetry(RetryPolicy.DEFAULT_CLIENT_BACKOFF, azureStorageSettings.getMaxRetries()));
return client;
}

23
plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.cloud.azure.storage;
import com.microsoft.azure.storage.RetryPolicy;
import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Setting;
@ -41,20 +42,27 @@ public final class AzureStorageSettings {
Setting.affixKeySetting(Storage.PREFIX, "key", (key) -> Setting.simpleString(key, Setting.Property.NodeScope));
private static final Setting<Boolean> DEFAULT_SETTING =
Setting.affixKeySetting(Storage.PREFIX, "default", (key) -> Setting.boolSetting(key, false, Setting.Property.NodeScope));
/**
* max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT).
*/
private static final Setting<Integer> MAX_RETRIES_SETTING =
Setting.affixKeySetting(Storage.PREFIX, "max_retries",
(key) -> Setting.intSetting(key, RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT, Setting.Property.NodeScope));
private final String name;
private final String account;
private final String key;
private final TimeValue timeout;
private final boolean activeByDefault;
private final int maxRetries;
public AzureStorageSettings(String name, String account, String key, TimeValue timeout, boolean activeByDefault) {
public AzureStorageSettings(String name, String account, String key, TimeValue timeout, boolean activeByDefault, int maxRetries) {
this.name = name;
this.account = account;
this.key = key;
this.timeout = timeout;
this.activeByDefault = activeByDefault;
this.maxRetries = maxRetries;
}
public String getName() {
@ -77,6 +85,10 @@ public final class AzureStorageSettings {
return activeByDefault;
}
public int getMaxRetries() {
return maxRetries;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("AzureStorageSettings{");
@ -85,6 +97,7 @@ public final class AzureStorageSettings {
sb.append(", key='").append(key).append('\'');
sb.append(", activeByDefault='").append(activeByDefault).append('\'');
sb.append(", timeout=").append(timeout);
sb.append(", maxRetries=").append(maxRetries);
sb.append('}');
return sb.toString();
}
@ -110,7 +123,8 @@ public final class AzureStorageSettings {
getValue(settings, groupName, ACCOUNT_SETTING),
getValue(settings, groupName, KEY_SETTING),
getValue(settings, groupName, TIMEOUT_SETTING),
getValue(settings, groupName, DEFAULT_SETTING))
getValue(settings, groupName, DEFAULT_SETTING),
getValue(settings, groupName, MAX_RETRIES_SETTING))
);
}
return storageSettings;
@ -128,7 +142,8 @@ public final class AzureStorageSettings {
} else if (settings.size() == 1) {
// the only storage settings belong (implicitly) to the default primary storage
AzureStorageSettings storage = settings.get(0);
return new AzureStorageSettings(storage.getName(), storage.getAccount(), storage.getKey(), storage.getTimeout(), true);
return new AzureStorageSettings(storage.getName(), storage.getAccount(), storage.getKey(), storage.getTimeout(), true,
storage.getMaxRetries());
} else {
AzureStorageSettings primary = null;
for (AzureStorageSettings setting : settings) {

28
plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.cloud.azure.storage;
import com.microsoft.azure.storage.LocationMode;
import com.microsoft.azure.storage.RetryExponentialRetry;
import com.microsoft.azure.storage.blob.CloudBlobClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;
@ -28,7 +29,9 @@ import java.net.URI;
import java.net.URISyntaxException;
import static org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl.blobNameFromUri;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
public class AzureStorageServiceTests extends ESTestCase {
@ -143,6 +146,31 @@ public class AzureStorageServiceTests extends ESTestCase {
assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(nullValue()));
}
public void testGetSelectedClientBackoffPolicy() {
Settings timeoutSettings = Settings.builder()
.put("cloud.azure.storage.azure.account", "myaccount")
.put("cloud.azure.storage.azure.key", "mykey")
.build();
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings);
CloudBlobClient client1 = azureStorageService.getSelectedClient("azure", LocationMode.PRIMARY_ONLY);
assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), is(notNullValue()));
assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), instanceOf(RetryExponentialRetry.class));
}
public void testGetSelectedClientBackoffPolicyNbRetries() {
Settings timeoutSettings = Settings.builder()
.put("cloud.azure.storage.azure.account", "myaccount")
.put("cloud.azure.storage.azure.key", "mykey")
.put("cloud.azure.storage.azure.max_retries", 7)
.build();
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings);
CloudBlobClient client1 = azureStorageService.getSelectedClient("azure", LocationMode.PRIMARY_ONLY);
assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), is(notNullValue()));
assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), instanceOf(RetryExponentialRetry.class));
}
/**
* This internal class just overload createClient method which is called by AzureStorageServiceImpl.doStart()
*/