honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add SAS Token Authentication Support to Azure Repo Plugin (#42982) (#43618) 

* Added setting for SAS token
* Added support for the token in tests
* Relates #42117
This commit is contained in:
Armin Braun 2019-06-26 13:43:32 +02:00 committed by GitHub
parent 31dc5b7d3a
commit 83067968ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 81 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/plugins/repository-azure.asciidoc
View File

@ -19,7 +19,11 @@ bin/elasticsearch-keystore add azure.client.default.account
bin/elasticsearch-keystore add azure.client.default.key
----------------------------------------------------------------
Where `account` is the azure account name and `key` the azure secret key.
Where `account` is the azure account name and `key` the azure secret key. Instead of an azure secret key under `key`, you can alternatively
define a shared access signatures (SAS) token under `sas_token` to use for authentication instead. When using an SAS token instead of an
account key, the SAS token must have read (r), write (w), list (l), and delete (d) permissions for the repository base path and
all its contents. These permissions need to be granted for the blob service (b) and apply to resource types service (s), container (c), and
object (o).
These settings are used by the repository's internal azure client.
Note that you can also define more than one account:
@ -29,14 +33,14 @@ Note that you can also define more than one account:
bin/elasticsearch-keystore add azure.client.default.account
bin/elasticsearch-keystore add azure.client.default.key
bin/elasticsearch-keystore add azure.client.secondary.account
bin/elasticsearch-keystore add azure.client.secondary.key
bin/elasticsearch-keystore add azure.client.secondary.sas_token
----------------------------------------------------------------
`default` is the default account name which will be used by a repository,
unless you set an explicit one in the
<<repository-azure-repository-settings, repository settings>>.
Both `account` and `key` storage settings are
The `account`, `key`, and `sas_token` storage settings are
{ref}/secure-settings.html#reloadable-secure-settings[reloadable]. After you
reload the settings, the internal azure clients, which are used to transfer the
snapshot, will utilize the latest settings from the keystore.

4
plugins/repository-azure/build.gradle
View File

@ -76,6 +76,7 @@ String azureAccount = System.getenv("azure_storage_account")
String azureKey = System.getenv("azure_storage_key")
String azureContainer = System.getenv("azure_storage_container")
String azureBasePath = System.getenv("azure_storage_base_path")
String azureSasToken = System.getenv("azure_storage_sas_token")
test {
exclude '**/AzureStorageCleanupThirdPartyTests.class'
@ -85,10 +86,11 @@ task thirdPartyTest(type: Test) {
include '**/AzureStorageCleanupThirdPartyTests.class'
systemProperty 'test.azure.account', azureAccount ? azureAccount : ""
systemProperty 'test.azure.key', azureKey ? azureKey : ""
systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : ""
systemProperty 'test.azure.container', azureContainer ? azureContainer : ""
systemProperty 'test.azure.base', azureBasePath ? azureBasePath : ""
}
if (azureAccount || azureKey || azureContainer || azureBasePath) {
if (azureAccount || azureKey || azureContainer || azureBasePath || azureSasToken) {
check.dependsOn(thirdPartyTest)
}

13
plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
View File

@ -29,12 +29,14 @@ String azureAccount = System.getenv("azure_storage_account")
String azureKey = System.getenv("azure_storage_key")
String azureContainer = System.getenv("azure_storage_container")
String azureBasePath = System.getenv("azure_storage_base_path")
String azureSasToken = System.getenv("azure_storage_sas_token")
if (!azureAccount && !azureKey && !azureContainer && !azureBasePath) {
if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
azureAccount = 'azure_integration_test_account'
azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
azureContainer = 'container_test'
azureBasePath = 'integration_test'
azureSasToken = ''
useFixture = true
}
@ -63,7 +65,14 @@ integTest {
testClusters.integTest {
plugin file(project(':plugins:repository-azure').bundlePlugin.archiveFile)
keystore 'azure.client.integration_test.account', azureAccount
keystore 'azure.client.integration_test.key', azureKey
if (azureKey != null && azureKey.isEmpty() == false) {
println "Using access key in external service tests."
keystore 'azure.client.integration_test.key', azureKey
}
if (azureSasToken != null && azureSasToken.isEmpty() == false) {
println "Using SAS token in external service tests."
keystore 'azure.client.integration_test.sas_token', azureSasToken
}
if (useFixture) {
tasks.integTest.dependsOn azureStorageFixture

1
plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
View File

@ -60,6 +60,7 @@ public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin, R
return Arrays.asList(
AzureStorageSettings.ACCOUNT_SETTING,
AzureStorageSettings.KEY_SETTING,
AzureStorageSettings.SAS_TOKEN_SETTING,
AzureStorageSettings.ENDPOINT_SUFFIX_SETTING,
AzureStorageSettings.TIMEOUT_SETTING,
AzureStorageSettings.MAX_RETRIES_SETTING,

2
plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
View File

@ -117,7 +117,7 @@ public class AzureStorageService {
}
private static CloudBlobClient createClient(AzureStorageSettings azureStorageSettings) throws InvalidKeyException, URISyntaxException {
final String connectionString = azureStorageSettings.buildConnectionString();
final String connectionString = azureStorageSettings.getConnectString();
return CloudStorageAccount.parse(connectionString).createCloudBlobClient();
}

57
plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.repositories.azure;
import com.microsoft.azure.storage.LocationMode;
import com.microsoft.azure.storage.RetryPolicy;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.settings.SecureSetting;
@ -53,6 +54,10 @@ final class AzureStorageSettings {
public static final AffixSetting<SecureString> KEY_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "key",
key -> SecureSetting.secureString(key, null));
/** Azure SAS token */
public static final AffixSetting<SecureString> SAS_TOKEN_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "sas_token",
key -> SecureSetting.secureString(key, null));
/** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */
public static final Setting<Integer> MAX_RETRIES_SETTING =
Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "max_retries",
@ -82,7 +87,7 @@ final class AzureStorageSettings {
PROXY_HOST_SETTING);
private final String account;
private final String key;
private final String connectString;
private final String endpointSuffix;
private final TimeValue timeout;
private final int maxRetries;
@ -90,10 +95,10 @@ final class AzureStorageSettings {
private final LocationMode locationMode;
// copy-constructor
private AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries, Proxy proxy,
LocationMode locationMode) {
private AzureStorageSettings(String account, String connectString, String endpointSuffix, TimeValue timeout, int maxRetries,
Proxy proxy, LocationMode locationMode) {
this.account = account;
this.key = key;
this.connectString = connectString;
this.endpointSuffix = endpointSuffix;
this.timeout = timeout;
this.maxRetries = maxRetries;
@ -101,10 +106,10 @@ final class AzureStorageSettings {
this.locationMode = locationMode;
}
AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries,
Proxy.Type proxyType, String proxyHost, Integer proxyPort) {
private AzureStorageSettings(String account, String key, String sasToken, String endpointSuffix, TimeValue timeout, int maxRetries,
Proxy.Type proxyType, String proxyHost, Integer proxyPort) {
this.account = account;
this.key = key;
this.connectString = buildConnectString(account, key, sasToken, endpointSuffix);
this.endpointSuffix = endpointSuffix;
this.timeout = timeout;
this.maxRetries = maxRetries;
@ -145,13 +150,26 @@ final class AzureStorageSettings {
return proxy;
}
public String buildConnectionString() {
public String getConnectString() {
return connectString;
}
private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix) {
final boolean hasSasToken = Strings.hasText(sasToken);
final boolean hasKey = Strings.hasText(key);
if (hasSasToken == false && hasKey == false) {
throw new SettingsException("Neither a secret key nor a shared access token was set.");
}
if (hasSasToken && hasKey) {
throw new SettingsException("Both a secret as well as a shared access token were set.");
}
final StringBuilder connectionStringBuilder = new StringBuilder();
connectionStringBuilder.append("DefaultEndpointsProtocol=https")
.append(";AccountName=")
.append(account)
.append(";AccountKey=")
.append(key);
connectionStringBuilder.append("DefaultEndpointsProtocol=https").append(";AccountName=").append(account);
if (hasKey) {
connectionStringBuilder.append(";AccountKey=").append(key);
} else {
connectionStringBuilder.append(";SharedAccessSignature=").append(sasToken);
}
if (Strings.hasText(endpointSuffix)) {
connectionStringBuilder.append(";EndpointSuffix=").append(endpointSuffix);
}
@ -166,7 +184,6 @@ final class AzureStorageSettings {
public String toString() {
final StringBuilder sb = new StringBuilder("AzureStorageSettings{");
sb.append("account='").append(account).append('\'');
sb.append(", key='").append(key).append('\'');
sb.append(", timeout=").append(timeout);
sb.append(", endpointSuffix='").append(endpointSuffix).append('\'');
sb.append(", maxRetries=").append(maxRetries);
@ -201,8 +218,9 @@ final class AzureStorageSettings {
/** Parse settings for a single client. */
private static AzureStorageSettings getClientSettings(Settings settings, String clientName) {
try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING);
SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) {
return new AzureStorageSettings(account.toString(), key.toString(),
SecureString key = getConfigValue(settings, clientName, KEY_SETTING);
SecureString sasToken = getConfigValue(settings, clientName, SAS_TOKEN_SETTING)) {
return new AzureStorageSettings(account.toString(), key.toString(), sasToken.toString(),
getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING),
getValue(settings, clientName, TIMEOUT_SETTING),
getValue(settings, clientName, MAX_RETRIES_SETTING),
@ -228,10 +246,9 @@ final class AzureStorageSettings {
LocationMode locationMode) {
final MapBuilder<String, AzureStorageSettings> mapBuilder = new MapBuilder<>();
for (final Map.Entry<String, AzureStorageSettings> entry : clientsSettings.entrySet()) {
final AzureStorageSettings azureSettings = new AzureStorageSettings(entry.getValue().account, entry.getValue().key,
entry.getValue().endpointSuffix, entry.getValue().timeout, entry.getValue().maxRetries, entry.getValue().proxy,
locationMode);
mapBuilder.put(entry.getKey(), azureSettings);
mapBuilder.put(entry.getKey(),
new AzureStorageSettings(entry.getValue().account, entry.getValue().connectString, entry.getValue().endpointSuffix,
entry.getValue().timeout, entry.getValue().maxRetries, entry.getValue().proxy, locationMode));
}
return mapBuilder.immutableMap();
}

14
plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.repositories.azure;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.SecureSettings;
import org.elasticsearch.common.settings.Settings;
@ -42,13 +43,22 @@ public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyReposi
@Override
protected SecureSettings credentials() {
assertThat(System.getProperty("test.azure.account"), not(blankOrNullString()));
assertThat(System.getProperty("test.azure.key"), not(blankOrNullString()));
final boolean hasSasToken = Strings.hasText(System.getProperty("test.azure.sas_token"));
if (hasSasToken == false) {
assertThat(System.getProperty("test.azure.key"), not(blankOrNullString()));
} else {
assertThat(System.getProperty("test.azure.key"), blankOrNullString());
}
assertThat(System.getProperty("test.azure.container"), not(blankOrNullString()));
assertThat(System.getProperty("test.azure.base"), not(blankOrNullString()));
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString("azure.client.default.account", System.getProperty("test.azure.account"));
secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key"));
if (hasSasToken) {
secureSettings.setString("azure.client.default.sas_token", System.getProperty("test.azure.sas_token"));
} else {
secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key"));
}
return secureSettings;
}

12
plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java
View File

@ -158,15 +158,21 @@ public class AzureStorageServiceTests extends ESTestCase {
secureSettings2.setString("azure.client.azure1.account", "myaccount1");
// missing key
final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build();
final MockSecureSettings secureSettings3 = new MockSecureSettings();
secureSettings3.setString("azure.client.azure1.account", "myaccount3");
secureSettings3.setString("azure.client.azure1.key", encodeKey("mykey33"));
secureSettings3.setString("azure.client.azure1.sas_token", encodeKey("mysasToken33"));
final Settings settings3 = Settings.builder().setSecureSettings(secureSettings3).build();
try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
final AzureStorageService azureStorageService = plugin.azureStoreService;
final CloudBlobClient client11 = azureStorageService.client("azure1").v1();
assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net"));
plugin.reload(settings2);
final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2));
assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set."));
final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3));
assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set."));
// existing client untouched
assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net"));
final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure1"));
assertThat(e.getMessage(), is("Invalid azure client settings with name [azure1]"));
}
}