honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-24 17:09:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support global repositories.azure. settings

Browse Source 
All those repository settings can also be defined globally in `elasticsearch.yml` file using prefix `repositories.azure.`. For example:

```yml
repositories.azure:
    container: backup-container
    base_path: backups
    chunk_size: 32m
    compress": true
```

Closes #13776.
This commit is contained in:
David Pilato 2015-12-01 11:35:56 +01:00
parent 60cbb2d7bc
commit a49fe189b0
9 changed files with 85 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
docs/plugins/repository-azure.asciidoc
View File

@ -64,6 +64,8 @@ cloud:
`my_account1` is the default account which will be used by a repository unless you set an explicit one.
[[repository-azure-repository-settings]]
===== Repository settings
The Azure repository supports following settings:
@ -155,6 +157,22 @@ client.admin().cluster().preparePutRepository("my_backup_java1")
).get();
----
[[repository-azure-global-settings]]
===== Global repositories settings
All those repository settings can also be defined globally in `elasticsearch.yml` file using prefix
`repositories.azure.`. For example:
[source,yaml]
----
repositories.azure:
container: backup-container
base_path: backups
chunk_size: 32m
compress": true
----
[[repository-azure-validation]]
===== Repository validation rules

11
plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.cloud.azure.blobstore;
import com.microsoft.azure.storage.LocationMode;
import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.BlobPath;
@ -31,6 +32,7 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.RepositoryName;
import org.elasticsearch.repositories.RepositorySettings;
import org.elasticsearch.repositories.azure.AzureRepository.Defaults;
import java.io.InputStream;
import java.io.OutputStream;
@ -38,8 +40,7 @@ import java.net.URISyntaxException;
import java.util.Locale;
import java.util.Map;
import static org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage.CONTAINER;
import static org.elasticsearch.repositories.azure.AzureRepository.CONTAINER_DEFAULT;
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepositorySettings;
import static org.elasticsearch.repositories.azure.AzureRepository.Repository;
public class AzureBlobStore extends AbstractComponent implements BlobStore {
@ -56,13 +57,13 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore {
AzureStorageService client) throws URISyntaxException, StorageException {
super(settings);
this.client = client.start();
this.container = repositorySettings.settings().get("container", settings.get(CONTAINER, CONTAINER_DEFAULT));
this.container = getRepositorySettings(repositorySettings, Repository.CONTAINER, Storage.CONTAINER, Defaults.CONTAINER);
this.repositoryName = name.getName();
// NOTE: null account means to use the first one specified in config
this.accountName = repositorySettings.settings().get(Repository.ACCOUNT, null);
this.accountName = getRepositorySettings(repositorySettings, Repository.ACCOUNT, Storage.ACCOUNT, null);
String modeStr = repositorySettings.settings().get(Repository.LOCATION_MODE, null);
String modeStr = getRepositorySettings(repositorySettings, Repository.LOCATION_MODE, Storage.LOCATION_MODE, null);
if (modeStr == null) {
this.locMode = LocationMode.PRIMARY_ONLY;
} else {

7
plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
View File

@ -37,9 +37,12 @@ public interface AzureStorageService {
final class Storage {
public static final String PREFIX = "cloud.azure.storage.";
@Deprecated
public static final String ACCOUNT = "cloud.azure.storage.account";
public static final String ACCOUNT_DEPRECATED = "cloud.azure.storage.account";
@Deprecated
public static final String KEY = "cloud.azure.storage.key";
public static final String KEY_DEPRECATED = "cloud.azure.storage.key";
public static final String ACCOUNT = "repositories.azure.account";
public static final String LOCATION_MODE = "repositories.azure.location_mode";
public static final String CONTAINER = "repositories.azure.container";
public static final String BASE_PATH = "repositories.azure.base_path";
public static final String CHUNK_SIZE = "repositories.azure.chunk_size";

32
plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
View File

@ -24,6 +24,8 @@ import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.repositories.RepositorySettings;
import java.util.HashMap;
import java.util.Map;
@ -73,11 +75,11 @@ public class AzureStorageSettings {
Map<String, AzureStorageSettings> secondaryStorage = new HashMap<>();
// We check for deprecated settings
String account = settings.get(Storage.ACCOUNT);
String key = settings.get(Storage.KEY);
String account = settings.get(Storage.ACCOUNT_DEPRECATED);
String key = settings.get(Storage.KEY_DEPRECATED);
if (account != null) {
logger.warn("[{}] and [{}] have been deprecated. Use now [{}xxx.account] and [{}xxx.key] where xxx is any name",
Storage.ACCOUNT, Storage.KEY, Storage.PREFIX, Storage.PREFIX);
Storage.ACCOUNT_DEPRECATED, Storage.KEY_DEPRECATED, Storage.PREFIX, Storage.PREFIX);
primaryStorage = new AzureStorageSettings(null, account, key);
} else {
Settings storageSettings = settings.getByPrefix(Storage.PREFIX);
@ -119,4 +121,28 @@ public class AzureStorageSettings {
return Tuple.tuple(primaryStorage, secondaryStorage);
}
public static String getRepositorySettings(RepositorySettings repositorySettings,
String repositorySettingName,
String repositoriesSettingName,
String defaultValue) {
return repositorySettings.settings().get(repositorySettingName,
repositorySettings.globalSettings().get(repositoriesSettingName, defaultValue));
}
public static ByteSizeValue getRepositorySettingsAsBytesSize(RepositorySettings repositorySettings,
String repositorySettingName,
String repositoriesSettingName,
ByteSizeValue defaultValue) {
return repositorySettings.settings().getAsBytesSize(repositorySettingName,
repositorySettings.globalSettings().getAsBytesSize(repositoriesSettingName, defaultValue));
}
public static Boolean getRepositorySettingsAsBoolean(RepositorySettings repositorySettings,
String repositorySettingName,
String repositoriesSettingName,
Boolean defaultValue) {
return repositorySettings.settings().getAsBoolean(repositorySettingName,
repositorySettings.globalSettings().getAsBoolean(repositoriesSettingName, defaultValue));
}
}

5
plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilter.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.cloud.azure.storage;
import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
@ -30,6 +31,8 @@ public class AzureStorageSettingsFilter extends AbstractComponent {
public AzureStorageSettingsFilter(Settings settings, SettingsFilter settingsFilter) {
super(settings);
// Cloud storage API settings needed to be hidden
settingsFilter.addFilter("cloud.azure.storage.*");
settingsFilter.addFilter(Storage.PREFIX + "*.account");
settingsFilter.addFilter(Storage.PREFIX + "*.key");
settingsFilter.addFilter(Storage.ACCOUNT);
}
}

25
plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
View File

@ -43,6 +43,10 @@ import java.net.URISyntaxException;
import java.util.List;
import java.util.Locale;
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepositorySettings;
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepositorySettingsAsBoolean;
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getRepositorySettingsAsBytesSize;
/**
* Azure file system implementation of the BlobStoreRepository
* <p>
@ -57,7 +61,13 @@ import java.util.Locale;
public class AzureRepository extends BlobStoreRepository {
public final static String TYPE = "azure";
public final static String CONTAINER_DEFAULT = "elasticsearch-snapshots";
static public final class Defaults {
public static final String CONTAINER = "elasticsearch-snapshots";
public static final ByteSizeValue CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB);
public static final Boolean COMPRESS = false;
}
static public final class Repository {
public static final String ACCOUNT = "account";
@ -83,21 +93,18 @@ public class AzureRepository extends BlobStoreRepository {
AzureBlobStore azureBlobStore) throws IOException, URISyntaxException, StorageException {
super(name.getName(), repositorySettings, indexShardRepository);
String container = repositorySettings.settings().get(Repository.CONTAINER,
settings.get(Storage.CONTAINER, CONTAINER_DEFAULT));
String container = getRepositorySettings(repositorySettings, Repository.CONTAINER, Storage.CONTAINER, Defaults.CONTAINER);
this.blobStore = azureBlobStore;
this.chunkSize = repositorySettings.settings().getAsBytesSize(Repository.CHUNK_SIZE,
settings.getAsBytesSize(Storage.CHUNK_SIZE, new ByteSizeValue(64, ByteSizeUnit.MB)));
this.chunkSize = getRepositorySettingsAsBytesSize(repositorySettings, Repository.CHUNK_SIZE, Storage.CHUNK_SIZE, Defaults.CHUNK_SIZE);
if (this.chunkSize.getMb() > 64) {
logger.warn("azure repository does not support yet size > 64mb. Fall back to 64mb.");
this.chunkSize = new ByteSizeValue(64, ByteSizeUnit.MB);
}
this.compress = repositorySettings.settings().getAsBoolean(Repository.COMPRESS,
settings.getAsBoolean(Storage.COMPRESS, false));
String modeStr = repositorySettings.settings().get(Repository.LOCATION_MODE, null);
this.compress = getRepositorySettingsAsBoolean(repositorySettings, Repository.COMPRESS, Storage.COMPRESS, Defaults.COMPRESS);
String modeStr = getRepositorySettings(repositorySettings, Repository.LOCATION_MODE, Storage.LOCATION_MODE, null);
if (modeStr != null) {
LocationMode locationMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT));
if (locationMode == LocationMode.SECONDARY_ONLY) {
@ -109,7 +116,7 @@ public class AzureRepository extends BlobStoreRepository {
readonly = false;
}
String basePath = repositorySettings.settings().get(Repository.BASE_PATH, null);
String basePath = getRepositorySettings(repositorySettings, Repository.BASE_PATH, Storage.BASE_PATH, null);
if (Strings.hasLength(basePath)) {
// Remove starting / if any

4
plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java
View File

@ -81,8 +81,8 @@ public abstract class AbstractAzureRepositoryServiceTestCase extends AbstractAzu
.put(Storage.CONTAINER, "snapshots");
// We use sometime deprecated settings in tests
builder.put(Storage.ACCOUNT, "mock_azure_account")
.put(Storage.KEY, "mock_azure_key");
builder.put(Storage.ACCOUNT_DEPRECATED, "mock_azure_account")
.put(Storage.KEY_DEPRECATED, "mock_azure_key");
return builder.build();
}

7
plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTest.java
View File

@ -29,8 +29,7 @@ import org.elasticsearch.test.rest.FakeRestRequest;
import java.io.IOException;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.contains;
public class AzureStorageSettingsFilterTest extends ESTestCase {
final static Settings settings = Settings.builder()
@ -52,7 +51,7 @@ public class AzureStorageSettingsFilterTest extends ESTestCase {
// Test using direct filtering
Settings filteredSettings = SettingsFilter.filterSettings(settingsFilter.getPatterns(), settings);
assertThat(filteredSettings.getAsMap().keySet(), is(empty()));
assertThat(filteredSettings.getAsMap().keySet(), contains("cloud.azure.storage.azure1.default"));
// Test using toXContent filtering
RestRequest request = new FakeRestRequest();
@ -63,7 +62,7 @@ public class AzureStorageSettingsFilterTest extends ESTestCase {
xContentBuilder.endObject();
String filteredSettingsString = xContentBuilder.string();
filteredSettings = Settings.builder().loadFromSource(filteredSettingsString).build();
assertThat(filteredSettings.getAsMap().keySet(), is(empty()));
assertThat(filteredSettings.getAsMap().keySet(), contains("cloud.azure.storage.azure1.default"));
}
}

4
plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTest.java
View File

@ -68,8 +68,8 @@ public class AzureSettingsParserTest extends LuceneTestCase {
public void testDeprecatedSettings() {
Settings settings = Settings.builder()
.put(Storage.ACCOUNT, "myaccount1")
.put(Storage.KEY, "mykey1")
.put(Storage.ACCOUNT_DEPRECATED, "myaccount1")
.put(Storage.KEY_DEPRECATED, "mykey1")
.build();
Tuple<AzureStorageSettings, Map<String, AzureStorageSettings>> tuple = AzureStorageSettings.parse(settings);