honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update settings filter and remove component settings from AbstractComponent 

Update settings filter to match elasticsearch/elasticsearch#9748
Remove component settings from AbstractComponent as seen in elasticsearch/elasticsearch#9919

Closes #71.
Closes #72.
This commit is contained in:
David Pilato 2015-02-27 11:15:45 +01:00
parent b7c44087cf
commit cd7b8d4840
22 changed files with 301 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
README.md
View File

@ -64,13 +64,13 @@ How to start (short story)
Azure credential API settings Azure credential API settings
----------------------------- -----------------------------
The following are a list of settings (prefixed with `cloud.azure.management`) that can further control the discovery: The following are a list of settings that can further control the credential API:
* `keystore.path`: /path/to/keystore * `cloud.azure.management.keystore.path`: /path/to/keystore
* `keystore.type`: `pkcs12`, `jceks` or `jks`. Defaults to `pkcs12`. * `cloud.azure.management.keystore.type`: `pkcs12`, `jceks` or `jks`. Defaults to `pkcs12`.
* `keystore.password`: your_password for the keystore * `cloud.azure.management.keystore.password`: your_password for the keystore
* `subscription.id`: your_azure_subscription_id * `cloud.azure.management.subscription.id`: your_azure_subscription_id
* `cloud.service.name`: your_azure_cloud_service_name * `cloud.azure.management.cloud.service.name`: your_azure_cloud_service_name
Note that in previous versions, it was: Note that in previous versions, it was:
@ -86,16 +86,16 @@ cloud:
Advanced settings Advanced settings
----------------- -----------------
The following are a list of settings (prefixed with `discovery.azure`) that can further control the discovery: The following are a list of settings that can further control the discovery:
* `host.type`: either `public_ip` or `private_ip` (default). Azure discovery will use the one you set to ping * `discovery.azure.host.type`: either `public_ip` or `private_ip` (default). Azure discovery will use the one you set to ping
other nodes. This feature was not documented before but was existing under `cloud.azure.host_type`. other nodes. This feature was not documented before but was existing under `cloud.azure.host_type`.
* `endpoint.name`: when using `public_ip` this setting is used to identify the endpoint name used to forward requests * `discovery.azure.endpoint.name`: when using `public_ip` this setting is used to identify the endpoint name used to forward requests
to elasticsearch (aka transport port name). Defaults to `elasticsearch`. In Azure management console, you could define to elasticsearch (aka transport port name). Defaults to `elasticsearch`. In Azure management console, you could define
an endpoint `elasticsearch` forwarding for example requests on public IP on port 8100 to the virtual machine on port 9300. an endpoint `elasticsearch` forwarding for example requests on public IP on port 8100 to the virtual machine on port 9300.
This feature was not documented before but was existing under `cloud.azure.port_name`. This feature was not documented before but was existing under `cloud.azure.port_name`.
* `deployment.name`: deployment name if any. Defaults to the value set with `cloud.azure.management.cloud.service.name`. * `discovery.azure.deployment.name`: deployment name if any. Defaults to the value set with `cloud.azure.management.cloud.service.name`.
* `deployment.slot`: either `staging` or `production` (default). * `discovery.azure.deployment.slot`: either `staging` or `production` (default).
For example: For example:
@ -477,8 +477,8 @@ Example using Java:
```java ```java
client.admin().cluster().preparePutRepository("my_backup3") client.admin().cluster().preparePutRepository("my_backup3")
.setType("azure").setSettings(ImmutableSettings.settingsBuilder() .setType("azure").setSettings(ImmutableSettings.settingsBuilder()
.put(AzureStorageService.Fields.CONTAINER, "backup_container") .put(Storage.CONTAINER, "backup_container")
.put(AzureStorageService.Fields.CHUNK_SIZE, new ByteSizeValue(32, ByteSizeUnit.MB)) .put(Storage.CHUNK_SIZE, new ByteSizeValue(32, ByteSizeUnit.MB))
).get(); ).get();
``` ```

75
src/main/java/org/elasticsearch/cloud/azure/AzureModule.java
View File

@ -20,10 +20,13 @@
package org.elasticsearch.cloud.azure; package org.elasticsearch.cloud.azure;
import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
import org.elasticsearch.cloud.azure.management.AzureComputeService; import org.elasticsearch.cloud.azure.management.AzureComputeService;
import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery;
import org.elasticsearch.cloud.azure.management.AzureComputeService.Management;
import org.elasticsearch.cloud.azure.management.AzureComputeServiceImpl; import org.elasticsearch.cloud.azure.management.AzureComputeServiceImpl;
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
@ -63,7 +66,7 @@ public class AzureModule extends AbstractModule {
if (isDiscoveryReady(settings, logger)) { if (isDiscoveryReady(settings, logger)) {
logger.debug("starting azure discovery service"); logger.debug("starting azure discovery service");
bind(AzureComputeService.class) bind(AzureComputeService.class)
.to(settings.getAsClass("cloud.azure.api.impl", AzureComputeServiceImpl.class)) .to(settings.getAsClass(Management.API_IMPLEMENTATION, AzureComputeServiceImpl.class))
.asEagerSingleton(); .asEagerSingleton();
} }
@ -71,7 +74,7 @@ public class AzureModule extends AbstractModule {
if (isSnapshotReady(settings, logger)) { if (isSnapshotReady(settings, logger)) {
logger.debug("starting azure repository service"); logger.debug("starting azure repository service");
bind(AzureStorageService.class) bind(AzureStorageService.class)
.to(settings.getAsClass("repositories.azure.api.impl", AzureStorageServiceImpl.class)) .to(settings.getAsClass(Storage.API_IMPLEMENTATION, AzureStorageServiceImpl.class))
.asEagerSingleton(); .asEagerSingleton();
} }
} }
@ -102,22 +105,22 @@ public class AzureModule extends AbstractModule {
} }
if ( // We check new parameters if ( // We check new parameters
(isPropertyMissing(settings, "cloud.azure.management." + AzureComputeService.Fields.SUBSCRIPTION_ID) || (isPropertyMissing(settings, Management.SUBSCRIPTION_ID) ||
isPropertyMissing(settings, "cloud.azure.management." + AzureComputeService.Fields.SERVICE_NAME) || isPropertyMissing(settings, Management.SERVICE_NAME) ||
isPropertyMissing(settings, "cloud.azure.management." + AzureComputeService.Fields.KEYSTORE_PATH) || isPropertyMissing(settings, Management.KEYSTORE_PATH) ||
isPropertyMissing(settings, "cloud.azure.management." + AzureComputeService.Fields.KEYSTORE_PASSWORD)) isPropertyMissing(settings, Management.KEYSTORE_PASSWORD))
// We check deprecated // We check deprecated
&& (isPropertyMissing(settings, "cloud.azure." + AzureComputeService.Fields.SUBSCRIPTION_ID_DEPRECATED) || && (isPropertyMissing(settings, Management.SUBSCRIPTION_ID_DEPRECATED) ||
isPropertyMissing(settings, "cloud.azure." + AzureComputeService.Fields.SERVICE_NAME_DEPRECATED) || isPropertyMissing(settings, Management.SERVICE_NAME_DEPRECATED) ||
isPropertyMissing(settings, "cloud.azure." + AzureComputeService.Fields.KEYSTORE_DEPRECATED) || isPropertyMissing(settings, Management.KEYSTORE_DEPRECATED) ||
isPropertyMissing(settings, "cloud.azure." + AzureComputeService.Fields.PASSWORD_DEPRECATED)) isPropertyMissing(settings, Management.PASSWORD_DEPRECATED))
) { ) {
logger.debug("one or more azure discovery settings are missing. " + logger.debug("one or more azure discovery settings are missing. " +
"Check elasticsearch.yml file and `cloud.azure.management`. Should have [{}], [{}], [{}] and [{}].", "Check elasticsearch.yml file. Should have [{}], [{}], [{}] and [{}].",
AzureComputeService.Fields.SUBSCRIPTION_ID, Management.SUBSCRIPTION_ID,
AzureComputeService.Fields.SERVICE_NAME, Management.SERVICE_NAME,
AzureComputeService.Fields.KEYSTORE_PATH, Management.KEYSTORE_PATH,
AzureComputeService.Fields.KEYSTORE_PASSWORD); Management.KEYSTORE_PASSWORD);
return false; return false;
} }
@ -137,13 +140,13 @@ public class AzureModule extends AbstractModule {
return false; return false;
} }
if ((isPropertyMissing(settings, "cloud.azure.storage." + AzureStorageService.Fields.ACCOUNT) || if ((isPropertyMissing(settings, Storage.ACCOUNT) ||
isPropertyMissing(settings, "cloud.azure.storage." + AzureStorageService.Fields.KEY)) && isPropertyMissing(settings, Storage.KEY)) &&
(isPropertyMissing(settings, "cloud.azure." + AzureStorageService.Fields.ACCOUNT_DEPRECATED) || (isPropertyMissing(settings, Storage.ACCOUNT_DEPRECATED) ||
isPropertyMissing(settings, "cloud.azure." + AzureStorageService.Fields.KEY_DEPRECATED))) { isPropertyMissing(settings, Storage.KEY_DEPRECATED))) {
logger.debug("azure repository is not set [using cloud.azure.storage.{}] and [cloud.azure.storage.{}] properties", logger.debug("azure repository is not set using [{}] and [{}] properties",
AzureStorageService.Fields.ACCOUNT, Storage.ACCOUNT,
AzureStorageService.Fields.KEY); Storage.KEY);
return false; return false;
} }
@ -171,24 +174,16 @@ public class AzureModule extends AbstractModule {
public static void checkDeprecated(Settings settings, ESLogger logger) { public static void checkDeprecated(Settings settings, ESLogger logger) {
// Cloud services are disabled // Cloud services are disabled
if (isCloudReady(settings)) { if (isCloudReady(settings)) {
checkDeprecatedSettings(settings, "cloud.azure." + AzureStorageService.Fields.ACCOUNT_DEPRECATED, checkDeprecatedSettings(settings, Storage.ACCOUNT_DEPRECATED, Storage.ACCOUNT, logger);
"cloud.azure.storage." + AzureStorageService.Fields.ACCOUNT, logger); checkDeprecatedSettings(settings, Storage.KEY_DEPRECATED, Storage.KEY, logger);
checkDeprecatedSettings(settings, "cloud.azure." + AzureStorageService.Fields.KEY_DEPRECATED,
"cloud.azure.storage." + AzureStorageService.Fields.KEY, logger);
// TODO Remove in 3.0.0 // TODO Remove in 3.0.0
checkDeprecatedSettings(settings, "cloud.azure." + AzureComputeService.Fields.KEYSTORE_DEPRECATED, checkDeprecatedSettings(settings, Management.KEYSTORE_DEPRECATED, Management.KEYSTORE_PATH, logger);
"cloud.azure.management." + AzureComputeService.Fields.KEYSTORE_PATH, logger); checkDeprecatedSettings(settings, Management.PASSWORD_DEPRECATED, Management.KEYSTORE_PASSWORD, logger);
checkDeprecatedSettings(settings, "cloud.azure." + AzureComputeService.Fields.PASSWORD_DEPRECATED, checkDeprecatedSettings(settings, Management.SERVICE_NAME_DEPRECATED, Management.SERVICE_NAME, logger);
"cloud.azure.management." + AzureComputeService.Fields.KEYSTORE_PASSWORD, logger); checkDeprecatedSettings(settings, Management.SUBSCRIPTION_ID_DEPRECATED, Management.SUBSCRIPTION_ID, logger);
checkDeprecatedSettings(settings, "cloud.azure." + AzureComputeService.Fields.SERVICE_NAME_DEPRECATED, checkDeprecatedSettings(settings, Discovery.HOST_TYPE_DEPRECATED, Discovery.HOST_TYPE, logger);
"cloud.azure.management." + AzureComputeService.Fields.SERVICE_NAME, logger); checkDeprecatedSettings(settings, Discovery.PORT_NAME_DEPRECATED, Discovery.ENDPOINT_NAME, logger);
checkDeprecatedSettings(settings, "cloud.azure." + AzureComputeService.Fields.SUBSCRIPTION_ID_DEPRECATED,
"cloud.azure.management." + AzureComputeService.Fields.SUBSCRIPTION_ID, logger);
checkDeprecatedSettings(settings, "cloud.azure." + AzureComputeService.Fields.HOST_TYPE_DEPRECATED,
"discovery.azure." + AzureComputeService.Fields.HOST_TYPE, logger);
checkDeprecatedSettings(settings, "cloud.azure." + AzureComputeService.Fields.PORT_NAME_DEPRECATED,
"discovery.azure." + AzureComputeService.Fields.ENDPOINT_NAME, logger);
} }
} }

60
src/main/java/org/elasticsearch/cloud/azure/AzureSettingsFilter.java
View File

@ -1,60 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cloud.azure;
import org.elasticsearch.cloud.azure.management.AzureComputeService;
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.SettingsFilter;
/**
* Filtering cloud.azure.* settings
*/
public class AzureSettingsFilter implements SettingsFilter.Filter {
@Override
public void filter(ImmutableSettings.Builder settings) {
// Cloud global settings
settings.remove("cloud.azure." + AzureComputeService.Fields.REFRESH);
// Cloud management API settings
settings.remove("cloud.azure.management." + AzureComputeService.Fields.KEYSTORE_PATH);
settings.remove("cloud.azure.management." + AzureComputeService.Fields.KEYSTORE_PASSWORD);
settings.remove("cloud.azure.management." + AzureComputeService.Fields.KEYSTORE_TYPE);
settings.remove("cloud.azure.management." + AzureComputeService.Fields.SUBSCRIPTION_ID);
settings.remove("cloud.azure.management." + AzureComputeService.Fields.SERVICE_NAME);
// Deprecated Cloud management API settings
// TODO Remove in 3.0.0
settings.remove("cloud.azure." + AzureComputeService.Fields.KEYSTORE_DEPRECATED);
settings.remove("cloud.azure." + AzureComputeService.Fields.PASSWORD_DEPRECATED);
settings.remove("cloud.azure." + AzureComputeService.Fields.SUBSCRIPTION_ID_DEPRECATED);
settings.remove("cloud.azure." + AzureComputeService.Fields.SERVICE_NAME_DEPRECATED);
// Cloud storage API settings
settings.remove("cloud.azure.storage." + AzureStorageService.Fields.ACCOUNT);
settings.remove("cloud.azure.storage." + AzureStorageService.Fields.KEY);
// Deprecated Cloud storage API settings
// TODO Remove in 3.0.0
settings.remove("cloud.azure." + AzureStorageService.Fields.ACCOUNT_DEPRECATED);
settings.remove("cloud.azure." + AzureStorageService.Fields.KEY_DEPRECATED);
}
}

7
src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
View File

@ -29,10 +29,12 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.RepositoryName; import org.elasticsearch.repositories.RepositoryName;
import org.elasticsearch.repositories.RepositorySettings; import org.elasticsearch.repositories.RepositorySettings;
import org.elasticsearch.repositories.azure.AzureRepository;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import static org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage.CONTAINER;
import static org.elasticsearch.repositories.azure.AzureRepository.CONTAINER_DEFAULT;
/** /**
* *
*/ */
@ -48,8 +50,7 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore {
AzureStorageService client) throws URISyntaxException, StorageException { AzureStorageService client) throws URISyntaxException, StorageException {
super(settings); super(settings);
this.client = client; this.client = client;
this.container = repositorySettings.settings().get(AzureStorageService.Fields.CONTAINER, this.container = repositorySettings.settings().get("container", settings.get(CONTAINER, CONTAINER_DEFAULT));
componentSettings.get(AzureStorageService.Fields.CONTAINER, AzureRepository.CONTAINER_DEFAULT));
this.repositoryName = name.getName(); this.repositoryName = name.getName();
} }

49
src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java
View File

@ -26,36 +26,43 @@ import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDeta
*/ */
public interface AzureComputeService { public interface AzureComputeService {
static public final class Fields { static public final class Management {
// Deprecated azure management cloud settings // Deprecated azure management cloud settings
@Deprecated @Deprecated
public static final String SUBSCRIPTION_ID_DEPRECATED = "subscription_id"; public static final String SUBSCRIPTION_ID_DEPRECATED = "cloud.azure.subscription_id";
@Deprecated @Deprecated
public static final String SERVICE_NAME_DEPRECATED = "service_name"; public static final String SERVICE_NAME_DEPRECATED = "cloud.azure.service_name";
@Deprecated @Deprecated
public static final String KEYSTORE_DEPRECATED = "keystore"; public static final String KEYSTORE_DEPRECATED = "cloud.azure.keystore";
@Deprecated @Deprecated
public static final String PASSWORD_DEPRECATED = "password"; public static final String PASSWORD_DEPRECATED = "cloud.azure.password";
@Deprecated
public static final String PORT_NAME_DEPRECATED = "port_name";
@Deprecated
public static final String HOST_TYPE_DEPRECATED = "host_type";
public static final String SUBSCRIPTION_ID = "subscription.id"; public static final String API_IMPLEMENTATION = "cloud.azure.management.api.impl";
public static final String SERVICE_NAME = "cloud.service.name";
public static final String SUBSCRIPTION_ID = "cloud.azure.management.subscription.id";
public static final String SERVICE_NAME = "cloud.azure.management.cloud.service.name";
// Keystore settings // Keystore settings
public static final String KEYSTORE_PATH = "keystore.path"; public static final String KEYSTORE_PATH = "cloud.azure.management.keystore.path";
public static final String KEYSTORE_PASSWORD = "keystore.password"; public static final String KEYSTORE_PASSWORD = "cloud.azure.management.keystore.password";
public static final String KEYSTORE_TYPE = "keystore.type"; public static final String KEYSTORE_TYPE = "cloud.azure.management.keystore.type";
public static final String REFRESH = "refresh_interval";
public static final String HOST_TYPE = "host.type";
public static final String ENDPOINT_NAME = "endpoint.name";
public static final String DEPLOYMENT_NAME = "deployment.name";
public static final String DEPLOYMENT_SLOT = "deployment.slot";
} }
static public final class Discovery {
// Deprecated azure discovery cloud settings
@Deprecated
public static final String PORT_NAME_DEPRECATED = "cloud.azure.port_name";
@Deprecated
public static final String HOST_TYPE_DEPRECATED = "cloud.azure.host_type";
@Deprecated
public static final String REFRESH_DEPRECATED = "cloud.azure.refresh_interval";
public static final String REFRESH = "discovery.azure.refresh_interval";
public static final String HOST_TYPE = "discovery.azure.host.type";
public static final String ENDPOINT_NAME = "discovery.azure.endpoint.name";
public static final String DEPLOYMENT_NAME = "discovery.azure.deployment.name";
public static final String DEPLOYMENT_SLOT = "discovery.azure.deployment.slot";
}
public HostedServiceGetDetailedResponse getServiceDetails(); public HostedServiceGetDetailedResponse getServiceDetails();
} }

20
src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java
View File

@ -28,16 +28,16 @@ import com.microsoft.windowsazure.management.configuration.ManagementConfigurati
import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cloud.azure.AzureServiceDisableException; import org.elasticsearch.cloud.azure.AzureServiceDisableException;
import org.elasticsearch.cloud.azure.AzureServiceRemoteException; import org.elasticsearch.cloud.azure.AzureServiceRemoteException;
import org.elasticsearch.cloud.azure.AzureSettingsFilter;
import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.*;
/** /**
* *
*/ */
@ -52,21 +52,19 @@ public class AzureComputeServiceImpl extends AbstractLifecycleComponent<AzureCom
private final String serviceName; private final String serviceName;
@Inject @Inject
public AzureComputeServiceImpl(Settings settings, SettingsFilter settingsFilter) { public AzureComputeServiceImpl(Settings settings) {
super(settings); super(settings);
settingsFilter.addFilter(new AzureSettingsFilter()); String subscriptionId = settings.get(SUBSCRIPTION_ID, settings.get(SUBSCRIPTION_ID_DEPRECATED));
String subscriptionId = componentSettings.get(Fields.SUBSCRIPTION_ID, settings.get("cloud.azure." + Fields.SUBSCRIPTION_ID_DEPRECATED)); serviceName = settings.get(Management.SERVICE_NAME, settings.get(Management.SERVICE_NAME_DEPRECATED));
String keystorePath = settings.get(KEYSTORE_PATH, settings.get(KEYSTORE_DEPRECATED));
serviceName = componentSettings.get(Fields.SERVICE_NAME, settings.get("cloud.azure." + Fields.SERVICE_NAME_DEPRECATED)); String keystorePassword = settings.get(KEYSTORE_PASSWORD, settings.get(PASSWORD_DEPRECATED));
String keystorePath = componentSettings.get(Fields.KEYSTORE_PATH, settings.get("cloud.azure." + Fields.KEYSTORE_DEPRECATED)); String strKeyStoreType = settings.get(KEYSTORE_TYPE, KeyStoreType.pkcs12.name());
String keystorePassword = componentSettings.get(Fields.KEYSTORE_PASSWORD, settings.get("cloud.azure." + Fields.PASSWORD_DEPRECATED));
String strKeyStoreType = componentSettings.get(Fields.KEYSTORE_TYPE, KeyStoreType.pkcs12.name());
KeyStoreType tmpKeyStoreType = KeyStoreType.pkcs12; KeyStoreType tmpKeyStoreType = KeyStoreType.pkcs12;
try { try {
tmpKeyStoreType = KeyStoreType.fromString(strKeyStoreType); tmpKeyStoreType = KeyStoreType.fromString(strKeyStoreType);
} catch (Exception e) { } catch (Exception e) {
logger.warn("wrong value for [{}]: [{}]. falling back to [{}]...", Fields.KEYSTORE_TYPE, logger.warn("wrong value for [{}]: [{}]. falling back to [{}]...", KEYSTORE_TYPE,
strKeyStoreType, KeyStoreType.pkcs12.name()); strKeyStoreType, KeyStoreType.pkcs12.name());
} }
KeyStoreType keystoreType = tmpKeyStoreType; KeyStoreType keystoreType = tmpKeyStoreType;

46
src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeSettingsFilter.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cloud.azure.management;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import static org.elasticsearch.cloud.azure.management.AzureComputeService.Management.*;
public class AzureComputeSettingsFilter extends AbstractComponent {
@Inject
protected AzureComputeSettingsFilter(Settings settings, SettingsFilter settingsFilter) {
super(settings);
// Cloud management API settings we need to hide
settingsFilter.addFilter(KEYSTORE_PATH);
settingsFilter.addFilter(KEYSTORE_PASSWORD);
settingsFilter.addFilter(KEYSTORE_TYPE);
settingsFilter.addFilter(SUBSCRIPTION_ID);
// Deprecated Cloud management API settings we need to hide
// TODO Remove in 3.0.0
settingsFilter.addFilter(KEYSTORE_DEPRECATED);
settingsFilter.addFilter(PASSWORD_DEPRECATED);
settingsFilter.addFilter(SUBSCRIPTION_ID_DEPRECATED);
}
}

20
src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
View File

@ -32,17 +32,19 @@ import java.net.URISyntaxException;
* @see AzureStorageServiceImpl for Azure REST API implementation * @see AzureStorageServiceImpl for Azure REST API implementation
*/ */
public interface AzureStorageService { public interface AzureStorageService {
static public final class Fields { static public final class Storage {
@Deprecated @Deprecated
public static final String ACCOUNT_DEPRECATED = "storage_account"; public static final String ACCOUNT_DEPRECATED = "cloud.azure.storage_account";
@Deprecated @Deprecated
public static final String KEY_DEPRECATED = "storage_key"; public static final String KEY_DEPRECATED = "cloud.azure.storage_key";
public static final String ACCOUNT = "account";
public static final String KEY = "key"; public static final String API_IMPLEMENTATION = "cloud.azure.storage.api.impl";
public static final String CONTAINER = "container"; public static final String ACCOUNT = "cloud.azure.storage.account";
public static final String BASE_PATH = "base_path"; public static final String KEY = "cloud.azure.storage.key";
public static final String CHUNK_SIZE = "chunk_size"; public static final String CONTAINER = "repositories.azure.container";
public static final String COMPRESS = "compress"; public static final String BASE_PATH = "repositories.azure.base_path";
public static final String CHUNK_SIZE = "repositories.azure.chunk_size";
public static final String COMPRESS = "repositories.azure.compress";
} }
boolean doesContainerExist(String container); boolean doesContainerExist(String container);

14
src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
View File

@ -23,14 +23,12 @@ import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.*; import com.microsoft.azure.storage.blob.*;
import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cloud.azure.AzureSettingsFilter;
import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
import org.elasticsearch.common.collect.ImmutableMap; import org.elasticsearch.common.collect.ImmutableMap;
import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositoryException;
import java.io.InputStream; import java.io.InputStream;
@ -38,6 +36,8 @@ import java.io.OutputStream;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import static org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage.*;
/** /**
* *
*/ */
@ -51,13 +51,11 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
private CloudBlobClient client; private CloudBlobClient client;
@Inject @Inject
public AzureStorageServiceImpl(Settings settings, SettingsFilter settingsFilter) { public AzureStorageServiceImpl(Settings settings) {
super(settings); super(settings);
settingsFilter.addFilter(new AzureSettingsFilter()); // We try to load storage API settings from `cloud.azure.`
account = settings.get(ACCOUNT, settings.get(ACCOUNT_DEPRECATED));
// We try to load storage API settings from `repositories.azure.` key = settings.get(KEY, settings.get(KEY_DEPRECATED));
account = componentSettings.get(Fields.ACCOUNT, settings.get("cloud.azure." + Fields.ACCOUNT_DEPRECATED));
key = componentSettings.get(Fields.KEY, settings.get("cloud.azure." + Fields.KEY_DEPRECATED));
blob = "http://" + account + ".blob.core.windows.net/"; blob = "http://" + account + ".blob.core.windows.net/";
try { try {

43
src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilter.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cloud.azure.storage;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import static org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage.*;
public class AzureStorageSettingsFilter extends AbstractComponent {
@Inject
protected AzureStorageSettingsFilter(Settings settings, SettingsFilter settingsFilter) {
super(settings);
// Cloud storage API settings needed to be hidden
settingsFilter.addFilter(ACCOUNT);
settingsFilter.addFilter(KEY);
// Deprecated Cloud storage API settings needed to be hidden
// TODO Remove in 3.0.0
settingsFilter.addFilter(ACCOUNT_DEPRECATED);
settingsFilter.addFilter(KEY_DEPRECATED);
}
}

4
src/main/java/org/elasticsearch/discovery/azure/AzureDiscovery.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.discovery.azure;
import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.node.DiscoveryNodeService;
import org.elasticsearch.cluster.settings.DynamicSettings; import org.elasticsearch.cluster.settings.DynamicSettings;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
@ -43,9 +42,8 @@ public class AzureDiscovery extends ZenDiscovery {
@Inject @Inject
public AzureDiscovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, public AzureDiscovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService,
ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService,
DiscoveryNodeService discoveryNodeService,
DiscoverySettings discoverySettings, ElectMasterService electMasterService, DynamicSettings dynamicSettings) { DiscoverySettings discoverySettings, ElectMasterService electMasterService, DynamicSettings dynamicSettings) {
super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService,
discoveryNodeService, pingService, electMasterService, discoverySettings, dynamicSettings); pingService, electMasterService, discoverySettings, dynamicSettings);
} }
} }

4
src/main/java/org/elasticsearch/discovery/azure/AzureDiscoveryModule.java
View File

@ -20,7 +20,7 @@
package org.elasticsearch.discovery.azure; package org.elasticsearch.discovery.azure;
import org.elasticsearch.cloud.azure.AzureModule; import org.elasticsearch.cloud.azure.AzureModule;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.cloud.azure.management.AzureComputeSettingsFilter;
import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
@ -35,7 +35,6 @@ public class AzureDiscoveryModule extends ZenDiscoveryModule {
protected final ESLogger logger; protected final ESLogger logger;
private Settings settings; private Settings settings;
@Inject
public AzureDiscoveryModule(Settings settings) { public AzureDiscoveryModule(Settings settings) {
super(); super();
this.logger = Loggers.getLogger(getClass(), settings); this.logger = Loggers.getLogger(getClass(), settings);
@ -47,6 +46,7 @@ public class AzureDiscoveryModule extends ZenDiscoveryModule {
@Override @Override
protected void bindDiscovery() { protected void bindDiscovery() {
bind(AzureComputeSettingsFilter.class).asEagerSingleton();
if (AzureModule.isDiscoveryReady(settings, logger)) { if (AzureModule.isDiscoveryReady(settings, logger)) {
bind(Discovery.class).to(AzureDiscovery.class).asEagerSingleton(); bind(Discovery.class).to(AzureDiscovery.class).asEagerSingleton();
} else { } else {

33
src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java
View File

@ -24,7 +24,8 @@ import org.elasticsearch.Version;
import org.elasticsearch.cloud.azure.AzureServiceDisableException; import org.elasticsearch.cloud.azure.AzureServiceDisableException;
import org.elasticsearch.cloud.azure.AzureServiceRemoteException; import org.elasticsearch.cloud.azure.AzureServiceRemoteException;
import org.elasticsearch.cloud.azure.management.AzureComputeService; import org.elasticsearch.cloud.azure.management.AzureComputeService;
import org.elasticsearch.cloud.azure.management.AzureComputeService.Fields; import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery;
import org.elasticsearch.cloud.azure.management.AzureComputeService.Management;
import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.collect.Lists;
import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.component.AbstractComponent;
@ -111,38 +112,40 @@ public class AzureUnicastHostsProvider extends AbstractComponent implements Unic
this.networkService = networkService; this.networkService = networkService;
this.version = version; this.version = version;
this.refreshInterval = componentSettings.getAsTime(Fields.REFRESH, this.refreshInterval = settings.getAsTime(Discovery.REFRESH, settings.getAsTime(Discovery.REFRESH_DEPRECATED,
settings.getAsTime("cloud.azure." + Fields.REFRESH, TimeValue.timeValueSeconds(0))); TimeValue.timeValueSeconds(0)));
String strHostType = componentSettings.get(Fields.HOST_TYPE, String strHostType = settings.get(Discovery.HOST_TYPE, settings.get(Discovery.HOST_TYPE_DEPRECATED,
settings.get("cloud.azure." + Fields.HOST_TYPE_DEPRECATED, HostType.PRIVATE_IP.name())).toUpperCase(); HostType.PRIVATE_IP.name())).toUpperCase();
HostType tmpHostType = HostType.fromString(strHostType); HostType tmpHostType = HostType.fromString(strHostType);
if (tmpHostType == null) { if (tmpHostType == null) {
logger.warn("wrong value for [{}]: [{}]. falling back to [{}]...", Fields.HOST_TYPE, logger.warn("wrong value for [{}]: [{}]. falling back to [{}]...", Discovery.HOST_TYPE,
strHostType, HostType.PRIVATE_IP.name().toLowerCase()); strHostType, HostType.PRIVATE_IP.name().toLowerCase());
tmpHostType = HostType.PRIVATE_IP; tmpHostType = HostType.PRIVATE_IP;
} }
this.hostType = tmpHostType; this.hostType = tmpHostType;
// TODO Remove in 3.0.0 // TODO Remove in 3.0.0
String portName = settings.get("cloud.azure." + Fields.PORT_NAME_DEPRECATED); String portName = settings.get(Discovery.PORT_NAME_DEPRECATED);
if (portName != null) { if (portName != null) {
logger.warn("setting [{}] has been deprecated. please replace with [{}].", Fields.PORT_NAME_DEPRECATED, Fields.ENDPOINT_NAME); logger.warn("setting [{}] has been deprecated. please replace with [{}].", Discovery.PORT_NAME_DEPRECATED,
Discovery.ENDPOINT_NAME);
this.publicEndpointName = portName; this.publicEndpointName = portName;
} else { } else {
this.publicEndpointName = componentSettings.get(Fields.ENDPOINT_NAME, "elasticsearch"); this.publicEndpointName = settings.get(Discovery.ENDPOINT_NAME, "elasticsearch");
} }
this.deploymentName = componentSettings.get(Fields.DEPLOYMENT_NAME, // Deployment name could be set with discovery.azure.deployment.name
settings.get("cloud.azure.management." + Fields.SERVICE_NAME, // Default to cloud.azure.management.cloud.service.name
settings.get("cloud.azure." + Fields.SERVICE_NAME_DEPRECATED))); this.deploymentName = settings.get(Discovery.DEPLOYMENT_NAME,
settings.get(Management.SERVICE_NAME, settings.get(Management.SERVICE_NAME_DEPRECATED)));
// Reading deployment_slot // Reading deployment_slot
String strDeployment = componentSettings.get(Fields.DEPLOYMENT_SLOT, Deployment.PRODUCTION.deployment); String strDeployment = settings.get(Discovery.DEPLOYMENT_SLOT, Deployment.PRODUCTION.deployment);
Deployment tmpDeployment = Deployment.fromString(strDeployment); Deployment tmpDeployment = Deployment.fromString(strDeployment);
if (tmpDeployment == null) { if (tmpDeployment == null) {
logger.warn("wrong value for [{}]: [{}]. falling back to [{}]...", Fields.DEPLOYMENT_SLOT, logger.warn("wrong value for [{}]: [{}]. falling back to [{}]...", Discovery.DEPLOYMENT_SLOT, strDeployment,
strDeployment, Deployment.PRODUCTION.deployment); Deployment.PRODUCTION.deployment);
tmpDeployment = Deployment.PRODUCTION; tmpDeployment = Deployment.PRODUCTION;
} }
this.deploymentSlot = tmpDeployment.slot; this.deploymentSlot = tmpDeployment.slot;

23
src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
View File

@ -20,8 +20,8 @@
package org.elasticsearch.repositories.azure; package org.elasticsearch.repositories.azure;
import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore; import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.metadata.SnapshotId;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
@ -57,6 +57,13 @@ public class AzureRepository extends BlobStoreRepository {
public final static String TYPE = "azure"; public final static String TYPE = "azure";
public final static String CONTAINER_DEFAULT = "elasticsearch-snapshots"; public final static String CONTAINER_DEFAULT = "elasticsearch-snapshots";
static public final class Repository {
public static final String CONTAINER = "container";
public static final String CHUNK_SIZE = "chunk_size";
public static final String COMPRESS = "compress";
public static final String BASE_PATH = "base_path";
}
private final AzureBlobStore blobStore; private final AzureBlobStore blobStore;
private final BlobPath basePath; private final BlobPath basePath;
@ -71,21 +78,21 @@ public class AzureRepository extends BlobStoreRepository {
AzureBlobStore azureBlobStore) throws IOException, URISyntaxException, StorageException { AzureBlobStore azureBlobStore) throws IOException, URISyntaxException, StorageException {
super(name.getName(), repositorySettings, indexShardRepository); super(name.getName(), repositorySettings, indexShardRepository);
String container = repositorySettings.settings().get(AzureStorageService.Fields.CONTAINER, String container = repositorySettings.settings().get(Repository.CONTAINER,
componentSettings.get(AzureStorageService.Fields.CONTAINER, CONTAINER_DEFAULT)); settings.get(Storage.CONTAINER, CONTAINER_DEFAULT));
this.blobStore = azureBlobStore; this.blobStore = azureBlobStore;
this.chunkSize = repositorySettings.settings().getAsBytesSize(AzureStorageService.Fields.CHUNK_SIZE, this.chunkSize = repositorySettings.settings().getAsBytesSize(Repository.CHUNK_SIZE,
componentSettings.getAsBytesSize(AzureStorageService.Fields.CHUNK_SIZE, new ByteSizeValue(64, ByteSizeUnit.MB))); settings.getAsBytesSize(Storage.CHUNK_SIZE, new ByteSizeValue(64, ByteSizeUnit.MB)));
if (this.chunkSize.getMb() > 64) { if (this.chunkSize.getMb() > 64) {
logger.warn("azure repository does not support yet size > 64mb. Fall back to 64mb."); logger.warn("azure repository does not support yet size > 64mb. Fall back to 64mb.");
this.chunkSize = new ByteSizeValue(64, ByteSizeUnit.MB); this.chunkSize = new ByteSizeValue(64, ByteSizeUnit.MB);
} }
this.compress = repositorySettings.settings().getAsBoolean(AzureStorageService.Fields.COMPRESS, this.compress = repositorySettings.settings().getAsBoolean(Repository.COMPRESS,
componentSettings.getAsBoolean(AzureStorageService.Fields.COMPRESS, false)); settings.getAsBoolean(Storage.COMPRESS, false));
String basePath = repositorySettings.settings().get(AzureStorageService.Fields.BASE_PATH, null); String basePath = repositorySettings.settings().get(Repository.BASE_PATH, null);
if (Strings.hasLength(basePath)) { if (Strings.hasLength(basePath)) {
// Remove starting / if any // Remove starting / if any

4
src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryModule.java
View File

@ -20,8 +20,8 @@
package org.elasticsearch.repositories.azure; package org.elasticsearch.repositories.azure;
import org.elasticsearch.cloud.azure.AzureModule; import org.elasticsearch.cloud.azure.AzureModule;
import org.elasticsearch.cloud.azure.storage.AzureStorageSettingsFilter;
import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
@ -37,7 +37,6 @@ public class AzureRepositoryModule extends AbstractModule {
protected final ESLogger logger; protected final ESLogger logger;
private Settings settings; private Settings settings;
@Inject
public AzureRepositoryModule(Settings settings) { public AzureRepositoryModule(Settings settings) {
super(); super();
this.logger = Loggers.getLogger(getClass(), settings); this.logger = Loggers.getLogger(getClass(), settings);
@ -49,6 +48,7 @@ public class AzureRepositoryModule extends AbstractModule {
*/ */
@Override @Override
protected void configure() { protected void configure() {
bind(AzureStorageSettingsFilter.class).asEagerSingleton();
if (AzureModule.isSnapshotReady(settings, logger)) { if (AzureModule.isSnapshotReady(settings, logger)) {
bind(Repository.class).to(AzureRepository.class).asEagerSingleton(); bind(Repository.class).to(AzureRepository.class).asEagerSingleton();
bind(IndexShardRepository.class).to(BlobStoreIndexShardRepository.class).asEagerSingleton(); bind(IndexShardRepository.class).to(BlobStoreIndexShardRepository.class).asEagerSingleton();

2
src/test/java/org/elasticsearch/cloud/azure/AbstractAzureTest.java
View File

@ -57,7 +57,7 @@ public abstract class AbstractAzureTest extends ElasticsearchIntegrationTest {
protected Settings nodeSettings(int nodeOrdinal) { protected Settings nodeSettings(int nodeOrdinal) {
return ImmutableSettings.builder() return ImmutableSettings.builder()
.put(super.nodeSettings(nodeOrdinal)) .put(super.nodeSettings(nodeOrdinal))
.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) .put(PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true)
.put(readSettingsFromFile()) .put(readSettingsFromFile())
.build(); .build();
} }

28
src/test/java/org/elasticsearch/discovery/azure/AbstractAzureComputeServiceTest.java
View File

@ -21,6 +21,8 @@ package org.elasticsearch.discovery.azure;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.cloud.azure.management.AzureComputeService; import org.elasticsearch.cloud.azure.management.AzureComputeService;
import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery;
import org.elasticsearch.cloud.azure.management.AzureComputeService.Management;
import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.PluginsService;
@ -39,7 +41,7 @@ public abstract class AbstractAzureComputeServiceTest extends ElasticsearchInteg
protected Settings nodeSettings(int nodeOrdinal) { protected Settings nodeSettings(int nodeOrdinal) {
ImmutableSettings.Builder settings = ImmutableSettings.builder() ImmutableSettings.Builder settings = ImmutableSettings.builder()
.put(super.nodeSettings(nodeOrdinal)) .put(super.nodeSettings(nodeOrdinal))
.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true); .put(PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true);
return settings.build(); return settings.build();
} }
@ -53,16 +55,26 @@ public abstract class AbstractAzureComputeServiceTest extends ElasticsearchInteg
protected Settings settingsBuilder() { protected Settings settingsBuilder() {
ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder() ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder()
.put("discovery.type", "azure") .put("discovery.type", "azure")
.put("cloud.azure.api.impl", mock) .put(Management.API_IMPLEMENTATION, mock)
// We add a fake subscription_id to start mock compute service
.put("cloud.azure.subscription_id", "fake")
.put("cloud.azure.refresh_interval", "5s")
.put("cloud.azure.keystore", "dummy")
.put("cloud.azure.password", "dummy")
.put("cloud.azure.service_name", "dummy")
// We need the network to make the mock working // We need the network to make the mock working
.put("node.mode", "network"); .put("node.mode", "network");
// We add a fake subscription_id to start mock compute service
if (rarely()) {
// We use sometime deprecated settings in tests
builder.put(Management.SUBSCRIPTION_ID_DEPRECATED, "fake")
.put(Discovery.REFRESH_DEPRECATED, "5s")
.put(Management.KEYSTORE_DEPRECATED, "dummy")
.put(Management.PASSWORD_DEPRECATED, "dummy")
.put(Management.SERVICE_NAME_DEPRECATED, "dummy");
} else {
builder.put(Management.SUBSCRIPTION_ID, "fake")
.put(Discovery.REFRESH, "5s")
.put(Management.KEYSTORE_PATH, "dummy")
.put(Management.KEYSTORE_PASSWORD, "dummy")
.put(Management.SERVICE_NAME, "dummy");
}
return builder.build(); return builder.build();
} }
} }

14
src/test/java/org/elasticsearch/discovery/azure/AzureSimpleTest.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.discovery.azure; package org.elasticsearch.discovery.azure;
import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery;
import org.elasticsearch.cloud.azure.management.AzureComputeService.Management;
import org.elasticsearch.cloud.azure.management.AzureComputeServiceSimpleMock; import org.elasticsearch.cloud.azure.management.AzureComputeServiceSimpleMock;
import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest;
@ -39,8 +41,8 @@ public class AzureSimpleTest extends AbstractAzureComputeServiceTest {
@Test @Test
public void one_node_should_run_using_private_ip() { public void one_node_should_run_using_private_ip() {
ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder() ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder()
.put("cloud.azure.service_name", "dummy") .put(Management.SERVICE_NAME, "dummy")
.put("cloud.azure.host_type", "private_ip") .put(Discovery.HOST_TYPE, "private_ip")
.put(super.settingsBuilder()); .put(super.settingsBuilder());
logger.info("--> start one node"); logger.info("--> start one node");
@ -54,8 +56,8 @@ public class AzureSimpleTest extends AbstractAzureComputeServiceTest {
@Test @Test
public void one_node_should_run_using_public_ip() { public void one_node_should_run_using_public_ip() {
ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder() ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder()
.put("cloud.azure.service_name", "dummy") .put(Management.SERVICE_NAME, "dummy")
.put("cloud.azure.host_type", "public_ip") .put(Discovery.HOST_TYPE, "public_ip")
.put(super.settingsBuilder()); .put(super.settingsBuilder());
logger.info("--> start one node"); logger.info("--> start one node");
@ -69,8 +71,8 @@ public class AzureSimpleTest extends AbstractAzureComputeServiceTest {
@Test @Test
public void one_node_should_run_using_wrong_settings() { public void one_node_should_run_using_wrong_settings() {
ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder() ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder()
.put("cloud.azure.service_name", "dummy") .put(Management.SERVICE_NAME, "dummy")
.put("cloud.azure.host_type", "do_not_exist") .put(Discovery.HOST_TYPE, "do_not_exist")
.put(super.settingsBuilder()); .put(super.settingsBuilder());
logger.info("--> start one node"); logger.info("--> start one node");

10
src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTest.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.discovery.azure; package org.elasticsearch.discovery.azure;
import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery;
import org.elasticsearch.cloud.azure.management.AzureComputeService.Management;
import org.elasticsearch.cloud.azure.management.AzureComputeServiceTwoNodesMock; import org.elasticsearch.cloud.azure.management.AzureComputeServiceTwoNodesMock;
import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest;
@ -39,8 +41,8 @@ public class AzureTwoStartedNodesTest extends AbstractAzureComputeServiceTest {
@Test @Test
public void two_nodes_should_run_using_private_ip() { public void two_nodes_should_run_using_private_ip() {
ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder() ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder()
.put("cloud.azure.service_name", "dummy") .put(Management.SERVICE_NAME, "dummy")
.put("cloud.azure.host_type", "private_ip") .put(Discovery.HOST_TYPE, "private_ip")
.put(super.settingsBuilder()); .put(super.settingsBuilder());
logger.info("--> start first node"); logger.info("--> start first node");
@ -58,8 +60,8 @@ public class AzureTwoStartedNodesTest extends AbstractAzureComputeServiceTest {
@Test @Test
public void two_nodes_should_run_using_public_ip() { public void two_nodes_should_run_using_public_ip() {
ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder() ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder()
.put("cloud.azure.service_name", "dummy") .put(Management.SERVICE_NAME, "dummy")
.put("cloud.azure.host_type", "public_ip") .put(Discovery.HOST_TYPE, "public_ip")
.put(super.settingsBuilder()); .put(super.settingsBuilder());
logger.info("--> start first node"); logger.info("--> start first node");

15
src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.repositories.azure;
import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.cloud.azure.AbstractAzureTest; import org.elasticsearch.cloud.azure.AbstractAzureTest;
import org.elasticsearch.cloud.azure.storage.AzureStorageService; import org.elasticsearch.cloud.azure.storage.AzureStorageService;
import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
@ -65,17 +66,17 @@ public abstract class AbstractAzureRepositoryServiceTest extends AbstractAzureTe
@Override @Override
protected Settings nodeSettings(int nodeOrdinal) { protected Settings nodeSettings(int nodeOrdinal) {
ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder() ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder()
.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) .put(PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true)
.put("repositories.azure.api.impl", mock) .put(Storage.API_IMPLEMENTATION, mock)
.put("repositories.azure.container", "snapshots"); .put(Storage.CONTAINER, "snapshots");
// We use sometime deprecated settings in tests // We use sometime deprecated settings in tests
if (rarely()) { if (rarely()) {
builder.put("cloud.azure." + AzureStorageService.Fields.ACCOUNT_DEPRECATED, "mock_azure_account") builder.put(Storage.ACCOUNT_DEPRECATED, "mock_azure_account")
.put("cloud.azure." + AzureStorageService.Fields.KEY_DEPRECATED, "mock_azure_key"); .put(Storage.KEY_DEPRECATED, "mock_azure_key");
} else { } else {
builder.put("cloud.azure.storage." + AzureStorageService.Fields.ACCOUNT, "mock_azure_account") builder.put(Storage.ACCOUNT, "mock_azure_account")
.put("cloud.azure.storage." + AzureStorageService.Fields.KEY, "mock_azure_key"); .put(Storage.KEY, "mock_azure_key");
} }
return builder.build(); return builder.build();

49
src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java
View File

@ -27,17 +27,17 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotR
import org.elasticsearch.client.Client; import org.elasticsearch.client.Client;
import org.elasticsearch.client.ClusterAdminClient; import org.elasticsearch.client.ClusterAdminClient;
import org.elasticsearch.cloud.azure.AbstractAzureTest; import org.elasticsearch.cloud.azure.AbstractAzureTest;
import org.elasticsearch.cloud.azure.AzureSettingsFilter;
import org.elasticsearch.cloud.azure.storage.AzureStorageService; import org.elasticsearch.cloud.azure.storage.AzureStorageService;
import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl; import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.base.Predicate; import org.elasticsearch.common.base.Predicate;
import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.repositories.RepositoryMissingException;
import org.elasticsearch.repositories.RepositoryVerificationException; import org.elasticsearch.repositories.RepositoryVerificationException;
import org.elasticsearch.repositories.azure.AzureRepository.Repository;
import org.elasticsearch.snapshots.SnapshotMissingException; import org.elasticsearch.snapshots.SnapshotMissingException;
import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.snapshots.SnapshotState;
import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest;
@ -109,9 +109,9 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
logger.info("--> creating azure repository with path [{}]", getRepositoryPath()); logger.info("--> creating azure repository with path [{}]", getRepositoryPath());
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("azure").setSettings(ImmutableSettings.settingsBuilder() .setType("azure").setSettings(ImmutableSettings.settingsBuilder()
.put(AzureStorageService.Fields.CONTAINER, getContainerName()) .put(Storage.CONTAINER, getContainerName())
.put(AzureStorageService.Fields.BASE_PATH, getRepositoryPath()) .put(Storage.BASE_PATH, getRepositoryPath())
.put(AzureStorageService.Fields.CHUNK_SIZE, randomIntBetween(1000, 10000)) .put(Storage.CHUNK_SIZE, randomIntBetween(1000, 10000))
).get(); ).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@ -182,16 +182,16 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
logger.info("--> creating azure repository with path [{}]", getRepositoryPath()); logger.info("--> creating azure repository with path [{}]", getRepositoryPath());
PutRepositoryResponse putRepositoryResponse1 = client.admin().cluster().preparePutRepository("test-repo1") PutRepositoryResponse putRepositoryResponse1 = client.admin().cluster().preparePutRepository("test-repo1")
.setType("azure").setSettings(ImmutableSettings.settingsBuilder() .setType("azure").setSettings(ImmutableSettings.settingsBuilder()
.put(AzureStorageService.Fields.CONTAINER, getContainerName().concat("-1")) .put(Repository.CONTAINER, getContainerName().concat("-1"))
.put(AzureStorageService.Fields.BASE_PATH, getRepositoryPath()) .put(Repository.BASE_PATH, getRepositoryPath())
.put(AzureStorageService.Fields.CHUNK_SIZE, randomIntBetween(1000, 10000)) .put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000))
).get(); ).get();
assertThat(putRepositoryResponse1.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse1.isAcknowledged(), equalTo(true));
PutRepositoryResponse putRepositoryResponse2 = client.admin().cluster().preparePutRepository("test-repo2") PutRepositoryResponse putRepositoryResponse2 = client.admin().cluster().preparePutRepository("test-repo2")
.setType("azure").setSettings(ImmutableSettings.settingsBuilder() .setType("azure").setSettings(ImmutableSettings.settingsBuilder()
.put(AzureStorageService.Fields.CONTAINER, getContainerName().concat("-2")) .put(Repository.CONTAINER, getContainerName().concat("-2"))
.put(AzureStorageService.Fields.BASE_PATH, getRepositoryPath()) .put(Repository.BASE_PATH, getRepositoryPath())
.put(AzureStorageService.Fields.CHUNK_SIZE, randomIntBetween(1000, 10000)) .put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000))
).get(); ).get();
assertThat(putRepositoryResponse2.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse2.isAcknowledged(), equalTo(true));
@ -262,7 +262,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
logger.info("--> creating azure repository without any path"); logger.info("--> creating azure repository without any path");
PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure") PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure")
.setSettings(ImmutableSettings.settingsBuilder() .setSettings(ImmutableSettings.settingsBuilder()
.put(AzureStorageService.Fields.CONTAINER, getContainerName()) .put(Repository.CONTAINER, getContainerName())
).get(); ).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@ -283,8 +283,8 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
logger.info("--> creating azure repository path [{}]", getRepositoryPath()); logger.info("--> creating azure repository path [{}]", getRepositoryPath());
putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure") putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure")
.setSettings(ImmutableSettings.settingsBuilder() .setSettings(ImmutableSettings.settingsBuilder()
.put(AzureStorageService.Fields.CONTAINER, getContainerName()) .put(Repository.CONTAINER, getContainerName())
.put(AzureStorageService.Fields.BASE_PATH, getRepositoryPath()) .put(Repository.BASE_PATH, getRepositoryPath())
).get(); ).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@ -310,7 +310,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
logger.info("--> creating azure repository without any path"); logger.info("--> creating azure repository without any path");
PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure") PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure")
.setSettings(ImmutableSettings.settingsBuilder() .setSettings(ImmutableSettings.settingsBuilder()
.put(AzureStorageService.Fields.CONTAINER, getContainerName()) .put(Repository.CONTAINER, getContainerName())
).get(); ).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@ -360,9 +360,9 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
try { try {
PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo") PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
.setType("azure").setSettings(ImmutableSettings.settingsBuilder() .setType("azure").setSettings(ImmutableSettings.settingsBuilder()
.put(AzureStorageService.Fields.CONTAINER, container) .put(Repository.CONTAINER, container)
.put(AzureStorageService.Fields.BASE_PATH, getRepositoryPath()) .put(Repository.BASE_PATH, getRepositoryPath())
.put(AzureStorageService.Fields.CHUNK_SIZE, randomIntBetween(1000, 10000)) .put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000))
).get(); ).get();
client().admin().cluster().prepareDeleteRepository("test-repo").get(); client().admin().cluster().prepareDeleteRepository("test-repo").get();
try { try {
@ -392,9 +392,9 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
logger.info("--> creating azure repository with path [{}]", getRepositoryPath()); logger.info("--> creating azure repository with path [{}]", getRepositoryPath());
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("azure").setSettings(ImmutableSettings.settingsBuilder() .setType("azure").setSettings(ImmutableSettings.settingsBuilder()
.put(AzureStorageService.Fields.CONTAINER, getContainerName()) .put(Repository.CONTAINER, getContainerName())
.put(AzureStorageService.Fields.BASE_PATH, getRepositoryPath()) .put(Repository.BASE_PATH, getRepositoryPath())
.put(AzureStorageService.Fields.CHUNK_SIZE, randomIntBetween(1000, 10000)) .put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000))
).get(); ).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@ -440,7 +440,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
try { try {
client.preparePutRepository("test-repo").setType("azure") client.preparePutRepository("test-repo").setType("azure")
.setSettings(ImmutableSettings.settingsBuilder() .setSettings(ImmutableSettings.settingsBuilder()
.put(AzureStorageService.Fields.CONTAINER, container) .put(Repository.CONTAINER, container)
).get(); ).get();
fail("we should get a RepositoryVerificationException"); fail("we should get a RepositoryVerificationException");
} catch (RepositoryVerificationException e) { } catch (RepositoryVerificationException e) {
@ -470,10 +470,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
*/ */
public static void cleanRepositoryFiles(String... containers) throws StorageException, URISyntaxException { public static void cleanRepositoryFiles(String... containers) throws StorageException, URISyntaxException {
Settings settings = readSettingsFromFile(); Settings settings = readSettingsFromFile();
SettingsFilter settingsFilter = new SettingsFilter(settings); AzureStorageService client = new AzureStorageServiceImpl(settings);
settingsFilter.addFilter(new AzureSettingsFilter());
AzureStorageService client = new AzureStorageServiceImpl(settings, settingsFilter);
for (String container : containers) { for (String container : containers) {
client.removeContainer(container); client.removeContainer(container);
} }

40
src/test/resources/elasticsearch.yml
View File

@ -1,40 +0,0 @@
# Licensed to Elasticsearch under one or more contributor
# license agreements. See the NOTICE file distributed with this work for additional
# information regarding copyright ownership. ElasticSearch licenses this file to you
# under the Apache License, Version 2.0 (the "License"); you may not use this
# file except in compliance with the License. You may obtain a copy of the
# License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by
# applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the specific language
# governing permissions and limitations under the License.
#
# Azure discovery allows to use Azure API in order to perform discovery.
#
# You have to install the cloud-azure plugin for enabling the Azure discovery.
#
# See <http://elasticsearch.org/guide/reference/modules/discovery/azure.html>
# for more information.
#
# See <https://github.com/elasticsearch/elasticsearch-cloud-azure/>
# for a step-by-step tutorial.
#
# cloud:
# azure:
# keystore: FULLPATH-TO-YOUR-KEYSTORE
# password: YOUR-PASSWORD
# subscription_id: YOUR-AZURE-SUBSCRIPTION-ID
# service_name: YOUR-AZURE-SERVICE-NAME
# storage_account: "YOUR-AZURE-STORAGE-NAME"
# storage_key: "YOUR-AZURE-STORAGE-KEY"
#
# discovery:
# type: azure
#
# repositories:
# azure:
# container: "NAME-OF-CONTAINER-USED-FOR-SNAPSHOTS" #optional default to "elasticsearch-snapshots"
# base_path: "path/to/snapshots" #optional default to empty
# concurrent_streams: 5 #optional default to 5
# chunk_size: 64mb #optional default to "64mb"
# compress: false #optional default to false