honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update to elasticsearch 1.3.0 

And refactor integration tests

Closes #17.

(cherry picked from commit f5b444e)
(cherry picked from commit 1e63f86)
This commit is contained in:
David Pilato 2014-06-19 18:57:35 +02:00
parent d020012ef2
commit 943f2552f9
11 changed files with 174 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
README.md
View File

@ -3,9 +3,10 @@ Azure Cloud Plugin for Elasticsearch
The Azure Cloud plugin allows to use Azure API for the unicast discovery mechanism.
In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-azure/2.1.0`.
In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-azure/2.3.0`.
* For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-azure/tree/master).
* For 1.3.x elasticsearch versions, look at [es-1.3 branch](https://github.com/elasticsearch/elasticsearch-cloud-azure/tree/es-1.3).
* For 1.2.x elasticsearch versions, look at [es-1.2 branch](https://github.com/elasticsearch/elasticsearch-cloud-azure/tree/es-1.2).
* For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-cloud-azure/tree/es-1.1).
* For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-cloud-azure/tree/es-1.0).
@ -27,17 +28,17 @@ Azure VM discovery allows to use the azure APIs to perform automatic discovery (
multicast environments). Here is a simple sample configuration:
```
cloud:
azure:
keystore: /path/to/keystore
password: your_password_for_keystore
subscription_id: your_azure_subscription_id
service_name: your_azure_cloud_service_name
discovery:
type: azure
cloud:
azure:
keystore: /path/to/keystore
password: your_password_for_keystore
subscription_id: your_azure_subscription_id
service_name: your_azure_cloud_service_name
discovery:
type: azure
# recommended
# path.data: /mnt/resource/elasticsearch/data
# recommended
# path.data: /mnt/resource/elasticsearch/data
```
How to start (short story)
@ -261,17 +262,17 @@ And add the following lines:
```yaml
# If you don't remember your account id, you may get it with `azure account list`
cloud:
azure:
keystore: /home/elasticsearch/azurekeystore.pkcs12
password: your_password_for_keystore
subscription_id: your_azure_subscription_id
service_name: your_azure_cloud_service_name
discovery:
type: azure
cloud:
azure:
keystore: /home/elasticsearch/azurekeystore.pkcs12
password: your_password_for_keystore
subscription_id: your_azure_subscription_id
service_name: your_azure_cloud_service_name
discovery:
type: azure
# Recommended
path.data: /mnt/resource/elasticsearch/data
path.data: /mnt/resource/elasticsearch/data
```
Restart elasticsearch:
@ -347,10 +348,10 @@ Azure Repository
To enable Azure repositories, you have first to set your azure storage settings:
```
cloud:
azure:
storage_account: your_azure_storage_account
storage_key: your_azure_storage_key
cloud:
azure:
storage_account: your_azure_storage_account
storage_key: your_azure_storage_key
```
The Azure repository supports following settings:
@ -402,10 +403,10 @@ Integrations tests in this plugin require working Azure configuration and theref
To enable tests prepare a config file elasticsearch.yml with the following content:
```
cloud:
azure:
account: "YOUR-AZURE-STORAGE-NAME"
key: "YOUR-AZURE-STORAGE-KEY"
cloud:
azure:
account: "YOUR-AZURE-STORAGE-NAME"
key: "YOUR-AZURE-STORAGE-KEY"
```
Replaces `account`, `key` with your settings. Please, note that the test will delete all snapshot/restore related files in the specified bucket.

31
pom.xml
View File

@ -44,9 +44,8 @@ governing permissions and limitations under the License. -->
<properties>
<elasticsearch.version>2.0.0-SNAPSHOT</elasticsearch.version>
<lucene.version>4.8.0</lucene.version>
<lucene.version>4.8.1</lucene.version>
<tests.output>onerror</tests.output>
<tests.jvms>1</tests.jvms>
<tests.shuffle>true</tests.shuffle>
<tests.output>onerror</tests.output>
<tests.client.ratio></tests.client.ratio>
@ -56,6 +55,18 @@ governing permissions and limitations under the License. -->
</properties>
<dependencies>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
<version>1.3.RC2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<version>1.3.RC2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-test-framework</artifactId>
@ -86,18 +97,6 @@ governing permissions and limitations under the License. -->
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
<version>1.3.RC2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<version>1.3.RC2</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
@ -157,7 +156,7 @@ governing permissions and limitations under the License. -->
<assertions>
<enable/>
</assertions>
<parallelism>${tests.jvms}</parallelism>
<parallelism>1</parallelism>
<balancers>
<execution-times>
<fileset dir="${basedir}" includes=".local-execution-hints.log"/>
@ -204,8 +203,6 @@ governing permissions and limitations under the License. -->
<tests.integration>${tests.integration}</tests.integration>
<tests.cluster_seed>${tests.cluster_seed}</tests.cluster_seed>
<tests.client.ratio>${tests.client.ratio}</tests.client.ratio>
<es.node.local>${env.ES_TEST_LOCAL}</es.node.local>
<es.node.mode>${es.node.mode}</es.node.mode>
<es.config>${es.config}</es.config>
<es.logger.level>${es.logger.level}</es.logger.level>
<java.awt.headless>true</java.awt.headless>

7
src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java
View File

@ -32,7 +32,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.discovery.zen.ping.unicast.UnicastHostsProvider;
import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing;
import org.elasticsearch.transport.TransportService;
import java.io.IOException;
@ -139,10 +138,8 @@ public class AzureUnicastHostsProvider extends AbstractComponent implements Unic
} else {
TransportAddress[] addresses = transportService.addressesFromString(networkAddress);
// we only limit to 1 addresses, makes no sense to ping 100 ports
for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) {
logger.trace("adding {}, transport_address {}", networkAddress, addresses[i]);
cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + instance.getName() + "-" + i, addresses[i], Version.CURRENT));
}
logger.trace("adding {}, transport_address {}", networkAddress, addresses[0]);
cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + instance.getName(), addresses[0], Version.CURRENT));
}
}

1
src/test/java/org/elasticsearch/azure/itest/AzureSimpleITest.java
View File

@ -36,6 +36,7 @@ import org.junit.Test;
@ElasticsearchIntegrationTest.ClusterScope(
scope = ElasticsearchIntegrationTest.Scope.TEST,
numDataNodes = 1,
numClientNodes = 0,
transportClientRatio = 0.0)
public class AzureSimpleITest extends AbstractAzureTest {

16
src/test/java/org/elasticsearch/discovery/azure/AbstractAzureComputeServiceTest.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.cloud.azure.AbstractAzureTest;
import org.elasticsearch.cloud.azure.AzureComputeService;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.plugins.PluginsService;
public abstract class AbstractAzureComputeServiceTest extends AbstractAzureTest {
@ -41,13 +42,22 @@ public abstract class AbstractAzureComputeServiceTest extends AbstractAzureTest
assertEquals(expected, nodeInfos.getNodes().length);
}
@Override
protected Settings nodeSettings(int nodeOrdinal) {
protected Settings settingsBuilder() {
ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder()
.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true)
.put("discovery.type", "azure")
.put("cloud.azure.api.impl", mock)
// We add a fake subscription_id to start mock compute service
.put("cloud.azure.subscription_id", "fake")
.put("cloud.azure.refresh_interval", "5s");
.put("cloud.azure.refresh_interval", "5s")
.put("cloud.azure.keystore", "dummy")
.put("cloud.azure.password", "dummy")
.put("cloud.azure.service_name", "dummy")
.put("cloud.azure.refresh_interval", "5s")
// Make the tests run faster
.put("discovery.zen.join.timeout", "100ms")
.put("discovery.zen.ping.timeout", "10ms")
.put("discovery.initial_state_timeout", "300ms");
return builder.build();
}

83
src/test/java/org/elasticsearch/discovery/azure/AzureMinimumMasterNodesTest.java Normal file
View File

@ -0,0 +1,83 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.discovery.azure;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.discovery.MasterNotDiscoveredException;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;
import java.io.IOException;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
/**
* Reported issue in #15
* (https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/15)
*/
@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.TEST,
numDataNodes = 0,
transportClientRatio = 0.0,
numClientNodes = 0)
public class AzureMinimumMasterNodesTest extends AbstractAzureComputeServiceTest {
public AzureMinimumMasterNodesTest() {
super(AzureComputeServiceTwoNodesMock.class);
}
@Override
protected final Settings settingsBuilder() {
ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder()
.put("discovery.zen.minimum_master_nodes", 2)
.put(super.settingsBuilder());
return builder.build();
}
@Test
public void simpleOnlyMasterNodeElection() throws IOException {
logger.info("--> start data node / non master node");
internalCluster().startNode(settingsBuilder());
try {
assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("100ms").execute().actionGet().getState().nodes().masterNodeId(), nullValue());
fail("should not be able to find master");
} catch (MasterNotDiscoveredException e) {
// all is well, no master elected
}
logger.info("--> start another node");
internalCluster().startNode(settingsBuilder());
assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().masterNodeId(), notNullValue());
logger.info("--> stop master node");
internalCluster().stopCurrentMasterNode();
try {
assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().masterNodeId(), nullValue());
fail("should not be able to find master");
} catch (MasterNotDiscoveredException e) {
// all is well, no master elected
}
logger.info("--> start another node");
internalCluster().startNode(settingsBuilder());
assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().masterNodeId(), notNullValue());
}
}

18
src/test/java/org/elasticsearch/discovery/azure/AzureSimpleTest.java
View File

@ -22,10 +22,12 @@ package org.elasticsearch.discovery.azure;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;
@ElasticsearchIntegrationTest.ClusterScope(
scope = ElasticsearchIntegrationTest.Scope.TEST,
numDataNodes = 1,
transportClientRatio = 0.0)
import static org.hamcrest.Matchers.notNullValue;
@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.TEST,
numDataNodes = 0,
transportClientRatio = 0.0,
numClientNodes = 0)
public class AzureSimpleTest extends AbstractAzureComputeServiceTest {
public AzureSimpleTest() {
@ -34,9 +36,11 @@ public class AzureSimpleTest extends AbstractAzureComputeServiceTest {
@Test
public void one_node_should_run() {
// We expect having 2 nodes as part of the cluster, let's test that
logger.info("--> start one node");
internalCluster().startNode(settingsBuilder());
assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().masterNodeId(), notNullValue());
// We expect having 1 node as part of the cluster, let's test that
checkNumberOfNodes(1);
}
}

18
src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTest.java
View File

@ -22,10 +22,12 @@ package org.elasticsearch.discovery.azure;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;
@ElasticsearchIntegrationTest.ClusterScope(
scope = ElasticsearchIntegrationTest.Scope.TEST,
numDataNodes = 2,
transportClientRatio = 0.0)
import static org.hamcrest.Matchers.notNullValue;
@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.TEST,
numDataNodes = 0,
transportClientRatio = 0.0,
numClientNodes = 0)
public class AzureTwoStartedNodesTest extends AbstractAzureComputeServiceTest {
public AzureTwoStartedNodesTest() {
@ -34,6 +36,14 @@ public class AzureTwoStartedNodesTest extends AbstractAzureComputeServiceTest {
@Test
public void two_nodes_should_run() {
logger.info("--> start first node");
internalCluster().startNode(settingsBuilder());
assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().masterNodeId(), notNullValue());
logger.info("--> start another node");
internalCluster().startNode(settingsBuilder());
assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().masterNodeId(), notNullValue());
// We expect having 2 nodes as part of the cluster, let's test that
checkNumberOfNodes(2);
}

6
src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.cloud.azure.AzureStorageService;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.repositories.RepositoryMissingException;
import org.elasticsearch.test.store.MockDirectoryHelper;
import org.junit.After;
@ -65,6 +66,7 @@ public abstract class AbstractAzureRepositoryServiceTest extends AbstractAzureTe
@Override
protected Settings nodeSettings(int nodeOrdinal) {
ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder()
.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true)
.put("cloud.azure." + AzureStorageService.Fields.ACCOUNT, "mock_azure_account")
.put("cloud.azure." + AzureStorageService.Fields.KEY, "mock_azure_key")
.put("repositories.azure.api.impl", mock)
@ -94,9 +96,9 @@ public abstract class AbstractAzureRepositoryServiceTest extends AbstractAzureTe
* Purge the test container
*/
public void cleanRepositoryFiles(String path) throws StorageException, ServiceException, URISyntaxException {
String container = cluster().getInstance(Settings.class).get("repositories.azure.container");
String container = internalCluster().getInstance(Settings.class).get("repositories.azure.container");
logger.info("--> remove blobs in container [{}]", container);
AzureStorageService client = cluster().getInstance(AzureStorageService.class);
AzureStorageService client = internalCluster().getInstance(AzureStorageService.class);
client.deleteFiles(container, path);
}
}

4
src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java
View File

@ -245,10 +245,10 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
* Purge the test container
*/
public void cleanRepositoryFiles(String path) throws StorageException, ServiceException, URISyntaxException {
String container = cluster().getInstance(Settings.class).get("repositories.azure.container",
String container = internalCluster().getInstance(Settings.class).get("repositories.azure.container",
AzureRepository.CONTAINER_DEFAULT);
logger.info("--> remove blobs in container [{}], path [{}]", container, path);
AzureStorageService client = cluster().getInstance(AzureStorageService.class);
AzureStorageService client = internalCluster().getInstance(AzureStorageService.class);
// Remove starting / if any
path = Strings.trimLeadingCharacter(path, '/');

1
src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTest.java
View File

@ -36,6 +36,7 @@ import static org.hamcrest.Matchers.greaterThan;
@ElasticsearchIntegrationTest.ClusterScope(
scope = ElasticsearchIntegrationTest.Scope.SUITE,
numDataNodes = 1,
numClientNodes = 0,
transportClientRatio = 0.0)
public class AzureSnapshotRestoreTest extends AbstractAzureRepositoryServiceTest {