honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[TEST] introduced ImmutableTestCluster abstract base class for TestCluster 

The new base class contains all the immutable methods for a cluster, which will be extended by the future ExternalCluster impl that relies on an external cluster and won't be adding nodes etc. but only sending requests to an existing cluster whose layout never changes

Closes #5620
This commit is contained in:
javanna 2014-03-27 21:31:29 +01:00 committed by Luca Cavanna
parent 056ad0a8d3
commit 2e26dc328e
10 changed files with 327 additions and 244 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/test/java/org/elasticsearch/indices/fielddata/breaker/RandomExceptionCircuitBreakerTests.java
View File

@ -38,7 +38,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.TestCluster;
import org.elasticsearch.test.ImmutableTestCluster;
import org.elasticsearch.test.engine.MockInternalEngine;
import org.elasticsearch.test.engine.ThrowingAtomicReaderWrapper;
import org.junit.Test;
@ -203,7 +203,7 @@ public class RandomExceptionCircuitBreakerTests extends ElasticsearchIntegration
private final double lowLevelRatio;
ThrowingSubReaderWrapper(Settings settings) {
final long seed = settings.getAsLong(TestCluster.SETTING_INDEX_SEED, 0l);
final long seed = settings.getAsLong(ImmutableTestCluster.SETTING_INDEX_SEED, 0l);
this.topLevelRatio = settings.getAsDouble(EXCEPTION_TOP_LEVEL_RATIO_KEY, 0.1d);
this.lowLevelRatio = settings.getAsDouble(EXCEPTION_LOW_LEVEL_RATIO_KEY, 0.1d);
this.random = new Random(seed);

4
src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsTests.java
View File

@ -35,7 +35,7 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.TestCluster;
import org.elasticsearch.test.ImmutableTestCluster;
import org.elasticsearch.test.engine.MockInternalEngine;
import org.elasticsearch.test.engine.ThrowingAtomicReaderWrapper;
import org.elasticsearch.test.store.MockDirectoryHelper;
@ -288,7 +288,7 @@ public class SearchWithRandomExceptionsTests extends ElasticsearchIntegrationTes
private final double lowLevelRatio;
ThrowingSubReaderWrapper(Settings settings) {
final long seed = settings.getAsLong(TestCluster.SETTING_INDEX_SEED, 0l);
final long seed = settings.getAsLong(ImmutableTestCluster.SETTING_INDEX_SEED, 0l);
this.topLevelRatio = settings.getAsDouble(EXCEPTION_TOP_LEVEL_RATIO_KEY, 0.1d);
this.lowLevelRatio = settings.getAsDouble(EXCEPTION_LOW_LEVEL_RATIO_KEY, 0.1d);
this.random = new Random(seed);

40
src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java
View File

@ -162,11 +162,11 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
* By default if no {@link ClusterScope} is configured this will hold a reference to the global cluster carried
* on across test suites.
*/
private static TestCluster currentCluster;
private static ImmutableTestCluster currentCluster;
private static final double TRANSPORT_CLIENT_RATIO = transportClientRatio();
private static final Map<Class<?>, TestCluster> clusters = new IdentityHashMap<>();
private static final Map<Class<?>, ImmutableTestCluster> clusters = new IdentityHashMap<>();
@BeforeClass
public static void beforeClass() throws Exception {
@ -201,9 +201,9 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
default:
fail("Unknown Scope: [" + currentClusterScope + "]");
}
currentCluster.beforeTest(getRandom(), getPerTestTransportClientRatio());
cluster().wipe();
cluster().randomIndexTemplate();
immutableCluster().beforeTest(getRandom(), getPerTestTransportClientRatio());
immutableCluster().wipe();
immutableCluster().randomIndexTemplate();
logger.info("[{}#{}]: before test", getTestClass().getSimpleName(), getTestName());
} catch (OutOfMemoryError e) {
if (e.getMessage().contains("unable to create new native thread")) {
@ -213,8 +213,8 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
}
}
public TestCluster buildAndPutCluster(Scope currentClusterScope, boolean createIfExists) throws IOException {
TestCluster testCluster = clusters.get(this.getClass());
public ImmutableTestCluster buildAndPutCluster(Scope currentClusterScope, boolean createIfExists) throws IOException {
ImmutableTestCluster testCluster = clusters.get(this.getClass());
if (createIfExists || testCluster == null) {
testCluster = buildTestCluster(currentClusterScope);
} else {
@ -227,7 +227,7 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
private void clearClusters() throws IOException {
if (!clusters.isEmpty()) {
for (TestCluster cluster : clusters.values()) {
for (ImmutableTestCluster cluster : clusters.values()) {
cluster.close();
}
clusters.clear();
@ -248,8 +248,8 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
assertThat("test leaves transient cluster metadata behind: " + metaData.transientSettings().getAsMap(), metaData
.transientSettings().getAsMap().size(), equalTo(0));
}
cluster().wipe(); // wipe after to make sure we fail in the test that didn't ack the delete
cluster().assertAfterTest();
immutableCluster().wipe(); // wipe after to make sure we fail in the test that didn't ack the delete
immutableCluster().assertAfterTest();
} finally {
if (currentClusterScope == Scope.TEST) {
clearClusters(); // it is ok to leave persistent / transient cluster state behind if scope is TEST
@ -279,32 +279,40 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
}
}
public static TestCluster cluster() {
public static ImmutableTestCluster immutableCluster() {
return currentCluster;
}
public static TestCluster cluster() {
if (!(currentCluster instanceof TestCluster)) {
throw new UnsupportedOperationException("current test cluster is immutable");
}
return (TestCluster) currentCluster;
}
public ClusterService clusterService() {
return cluster().clusterService();
}
public static Client client() {
Client client = cluster().client();
Client client = immutableCluster().client();
if (frequently()) {
client = new RandomizingClient((InternalClient) client, getRandom());
}
return client;
}
public static Iterable<Client> clients() {
return cluster();
return immutableCluster();
}
protected int minimumNumberOfShards() {
return TestCluster.DEFAULT_MIN_NUM_SHARDS;
return ImmutableTestCluster.DEFAULT_MIN_NUM_SHARDS;
}
protected int maximumNumberOfShards() {
return TestCluster.DEFAULT_MAX_NUM_SHARDS;
return ImmutableTestCluster.DEFAULT_MAX_NUM_SHARDS;
}
protected int numberOfShards() {
@ -357,7 +365,7 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
success = true;
} finally {
if (!success && !created.isEmpty()) {
cluster().wipeIndices(created.toArray(new String[created.size()]));
immutableCluster().wipeIndices(created.toArray(new String[created.size()]));
}
}
}

269
src/test/java/org/elasticsearch/test/ImmutableTestCluster.java Normal file
View File

@ -0,0 +1,269 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test;
import com.carrotsearch.hppc.ObjectArrayList;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.merge.policy.*;
import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider;
import org.elasticsearch.index.merge.scheduler.MergeSchedulerModule;
import org.elasticsearch.index.merge.scheduler.MergeSchedulerProvider;
import org.elasticsearch.index.merge.scheduler.SerialMergeSchedulerProvider;
import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.indices.IndexTemplateMissingException;
import org.elasticsearch.repositories.RepositoryMissingException;
import org.elasticsearch.search.SearchService;
import java.util.Arrays;
import java.util.Random;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllFilesClosed;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSearchersClosed;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertThat;
/**
* Base test cluster that exposes the basis to run tests against any elasticsearch cluster, whose layout
* (e.g. number of nodes) is predefined and cannot be changed during the tests execution
*/
public abstract class ImmutableTestCluster implements Iterable<Client> {
private final ESLogger logger = Loggers.getLogger(getClass());
/**
* Key used to retrieve the index random seed from the index settings on a running node.
* The value of this seed can be used to initialize a random context for a specific index.
* It's set once per test via a generic index template.
*/
public static final String SETTING_INDEX_SEED = "index.tests.seed";
public static final int DEFAULT_MIN_NUM_SHARDS = 1;
public static final int DEFAULT_MAX_NUM_SHARDS = 10;
protected Random random;
protected double transportClientRatio = 0.0;
/**
* This method should be executed before each test to reset the cluster to its initial state.
*/
public void beforeTest(Random random, double transportClientRatio) {
assert transportClientRatio >= 0.0 && transportClientRatio <= 1.0;
logger.debug("Reset test cluster with transport client ratio: [{}]", transportClientRatio);
this.transportClientRatio = transportClientRatio;
this.random = new Random(random.nextLong());
}
/**
* Wipes any data that a test can leave behind: indices, templates and repositories
*/
public void wipe() {
wipeIndices("_all");
wipeTemplates();
wipeRepositories();
}
/**
* This method checks all the things that need to be checked after each test
*/
public void assertAfterTest() {
assertAllSearchersClosed();
assertAllFilesClosed();
ensureEstimatedStats();
}
/**
* This method should be executed during tear down, after each test (but after assertAfterTest)
*/
public abstract void afterTest();
/**
* Returns a client connected to any node in the cluster
*/
public abstract Client client();
/**
* Returns the size of the cluster
*/
public abstract int size();
/**
* Closes the current cluster
*/
public abstract void close();
/**
* Deletes the given indices from the tests cluster. If no index name is passed to this method
* all indices are removed.
*/
public void wipeIndices(String... indices) {
assert indices != null && indices.length > 0;
if (size() > 0) {
try {
assertAcked(client().admin().indices().prepareDelete(indices));
} catch (IndexMissingException e) {
// ignore
} catch (ElasticsearchIllegalArgumentException e) {
// Happens if `action.destructive_requires_name` is set to true
// which is the case in the CloseIndexDisableCloseAllTests
if ("_all".equals(indices[0])) {
ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().execute().actionGet();
ObjectArrayList<String> concreteIndices = new ObjectArrayList<String>();
for (IndexMetaData indexMetaData : clusterStateResponse.getState().metaData()) {
concreteIndices.add(indexMetaData.getIndex());
}
if (!concreteIndices.isEmpty()) {
assertAcked(client().admin().indices().prepareDelete(concreteIndices.toArray(String.class)));
}
}
}
}
}
/**
* Deletes index templates, support wildcard notation.
* If no template name is passed to this method all templates are removed.
*/
public void wipeTemplates(String... templates) {
if (size() > 0) {
// if nothing is provided, delete all
if (templates.length == 0) {
templates = new String[]{"*"};
}
for (String template : templates) {
try {
client().admin().indices().prepareDeleteTemplate(template).execute().actionGet();
} catch (IndexTemplateMissingException e) {
// ignore
}
}
}
}
/**
* Deletes repositories, supports wildcard notation.
*/
public void wipeRepositories(String... repositories) {
if (size() > 0) {
// if nothing is provided, delete all
if (repositories.length == 0) {
repositories = new String[]{"*"};
}
for (String repository : repositories) {
try {
client().admin().cluster().prepareDeleteRepository(repository).execute().actionGet();
} catch (RepositoryMissingException ex) {
// ignore
}
}
}
}
/**
* Ensures that the breaker statistics are reset to 0 since we wiped all indices and that
* means all stats should be set to 0 otherwise something is wrong with the field data
* calculation.
*/
public void ensureEstimatedStats() {
if (size() > 0) {
NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats()
.clear().setBreaker(true).execute().actionGet();
for (NodeStats stats : nodeStats.getNodes()) {
assertThat("Breaker not reset to 0 on node: " + stats.getNode(),
stats.getBreaker().getEstimated(), equalTo(0L));
}
}
}
/**
* Creates a randomized index template. This template is used to pass in randomized settings on a
* per index basis.
*/
public void randomIndexTemplate() {
// TODO move settings for random directory etc here into the index based randomized settings.
if (size() > 0) {
ImmutableSettings.Builder builder = setRandomNormsLoading(setRandomMerge(random, ImmutableSettings.builder())
.put(SETTING_INDEX_SEED, random.nextLong()))
.put(SETTING_NUMBER_OF_SHARDS, RandomInts.randomIntBetween(random, DEFAULT_MIN_NUM_SHARDS, DEFAULT_MAX_NUM_SHARDS))
.put(SETTING_NUMBER_OF_REPLICAS, RandomInts.randomIntBetween(random, 0, 1));
client().admin().indices().preparePutTemplate("random_index_template")
.setTemplate("*")
.setOrder(0)
.setSettings(builder)
.execute().actionGet();
}
}
private ImmutableSettings.Builder setRandomNormsLoading(ImmutableSettings.Builder builder) {
if (random.nextBoolean()) {
builder.put(SearchService.NORMS_LOADING_KEY, RandomPicks.randomFrom(random, Arrays.asList(FieldMapper.Loading.EAGER, FieldMapper.Loading.LAZY)));
}
return builder;
}
private static ImmutableSettings.Builder setRandomMerge(Random random, ImmutableSettings.Builder builder) {
if (random.nextBoolean()) {
builder.put(AbstractMergePolicyProvider.INDEX_COMPOUND_FORMAT,
random.nextBoolean() ? random.nextDouble() : random.nextBoolean());
}
Class<? extends MergePolicyProvider<?>> mergePolicy = TieredMergePolicyProvider.class;
switch (random.nextInt(5)) {
case 4:
mergePolicy = LogByteSizeMergePolicyProvider.class;
break;
case 3:
mergePolicy = LogDocMergePolicyProvider.class;
break;
case 0:
mergePolicy = null;
}
if (mergePolicy != null) {
builder.put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, mergePolicy.getName());
}
if (random.nextBoolean()) {
builder.put(MergeSchedulerProvider.FORCE_ASYNC_MERGE, random.nextBoolean());
}
switch (random.nextInt(5)) {
case 4:
builder.put(MergeSchedulerModule.MERGE_SCHEDULER_TYPE_KEY, SerialMergeSchedulerProvider.class.getName());
break;
case 3:
builder.put(MergeSchedulerModule.MERGE_SCHEDULER_TYPE_KEY, ConcurrentMergeSchedulerProvider.class.getName());
break;
}
return builder;
}
}

229
src/test/java/org/elasticsearch/test/TestCluster.java
View File

@ -18,9 +18,7 @@
*/
package org.elasticsearch.test;
import com.carrotsearch.hppc.ObjectArrayList;
import com.carrotsearch.randomizedtesting.SeedUtils;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
@ -28,20 +26,16 @@ import com.google.common.collect.Collections2;
import com.google.common.collect.Iterators;
import com.google.common.collect.Sets;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.cache.recycler.CacheRecycler;
import org.elasticsearch.cache.recycler.PageCacheRecyclerModule;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.routing.ShardRouting;
@ -57,17 +51,8 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.BigArraysModule;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.engine.IndexEngineModule;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.merge.policy.*;
import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider;
import org.elasticsearch.index.merge.scheduler.MergeSchedulerModule;
import org.elasticsearch.index.merge.scheduler.MergeSchedulerProvider;
import org.elasticsearch.index.merge.scheduler.SerialMergeSchedulerProvider;
import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.indices.IndexTemplateMissingException;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.internal.InternalNode;
import org.elasticsearch.repositories.RepositoryMissingException;
import org.elasticsearch.search.SearchService;
import org.elasticsearch.test.cache.recycler.MockBigArraysModule;
import org.elasticsearch.test.cache.recycler.MockPageCacheRecyclerModule;
@ -82,7 +67,6 @@ import org.junit.Assert;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@ -91,15 +75,8 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAs
import static com.google.common.collect.Maps.newTreeMap;
import static org.apache.lucene.util.LuceneTestCase.rarely;
import static org.apache.lucene.util.LuceneTestCase.usually;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllFilesClosed;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSearchersClosed;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertThat;
/**
* TestCluster manages a set of JVM private nodes and allows convenient access to them.
@ -112,7 +89,7 @@ import static org.junit.Assert.assertThat;
* are involved reproducibility is very limited. This class should only be used through {@link ElasticsearchIntegrationTest}.
* </p>
*/
public final class TestCluster implements Iterable<Client> {
public final class TestCluster extends ImmutableTestCluster {
private final ESLogger logger = Loggers.getLogger(getClass());
@ -130,23 +107,11 @@ public final class TestCluster implements Iterable<Client> {
*/
public static final String SETTING_CLUSTER_NODE_SEED = "test.cluster.node.seed";
/**
* Key used to retrieve the index random seed from the index settings on a running node.
* The value of this seed can be used to initialize a random context for a specific index.
* It's set once per test via a generic index template.
*/
public static final String SETTING_INDEX_SEED = "index.tests.seed";
private static final String CLUSTER_NAME_KEY = "cluster.name";
private static final boolean ENABLE_MOCK_MODULES = systemPropertyAsBoolean(TESTS_ENABLE_MOCK_MODULES, true);
static final int DEFAULT_MIN_NUM_NODES = 2;
static final int DEFAULT_MAX_NUM_NODES = 6;
public static final int DEFAULT_MIN_NUM_SHARDS = 1;
public static final int DEFAULT_MAX_NUM_SHARDS = 10;
/* sorted map to make traverse order reproducible */
private final TreeMap<String, NodeAndClient> nodes = newTreeMap();
@ -158,8 +123,6 @@ public final class TestCluster implements Iterable<Client> {
private final Settings defaultSettings;
private Random random;
private AtomicInteger nextNodeId = new AtomicInteger(0);
/* Each shared node has a node seed that is used to start up the node and get default settings
@ -167,8 +130,6 @@ public final class TestCluster implements Iterable<Client> {
* fully shared cluster to be more reproducible */
private final long[] sharedNodesSeeds;
private double transportClientRatio = 0.0;
private final NodeSettingsSource nodeSettingsSource;
public TestCluster(long clusterSeed, String clusterName) {
@ -245,15 +206,15 @@ public final class TestCluster implements Iterable<Client> {
.put(getRandomNodeSettings(nodeSeed));
Settings settings = nodeSettingsSource.settings(nodeOrdinal);
if (settings != null) {
if (settings.get(CLUSTER_NAME_KEY) != null) {
throw new ElasticsearchIllegalStateException("Tests must not set a '" + CLUSTER_NAME_KEY + "' as a node setting set '" + CLUSTER_NAME_KEY + "': [" + settings.get(CLUSTER_NAME_KEY) + "]");
if (settings.get(ClusterName.SETTING) != null) {
throw new ElasticsearchIllegalStateException("Tests must not set a '" + ClusterName.SETTING + "' as a node setting set '" + ClusterName.SETTING + "': [" + settings.get(ClusterName.SETTING) + "]");
}
builder.put(settings);
}
if (others != null) {
builder.put(others);
}
builder.put(CLUSTER_NAME_KEY, clusterName);
builder.put(ClusterName.SETTING, clusterName);
return builder.build();
}
@ -422,6 +383,7 @@ public final class TestCluster implements Iterable<Client> {
return "node_" + id;
}
@Override
public synchronized Client client() {
ensureOpen();
/* Randomly return a client to one of the nodes in the cluster */
@ -522,6 +484,7 @@ public final class TestCluster implements Iterable<Client> {
return null;
}
@Override
public void close() {
ensureOpen();
if (this.open.compareAndSet(true, false)) {
@ -668,7 +631,7 @@ public final class TestCluster implements Iterable<Client> {
TransportAddress addr = ((InternalNode) node).injector().getInstance(TransportService.class).boundAddress().publishAddress();
TransportClient client = new TransportClient(settingsBuilder().put("client.transport.nodes_sampler_interval", "1s")
.put("name", "transport_client_" + node.settings().get("name"))
.put(CLUSTER_NAME_KEY, clusterName).put("client.transport.sniff", sniff).build());
.put(ClusterName.SETTING, clusterName).put("client.transport.sniff", sniff).build());
client.addTransportAddress(addr);
return client;
}
@ -693,18 +656,13 @@ public final class TestCluster implements Iterable<Client> {
}
}
/**
* This method should be executed before each test to reset the cluster to it's initial state.
*/
@Override
public synchronized void beforeTest(Random random, double transportClientRatio) {
reset(random, true, transportClientRatio);
super.beforeTest(random, transportClientRatio);
reset(true);
}
private synchronized void reset(Random random, boolean wipeData, double transportClientRatio) {
assert transportClientRatio >= 0.0 && transportClientRatio <= 1.0;
logger.debug("Reset test cluster with transport client ratio: [{}]", transportClientRatio);
this.transportClientRatio = transportClientRatio;
this.random = new Random(random.nextLong());
private synchronized void reset(boolean wipeData) {
resetClients(); /* reset all clients - each test gets its own client based on the Random instance created above. */
if (wipeData) {
wipeDataDirectories();
@ -758,170 +716,12 @@ public final class TestCluster implements Iterable<Client> {
logger.debug("Cluster is consistent again - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), sharedNodesSeeds.length);
}
public void wipe() {
wipeIndices("_all");
wipeTemplates();
wipeRepositories();
}
/**
* Deletes the given indices from the tests cluster. If no index name is passed to this method
* all indices are removed.
*/
public void wipeIndices(String... indices) {
assert indices != null && indices.length > 0;
if (size() > 0) {
try {
assertAcked(client().admin().indices().prepareDelete(indices));
} catch (IndexMissingException e) {
// ignore
} catch (ElasticsearchIllegalArgumentException e) {
// Happens if `action.destructive_requires_name` is set to true
// which is the case in the CloseIndexDisableCloseAllTests
if ("_all".equals(indices[0])) {
ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().execute().actionGet();
ObjectArrayList<String> concreteIndices = new ObjectArrayList<>();
for (IndexMetaData indexMetaData : clusterStateResponse.getState().metaData()) {
concreteIndices.add(indexMetaData.getIndex());
}
if (!concreteIndices.isEmpty()) {
assertAcked(client().admin().indices().prepareDelete(concreteIndices.toArray(String.class)));
}
}
}
}
}
/**
* Deletes index templates, support wildcard notation.
* If no template name is passed to this method all templates are removed.
*/
public void wipeTemplates(String... templates) {
if (size() > 0) {
// if nothing is provided, delete all
if (templates.length == 0) {
templates = new String[]{"*"};
}
for (String template : templates) {
try {
client().admin().indices().prepareDeleteTemplate(template).execute().actionGet();
} catch (IndexTemplateMissingException e) {
// ignore
}
}
}
}
/**
* Deletes repositories, supports wildcard notation.
*/
public void wipeRepositories(String... repositories) {
if (size() > 0) {
// if nothing is provided, delete all
if (repositories.length == 0) {
repositories = new String[]{"*"};
}
for (String repository : repositories) {
try {
client().admin().cluster().prepareDeleteRepository(repository).execute().actionGet();
} catch (RepositoryMissingException ex) {
// ignore
}
}
}
}
/**
* Ensures that the breaker statistics are reset to 0 since we wiped all indices and that
* means all stats should be set to 0 otherwise something is wrong with the field data
* calculation.
*/
public void ensureEstimatedStats() {
if (size() > 0) {
NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats()
.clear().setBreaker(true).execute().actionGet();
for (NodeStats stats : nodeStats.getNodes()) {
assertThat("Breaker not reset to 0 on node: " + stats.getNode(),
stats.getBreaker().getEstimated(), equalTo(0L));
}
}
}
/**
* Creates a randomized index template. This template is used to pass in randomized settings on a
* per index basis.
*/
public void randomIndexTemplate() {
// TODO move settings for random directory etc here into the index based randomized settings.
if (size() > 0) {
ImmutableSettings.Builder builder = setRandomNormsLoading(setRandomMerge(random, ImmutableSettings.builder())
.put(SETTING_INDEX_SEED, random.nextLong()))
.put(SETTING_NUMBER_OF_SHARDS, RandomInts.randomIntBetween(random, DEFAULT_MIN_NUM_SHARDS, DEFAULT_MAX_NUM_SHARDS))
.put(SETTING_NUMBER_OF_REPLICAS, RandomInts.randomIntBetween(random, 0, 1));
client().admin().indices().preparePutTemplate("random_index_template")
.setTemplate("*")
.setOrder(0)
.setSettings(builder)
.execute().actionGet();
}
}
private ImmutableSettings.Builder setRandomNormsLoading(ImmutableSettings.Builder builder) {
if (random.nextBoolean()) {
builder.put(SearchService.NORMS_LOADING_KEY, RandomPicks.randomFrom(random, Arrays.asList(FieldMapper.Loading.EAGER, FieldMapper.Loading.LAZY)));
}
return builder;
}
private static ImmutableSettings.Builder setRandomMerge(Random random, ImmutableSettings.Builder builder) {
if (random.nextBoolean()) {
builder.put(AbstractMergePolicyProvider.INDEX_COMPOUND_FORMAT,
random.nextBoolean() ? random.nextDouble() : random.nextBoolean());
}
Class<? extends MergePolicyProvider<?>> mergePolicy = TieredMergePolicyProvider.class;
switch (random.nextInt(5)) {
case 4:
mergePolicy = LogByteSizeMergePolicyProvider.class;
break;
case 3:
mergePolicy = LogDocMergePolicyProvider.class;
break;
case 0:
mergePolicy = null;
}
if (mergePolicy != null) {
builder.put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, mergePolicy.getName());
}
if (random.nextBoolean()) {
builder.put(MergeSchedulerProvider.FORCE_ASYNC_MERGE, random.nextBoolean());
}
switch (random.nextInt(5)) {
case 4:
builder.put(MergeSchedulerModule.MERGE_SCHEDULER_TYPE_KEY, SerialMergeSchedulerProvider.class.getName());
break;
case 3:
builder.put(MergeSchedulerModule.MERGE_SCHEDULER_TYPE_KEY, ConcurrentMergeSchedulerProvider.class.getName());
break;
}
return builder;
}
/**
* This method should be executed during tearDown
*/
@Override
public synchronized void afterTest() {
wipeDataDirectories();
resetClients(); /* reset all clients - each test gets its own client based on the Random instance created above. */
}
public void assertAfterTest() throws IOException {
assertAllSearchersClosed();
assertAllFilesClosed();
ensureEstimatedStats();
}
private void resetClients() {
final Collection<NodeAndClient> nodesAndClients = nodes.values();
for (NodeAndClient nodeAndClient : nodesAndClients) {
@ -991,6 +791,7 @@ public final class TestCluster implements Iterable<Client> {
/**
* Returns the number of nodes in the cluster.
*/
@Override
public synchronized int size() {
return this.nodes.size();
}
@ -1234,7 +1035,7 @@ public final class TestCluster implements Iterable<Client> {
}
public void closeNonSharedNodes(boolean wipeData) {
reset(random, wipeData, transportClientRatio);
reset(wipeData);
}
public int dataNodes() {

4
src/test/java/org/elasticsearch/test/engine/MockInternalEngine.java
View File

@ -46,7 +46,7 @@ import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.indices.warmer.IndicesWarmer;
import org.elasticsearch.test.TestCluster;
import org.elasticsearch.test.ImmutableTestCluster;
import org.elasticsearch.threadpool.ThreadPool;
import java.lang.reflect.Constructor;
@ -72,7 +72,7 @@ public final class MockInternalEngine extends InternalEngine implements Engine {
CodecService codecService) throws EngineException {
super(shardId, indexSettings, threadPool, indexSettingsService, indexingService, warmer, store,
deletionPolicy, translog, mergePolicyProvider, mergeScheduler, analysisService, similarityService, codecService);
final long seed = indexSettings.getAsLong(TestCluster.SETTING_INDEX_SEED, 0l);
final long seed = indexSettings.getAsLong(ImmutableTestCluster.SETTING_INDEX_SEED, 0l);
random = new Random(seed);
final double ratio = indexSettings.getAsDouble(WRAP_READER_RATIO, 0.0d); // DISABLED by default - AssertingDR is crazy slow
wrapper = indexSettings.getAsClass(READER_WRAPPER_TYPE, AssertingDirectoryReader.class);

9
src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
View File

@ -22,6 +22,7 @@ import com.google.common.base.Predicate;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionRequest;
@ -469,7 +470,7 @@ public class ElasticsearchAssertions {
}
}
public static void assertAllFilesClosed() throws IOException {
public static void assertAllFilesClosed() {
try {
for (final MockDirectoryHelper.ElasticsearchMockDirectoryWrapper w : MockDirectoryHelper.wrappers) {
try {
@ -484,7 +485,11 @@ public class ElasticsearchAssertions {
}
if (!w.successfullyClosed()) {
if (w.closeException() == null) {
w.close();
try {
w.close();
} catch (IOException e) {
throw new ElasticsearchIllegalStateException("directory close threw IOException", e);
}
if (w.closeException() != null) {
throw w.closeException();
}

4
src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java
View File

@ -39,7 +39,7 @@ import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.store.fs.FsDirectoryService;
import org.elasticsearch.indices.IndicesLifecycle;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.test.TestCluster;
import org.elasticsearch.test.ImmutableTestCluster;
import java.io.File;
import java.io.IOException;
@ -56,7 +56,7 @@ public class MockFSDirectoryService extends FsDirectoryService {
@Inject
public MockFSDirectoryService(final ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, final IndicesService service) {
super(shardId, indexSettings, indexStore);
final long seed = indexSettings.getAsLong(TestCluster.SETTING_INDEX_SEED, 0l);
final long seed = indexSettings.getAsLong(ImmutableTestCluster.SETTING_INDEX_SEED, 0l);
Random random = new Random(seed);
helper = new MockDirectoryHelper(shardId, indexSettings, logger, random, seed);
checkIndexOnClose = indexSettings.getAsBoolean(CHECK_INDEX_ON_CLOSE, random.nextDouble() < 0.1);

4
src/test/java/org/elasticsearch/test/store/MockRamDirectoryService.java
View File

@ -25,7 +25,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.DirectoryService;
import org.elasticsearch.test.TestCluster;
import org.elasticsearch.test.ImmutableTestCluster;
import java.io.IOException;
import java.util.Random;
@ -38,7 +38,7 @@ public class MockRamDirectoryService extends AbstractIndexShardComponent impleme
@Inject
public MockRamDirectoryService(ShardId shardId, Settings indexSettings) {
super(shardId, indexSettings);
final long seed = indexSettings.getAsLong(TestCluster.SETTING_INDEX_SEED, 0l);
final long seed = indexSettings.getAsLong(ImmutableTestCluster.SETTING_INDEX_SEED, 0l);
Random random = new Random(seed);
helper = new MockDirectoryHelper(shardId, indexSettings, logger, random, seed);
delegateService = helper.randomRamDirectoryService();

4
src/test/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
View File

@ -24,7 +24,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.elasticsearch.test.TestCluster;
import org.elasticsearch.test.ImmutableTestCluster;
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.*;
@ -42,7 +42,7 @@ public class AssertingLocalTransport extends LocalTransport {
@Inject
public AssertingLocalTransport(Settings settings, ThreadPool threadPool, Version version) {
super(settings, threadPool, version);
final long seed = settings.getAsLong(TestCluster.SETTING_INDEX_SEED, 0l);
final long seed = settings.getAsLong(ImmutableTestCluster.SETTING_INDEX_SEED, 0l);
random = new Random(seed);
}