[TEST] introduced ImmutableTestCluster abstract base class for TestCluster

The new base class contains all the immutable methods for a cluster, which will be extended by the future ExternalCluster impl that relies on an external cluster and won't be adding nodes etc. but only sending requests to an existing cluster whose layout never changes

Closes #5620

src/test/java/org/elasticsearch/indices/fielddata/breaker/RandomExceptionCircuitBreakerTests.java

@@ -38,7 +38,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
-import org.elasticsearch.test.TestCluster;
+import org.elasticsearch.test.ImmutableTestCluster;
 import org.elasticsearch.test.engine.MockInternalEngine;
 import org.elasticsearch.test.engine.ThrowingAtomicReaderWrapper;
 import org.junit.Test;
@@ -203,7 +203,7 @@ public class RandomExceptionCircuitBreakerTests extends ElasticsearchIntegration
             private final double lowLevelRatio;
 
             ThrowingSubReaderWrapper(Settings settings) {
-                final long seed = settings.getAsLong(TestCluster.SETTING_INDEX_SEED, 0l);
+                final long seed = settings.getAsLong(ImmutableTestCluster.SETTING_INDEX_SEED, 0l);
                 this.topLevelRatio = settings.getAsDouble(EXCEPTION_TOP_LEVEL_RATIO_KEY, 0.1d);
                 this.lowLevelRatio = settings.getAsDouble(EXCEPTION_LOW_LEVEL_RATIO_KEY, 0.1d);
                 this.random = new Random(seed);

src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsTests.java

@@ -35,7 +35,7 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
-import org.elasticsearch.test.TestCluster;
+import org.elasticsearch.test.ImmutableTestCluster;
 import org.elasticsearch.test.engine.MockInternalEngine;
 import org.elasticsearch.test.engine.ThrowingAtomicReaderWrapper;
 import org.elasticsearch.test.store.MockDirectoryHelper;
@@ -288,7 +288,7 @@ public class SearchWithRandomExceptionsTests extends ElasticsearchIntegrationTes
             private final double lowLevelRatio;
 
             ThrowingSubReaderWrapper(Settings settings) {
-                final long seed = settings.getAsLong(TestCluster.SETTING_INDEX_SEED, 0l);
+                final long seed = settings.getAsLong(ImmutableTestCluster.SETTING_INDEX_SEED, 0l);
                 this.topLevelRatio = settings.getAsDouble(EXCEPTION_TOP_LEVEL_RATIO_KEY, 0.1d);
                 this.lowLevelRatio = settings.getAsDouble(EXCEPTION_LOW_LEVEL_RATIO_KEY, 0.1d);
                 this.random = new Random(seed);

src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java

@@ -162,11 +162,11 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
      * By default if no {@link ClusterScope} is configured this will hold a reference to the global cluster carried
      * on across test suites.
      */
-    private static TestCluster currentCluster;
+    private static ImmutableTestCluster currentCluster;
 
     private static final double TRANSPORT_CLIENT_RATIO = transportClientRatio();
 
-    private static final Map<Class<?>, TestCluster> clusters = new IdentityHashMap<>();
+    private static final Map<Class<?>, ImmutableTestCluster> clusters = new IdentityHashMap<>();
 
     @BeforeClass
     public static void beforeClass() throws Exception {
@@ -201,9 +201,9 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
                 default:
                     fail("Unknown Scope: [" + currentClusterScope + "]");
             }
-            currentCluster.beforeTest(getRandom(), getPerTestTransportClientRatio());
+            immutableCluster().beforeTest(getRandom(), getPerTestTransportClientRatio());
-            cluster().wipe();
+            immutableCluster().wipe();
-            cluster().randomIndexTemplate();
+            immutableCluster().randomIndexTemplate();
             logger.info("[{}#{}]: before test", getTestClass().getSimpleName(), getTestName());
         } catch (OutOfMemoryError e) {
             if (e.getMessage().contains("unable to create new native thread")) {
@@ -213,8 +213,8 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
         }
     }
 
-    public TestCluster buildAndPutCluster(Scope currentClusterScope, boolean createIfExists) throws IOException {
+    public ImmutableTestCluster buildAndPutCluster(Scope currentClusterScope, boolean createIfExists) throws IOException {
-        TestCluster testCluster = clusters.get(this.getClass());
+        ImmutableTestCluster testCluster = clusters.get(this.getClass());
         if (createIfExists || testCluster == null) {
             testCluster = buildTestCluster(currentClusterScope);
         } else {
@@ -227,7 +227,7 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
 
     private void clearClusters() throws IOException {
         if (!clusters.isEmpty()) {
-            for (TestCluster cluster : clusters.values()) {
+            for (ImmutableTestCluster cluster : clusters.values()) {
                 cluster.close();
             }
             clusters.clear();
@@ -248,8 +248,8 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
                     assertThat("test leaves transient cluster metadata behind: " + metaData.transientSettings().getAsMap(), metaData
                             .transientSettings().getAsMap().size(), equalTo(0));
                 }
-                cluster().wipe(); // wipe after to make sure we fail in the test that didn't ack the delete
+                immutableCluster().wipe(); // wipe after to make sure we fail in the test that didn't ack the delete
-                cluster().assertAfterTest();
+                immutableCluster().assertAfterTest();
             } finally {
                 if (currentClusterScope == Scope.TEST) {
                     clearClusters(); // it is ok to leave persistent / transient cluster state behind if scope is TEST
@@ -279,32 +279,40 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
         }
     }
 
-    public static TestCluster cluster() {
+    public static ImmutableTestCluster immutableCluster() {
         return currentCluster;
     }
 
+    public static TestCluster cluster() {
+        if (!(currentCluster instanceof  TestCluster)) {
+            throw new UnsupportedOperationException("current test cluster is immutable");
+        }
+        return (TestCluster) currentCluster;
+    }
+
     public ClusterService clusterService() {
         return cluster().clusterService();
     }
 
     public static Client client() {
-        Client client = cluster().client();
+        Client client = immutableCluster().client();
         if (frequently()) {
             client = new RandomizingClient((InternalClient) client, getRandom());
         }
         return client;
     }
 
+
     public static Iterable<Client> clients() {
-        return cluster();
+        return immutableCluster();
     }
 
     protected int minimumNumberOfShards() {
-        return TestCluster.DEFAULT_MIN_NUM_SHARDS;
+        return ImmutableTestCluster.DEFAULT_MIN_NUM_SHARDS;
     }
 
     protected int maximumNumberOfShards() {
-        return TestCluster.DEFAULT_MAX_NUM_SHARDS;
+        return ImmutableTestCluster.DEFAULT_MAX_NUM_SHARDS;
     }
 
     protected int numberOfShards() {
@@ -357,7 +365,7 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
                 success = true;
             } finally {
                 if (!success && !created.isEmpty()) {
-                    cluster().wipeIndices(created.toArray(new String[created.size()]));
+                    immutableCluster().wipeIndices(created.toArray(new String[created.size()]));
                 }
             }
         }

src/test/java/org/elasticsearch/test/ImmutableTestCluster.java

@@ -0,0 +1,269 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import com.carrotsearch.hppc.ObjectArrayList;
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.elasticsearch.ElasticsearchIllegalArgumentException;
+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.merge.policy.*;
+import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider;
+import org.elasticsearch.index.merge.scheduler.MergeSchedulerModule;
+import org.elasticsearch.index.merge.scheduler.MergeSchedulerProvider;
+import org.elasticsearch.index.merge.scheduler.SerialMergeSchedulerProvider;
+import org.elasticsearch.indices.IndexMissingException;
+import org.elasticsearch.indices.IndexTemplateMissingException;
+import org.elasticsearch.repositories.RepositoryMissingException;
+import org.elasticsearch.search.SearchService;
+
+import java.util.Arrays;
+import java.util.Random;
+
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllFilesClosed;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSearchersClosed;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Base test cluster that exposes the basis to run tests against any elasticsearch cluster, whose layout
+ * (e.g. number of nodes) is predefined and cannot be changed during the tests execution
+ */
+public abstract class ImmutableTestCluster implements Iterable<Client> {
+
+    private final ESLogger logger = Loggers.getLogger(getClass());
+
+    /**
+     * Key used to retrieve the index random seed from the index settings on a running node.
+     * The value of this seed can be used to initialize a random context for a specific index.
+     * It's set once per test via a generic index template.
+     */
+    public static final String SETTING_INDEX_SEED = "index.tests.seed";
+
+    public static final int DEFAULT_MIN_NUM_SHARDS = 1;
+    public static final int DEFAULT_MAX_NUM_SHARDS = 10;
+
+    protected Random random;
+
+    protected double transportClientRatio = 0.0;
+
+    /**
+     * This method should be executed before each test to reset the cluster to its initial state.
+     */
+    public void beforeTest(Random random, double transportClientRatio) {
+        assert transportClientRatio >= 0.0 && transportClientRatio <= 1.0;
+        logger.debug("Reset test cluster with transport client ratio: [{}]", transportClientRatio);
+        this.transportClientRatio = transportClientRatio;
+        this.random = new Random(random.nextLong());
+    }
+
+    /**
+     * Wipes any data that a test can leave behind: indices, templates and repositories
+     */
+    public void wipe() {
+        wipeIndices("_all");
+        wipeTemplates();
+        wipeRepositories();
+    }
+
+    /**
+     * This method checks all the things that need to be checked after each test
+     */
+    public void assertAfterTest() {
+        assertAllSearchersClosed();
+        assertAllFilesClosed();
+        ensureEstimatedStats();
+    }
+
+    /**
+     * This method should be executed during tear down, after each test (but after assertAfterTest)
+     */
+    public abstract void afterTest();
+
+    /**
+     * Returns a client connected to any node in the cluster
+     */
+    public abstract Client client();
+
+    /**
+     * Returns the size of the cluster
+     */
+    public abstract int size();
+
+    /**
+     * Closes the current cluster
+     */
+    public abstract void close();
+
+    /**
+     * Deletes the given indices from the tests cluster. If no index name is passed to this method
+     * all indices are removed.
+     */
+    public void wipeIndices(String... indices) {
+        assert indices != null && indices.length > 0;
+        if (size() > 0) {
+            try {
+                assertAcked(client().admin().indices().prepareDelete(indices));
+            } catch (IndexMissingException e) {
+                // ignore
+            } catch (ElasticsearchIllegalArgumentException e) {
+                // Happens if `action.destructive_requires_name` is set to true
+                // which is the case in the CloseIndexDisableCloseAllTests
+                if ("_all".equals(indices[0])) {
+                    ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().execute().actionGet();
+                    ObjectArrayList<String> concreteIndices = new ObjectArrayList<String>();
+                    for (IndexMetaData indexMetaData : clusterStateResponse.getState().metaData()) {
+                        concreteIndices.add(indexMetaData.getIndex());
+                    }
+                    if (!concreteIndices.isEmpty()) {
+                        assertAcked(client().admin().indices().prepareDelete(concreteIndices.toArray(String.class)));
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Deletes index templates, support wildcard notation.
+     * If no template name is passed to this method all templates are removed.
+     */
+    public void wipeTemplates(String... templates) {
+        if (size() > 0) {
+            // if nothing is provided, delete all
+            if (templates.length == 0) {
+                templates = new String[]{"*"};
+            }
+            for (String template : templates) {
+                try {
+                    client().admin().indices().prepareDeleteTemplate(template).execute().actionGet();
+                } catch (IndexTemplateMissingException e) {
+                    // ignore
+                }
+            }
+        }
+    }
+
+    /**
+     * Deletes repositories, supports wildcard notation.
+     */
+    public void wipeRepositories(String... repositories) {
+        if (size() > 0) {
+            // if nothing is provided, delete all
+            if (repositories.length == 0) {
+                repositories = new String[]{"*"};
+            }
+            for (String repository : repositories) {
+                try {
+                    client().admin().cluster().prepareDeleteRepository(repository).execute().actionGet();
+                } catch (RepositoryMissingException ex) {
+                    // ignore
+                }
+            }
+        }
+    }
+
+    /**
+     * Ensures that the breaker statistics are reset to 0 since we wiped all indices and that
+     * means all stats should be set to 0 otherwise something is wrong with the field data
+     * calculation.
+     */
+    public void ensureEstimatedStats() {
+        if (size() > 0) {
+            NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats()
+                    .clear().setBreaker(true).execute().actionGet();
+            for (NodeStats stats : nodeStats.getNodes()) {
+                assertThat("Breaker not reset to 0 on node: " + stats.getNode(),
+                        stats.getBreaker().getEstimated(), equalTo(0L));
+            }
+        }
+    }
+
+    /**
+     * Creates a randomized index template. This template is used to pass in randomized settings on a
+     * per index basis.
+     */
+    public void randomIndexTemplate() {
+        // TODO move settings for random directory etc here into the index based randomized settings.
+        if (size() > 0) {
+            ImmutableSettings.Builder builder = setRandomNormsLoading(setRandomMerge(random, ImmutableSettings.builder())
+                    .put(SETTING_INDEX_SEED, random.nextLong()))
+                    .put(SETTING_NUMBER_OF_SHARDS, RandomInts.randomIntBetween(random, DEFAULT_MIN_NUM_SHARDS, DEFAULT_MAX_NUM_SHARDS))
+                    .put(SETTING_NUMBER_OF_REPLICAS, RandomInts.randomIntBetween(random, 0, 1));
+            client().admin().indices().preparePutTemplate("random_index_template")
+                    .setTemplate("*")
+                    .setOrder(0)
+                    .setSettings(builder)
+                    .execute().actionGet();
+        }
+    }
+
+    private ImmutableSettings.Builder setRandomNormsLoading(ImmutableSettings.Builder builder) {
+        if (random.nextBoolean()) {
+            builder.put(SearchService.NORMS_LOADING_KEY, RandomPicks.randomFrom(random, Arrays.asList(FieldMapper.Loading.EAGER, FieldMapper.Loading.LAZY)));
+        }
+        return builder;
+    }
+
+    private static ImmutableSettings.Builder setRandomMerge(Random random, ImmutableSettings.Builder builder) {
+        if (random.nextBoolean()) {
+            builder.put(AbstractMergePolicyProvider.INDEX_COMPOUND_FORMAT,
+                    random.nextBoolean() ? random.nextDouble() : random.nextBoolean());
+        }
+        Class<? extends MergePolicyProvider<?>> mergePolicy = TieredMergePolicyProvider.class;
+        switch (random.nextInt(5)) {
+            case 4:
+                mergePolicy = LogByteSizeMergePolicyProvider.class;
+                break;
+            case 3:
+                mergePolicy = LogDocMergePolicyProvider.class;
+                break;
+            case 0:
+                mergePolicy = null;
+        }
+        if (mergePolicy != null) {
+            builder.put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, mergePolicy.getName());
+        }
+
+        if (random.nextBoolean()) {
+            builder.put(MergeSchedulerProvider.FORCE_ASYNC_MERGE, random.nextBoolean());
+        }
+        switch (random.nextInt(5)) {
+            case 4:
+                builder.put(MergeSchedulerModule.MERGE_SCHEDULER_TYPE_KEY, SerialMergeSchedulerProvider.class.getName());
+                break;
+            case 3:
+                builder.put(MergeSchedulerModule.MERGE_SCHEDULER_TYPE_KEY, ConcurrentMergeSchedulerProvider.class.getName());
+                break;
+        }
+
+        return builder;
+    }
+}

src/test/java/org/elasticsearch/test/TestCluster.java

@@ -18,9 +18,7 @@
  */
 package org.elasticsearch.test;
 
-import com.carrotsearch.hppc.ObjectArrayList;
 import com.carrotsearch.randomizedtesting.SeedUtils;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
@@ -28,20 +26,16 @@ import com.google.common.collect.Collections2;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Sets;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.ElasticsearchIllegalStateException;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
-import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.cache.recycler.CacheRecycler;
 import org.elasticsearch.cache.recycler.PageCacheRecyclerModule;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -57,17 +51,8 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArraysModule;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.engine.IndexEngineModule;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.merge.policy.*;
-import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider;
-import org.elasticsearch.index.merge.scheduler.MergeSchedulerModule;
-import org.elasticsearch.index.merge.scheduler.MergeSchedulerProvider;
-import org.elasticsearch.index.merge.scheduler.SerialMergeSchedulerProvider;
-import org.elasticsearch.indices.IndexMissingException;
-import org.elasticsearch.indices.IndexTemplateMissingException;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.internal.InternalNode;
-import org.elasticsearch.repositories.RepositoryMissingException;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.test.cache.recycler.MockBigArraysModule;
 import org.elasticsearch.test.cache.recycler.MockPageCacheRecyclerModule;
@@ -82,7 +67,6 @@ import org.junit.Assert;
 
 import java.io.Closeable;
 import java.io.File;
-import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -91,15 +75,8 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAs
 import static com.google.common.collect.Maps.newTreeMap;
 import static org.apache.lucene.util.LuceneTestCase.rarely;
 import static org.apache.lucene.util.LuceneTestCase.usually;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
 import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllFilesClosed;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSearchersClosed;
-import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertThat;
 
 /**
  * TestCluster manages a set of JVM private nodes and allows convenient access to them.
@@ -112,7 +89,7 @@ import static org.junit.Assert.assertThat;
  * are involved reproducibility is very limited. This class should only be used through {@link ElasticsearchIntegrationTest}.
  * </p>
  */
-public final class TestCluster implements Iterable<Client> {
+public final class TestCluster extends ImmutableTestCluster {
 
     private final ESLogger logger = Loggers.getLogger(getClass());
 
@@ -130,23 +107,11 @@ public final class TestCluster implements Iterable<Client> {
      */
     public static final String SETTING_CLUSTER_NODE_SEED = "test.cluster.node.seed";
 
-    /**
-     * Key used to retrieve the index random seed from the index settings on a running node.
-     * The value of this seed can be used to initialize a random context for a specific index.
-     * It's set once per test via a generic index template.
-     */
-    public static final String SETTING_INDEX_SEED = "index.tests.seed";
-
-    private static final String CLUSTER_NAME_KEY = "cluster.name";
-
     private static final boolean ENABLE_MOCK_MODULES = systemPropertyAsBoolean(TESTS_ENABLE_MOCK_MODULES, true);
 
     static final int DEFAULT_MIN_NUM_NODES = 2;
     static final int DEFAULT_MAX_NUM_NODES = 6;
 
-    public static final int DEFAULT_MIN_NUM_SHARDS = 1;
-    public static final int DEFAULT_MAX_NUM_SHARDS = 10;
-
     /* sorted map to make traverse order reproducible */
     private final TreeMap<String, NodeAndClient> nodes = newTreeMap();
 
@@ -158,8 +123,6 @@ public final class TestCluster implements Iterable<Client> {
 
     private final Settings defaultSettings;
 
-    private Random random;
-
     private AtomicInteger nextNodeId = new AtomicInteger(0);
 
     /* Each shared node has a node seed that is used to start up the node and get default settings
@@ -167,8 +130,6 @@ public final class TestCluster implements Iterable<Client> {
      * fully shared cluster to be more reproducible */
     private final long[] sharedNodesSeeds;
 
-    private double transportClientRatio = 0.0;
-
     private final NodeSettingsSource nodeSettingsSource;
 
     public TestCluster(long clusterSeed, String clusterName) {
@@ -245,15 +206,15 @@ public final class TestCluster implements Iterable<Client> {
                 .put(getRandomNodeSettings(nodeSeed));
         Settings settings = nodeSettingsSource.settings(nodeOrdinal);
         if (settings != null) {
-            if (settings.get(CLUSTER_NAME_KEY) != null) {
+            if (settings.get(ClusterName.SETTING) != null) {
-                throw new ElasticsearchIllegalStateException("Tests must not set a '" + CLUSTER_NAME_KEY + "' as a node setting set '" + CLUSTER_NAME_KEY + "': [" + settings.get(CLUSTER_NAME_KEY) + "]");
+                throw new ElasticsearchIllegalStateException("Tests must not set a '" + ClusterName.SETTING + "' as a node setting set '" + ClusterName.SETTING + "': [" + settings.get(ClusterName.SETTING) + "]");
             }
             builder.put(settings);
         }
         if (others != null) {
             builder.put(others);
         }
-        builder.put(CLUSTER_NAME_KEY, clusterName);
+        builder.put(ClusterName.SETTING, clusterName);
         return builder.build();
     }
 
@@ -422,6 +383,7 @@ public final class TestCluster implements Iterable<Client> {
         return "node_" + id;
     }
 
+    @Override
     public synchronized Client client() {
         ensureOpen();
         /* Randomly return a client to one of the nodes in the cluster */
@@ -522,6 +484,7 @@ public final class TestCluster implements Iterable<Client> {
         return null;
     }
 
+    @Override
     public void close() {
         ensureOpen();
         if (this.open.compareAndSet(true, false)) {
@@ -668,7 +631,7 @@ public final class TestCluster implements Iterable<Client> {
             TransportAddress addr = ((InternalNode) node).injector().getInstance(TransportService.class).boundAddress().publishAddress();
             TransportClient client = new TransportClient(settingsBuilder().put("client.transport.nodes_sampler_interval", "1s")
                     .put("name", "transport_client_" + node.settings().get("name"))
-                    .put(CLUSTER_NAME_KEY, clusterName).put("client.transport.sniff", sniff).build());
+                    .put(ClusterName.SETTING, clusterName).put("client.transport.sniff", sniff).build());
             client.addTransportAddress(addr);
             return client;
         }
@@ -693,18 +656,13 @@ public final class TestCluster implements Iterable<Client> {
         }
     }
 
-    /**
+    @Override
-     * This method should be executed before each test to reset the cluster to it's initial state.
-     */
     public synchronized void beforeTest(Random random, double transportClientRatio) {
-        reset(random, true, transportClientRatio);
+        super.beforeTest(random, transportClientRatio);
+        reset(true);
     }
 
-    private synchronized void reset(Random random, boolean wipeData, double transportClientRatio) {
+    private synchronized void reset(boolean wipeData) {
-        assert transportClientRatio >= 0.0 && transportClientRatio <= 1.0;
-        logger.debug("Reset test cluster with transport client ratio: [{}]", transportClientRatio);
-        this.transportClientRatio = transportClientRatio;
-        this.random = new Random(random.nextLong());
         resetClients(); /* reset all clients - each test gets its own client based on the Random instance created above. */
         if (wipeData) {
             wipeDataDirectories();
@@ -758,170 +716,12 @@ public final class TestCluster implements Iterable<Client> {
         logger.debug("Cluster is consistent again - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), sharedNodesSeeds.length);
     }
 
-    public void wipe() {
+    @Override
-        wipeIndices("_all");
-        wipeTemplates();
-        wipeRepositories();
-    }
-
-    /**
-     * Deletes the given indices from the tests cluster. If no index name is passed to this method
-     * all indices are removed.
-     */
-    public void wipeIndices(String... indices) {
-        assert indices != null && indices.length > 0;
-        if (size() > 0) {
-            try {
-                assertAcked(client().admin().indices().prepareDelete(indices));
-            } catch (IndexMissingException e) {
-                // ignore
-            } catch (ElasticsearchIllegalArgumentException e) {
-                // Happens if `action.destructive_requires_name` is set to true
-                // which is the case in the CloseIndexDisableCloseAllTests
-                if ("_all".equals(indices[0])) {
-                    ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().execute().actionGet();
-                    ObjectArrayList<String> concreteIndices = new ObjectArrayList<>();
-                    for (IndexMetaData indexMetaData : clusterStateResponse.getState().metaData()) {
-                        concreteIndices.add(indexMetaData.getIndex());
-                    }
-                    if (!concreteIndices.isEmpty()) {
-                        assertAcked(client().admin().indices().prepareDelete(concreteIndices.toArray(String.class)));
-                    }
-                }
-            }
-        }
-    }
-
-    /**
-     * Deletes index templates, support wildcard notation.
-     * If no template name is passed to this method all templates are removed.
-     */
-    public void wipeTemplates(String... templates) {
-        if (size() > 0) {
-            // if nothing is provided, delete all
-            if (templates.length == 0) {
-                templates = new String[]{"*"};
-            }
-            for (String template : templates) {
-                try {
-                    client().admin().indices().prepareDeleteTemplate(template).execute().actionGet();
-                } catch (IndexTemplateMissingException e) {
-                    // ignore
-                }
-            }
-        }
-    }
-
-    /**
-     * Deletes repositories, supports wildcard notation.
-     */
-    public void wipeRepositories(String... repositories) {
-        if (size() > 0) {
-            // if nothing is provided, delete all
-            if (repositories.length == 0) {
-                repositories = new String[]{"*"};
-            }
-            for (String repository : repositories) {
-                try {
-                    client().admin().cluster().prepareDeleteRepository(repository).execute().actionGet();
-                } catch (RepositoryMissingException ex) {
-                    // ignore
-                }
-            }
-        }
-    }
-
-    /**
-     * Ensures that the breaker statistics are reset to 0 since we wiped all indices and that
-     * means all stats should be set to 0 otherwise something is wrong with the field data
-     * calculation.
-     */
-    public void ensureEstimatedStats() {
-        if (size() > 0) {
-            NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats()
-                    .clear().setBreaker(true).execute().actionGet();
-            for (NodeStats stats : nodeStats.getNodes()) {
-                assertThat("Breaker not reset to 0 on node: " + stats.getNode(),
-                        stats.getBreaker().getEstimated(), equalTo(0L));
-            }
-        }
-    }
-
-    /**
-     * Creates a randomized index template. This template is used to pass in randomized settings on a
-     * per index basis.
-     */
-    public void randomIndexTemplate() {
-        // TODO move settings for random directory etc here into the index based randomized settings.
-        if (size() > 0) {
-            ImmutableSettings.Builder builder = setRandomNormsLoading(setRandomMerge(random, ImmutableSettings.builder())
-                    .put(SETTING_INDEX_SEED, random.nextLong()))
-                    .put(SETTING_NUMBER_OF_SHARDS, RandomInts.randomIntBetween(random, DEFAULT_MIN_NUM_SHARDS, DEFAULT_MAX_NUM_SHARDS))
-                    .put(SETTING_NUMBER_OF_REPLICAS, RandomInts.randomIntBetween(random, 0, 1));
-            client().admin().indices().preparePutTemplate("random_index_template")
-                    .setTemplate("*")
-                    .setOrder(0)
-                    .setSettings(builder)
-                    .execute().actionGet();
-        }
-    }
-
-    private ImmutableSettings.Builder setRandomNormsLoading(ImmutableSettings.Builder builder) {
-        if (random.nextBoolean()) {
-            builder.put(SearchService.NORMS_LOADING_KEY, RandomPicks.randomFrom(random, Arrays.asList(FieldMapper.Loading.EAGER, FieldMapper.Loading.LAZY)));
-        }
-        return builder;
-    }
-
-    private static ImmutableSettings.Builder setRandomMerge(Random random, ImmutableSettings.Builder builder) {
-        if (random.nextBoolean()) {
-            builder.put(AbstractMergePolicyProvider.INDEX_COMPOUND_FORMAT,
-                    random.nextBoolean() ? random.nextDouble() : random.nextBoolean());
-        }
-        Class<? extends MergePolicyProvider<?>> mergePolicy = TieredMergePolicyProvider.class;
-        switch (random.nextInt(5)) {
-            case 4:
-                mergePolicy = LogByteSizeMergePolicyProvider.class;
-                break;
-            case 3:
-                mergePolicy = LogDocMergePolicyProvider.class;
-                break;
-            case 0:
-                mergePolicy = null;
-        }
-        if (mergePolicy != null) {
-            builder.put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, mergePolicy.getName());
-        }
-
-        if (random.nextBoolean()) {
-            builder.put(MergeSchedulerProvider.FORCE_ASYNC_MERGE, random.nextBoolean());
-        }
-        switch (random.nextInt(5)) {
-            case 4:
-                builder.put(MergeSchedulerModule.MERGE_SCHEDULER_TYPE_KEY, SerialMergeSchedulerProvider.class.getName());
-                break;
-            case 3:
-                builder.put(MergeSchedulerModule.MERGE_SCHEDULER_TYPE_KEY, ConcurrentMergeSchedulerProvider.class.getName());
-                break;
-        }
-
-        return builder;
-    }
-
-    /**
-     * This method should be executed during tearDown
-     */
     public synchronized void afterTest() {
         wipeDataDirectories();
         resetClients(); /* reset all clients - each test gets its own client based on the Random instance created above. */
     }
 
-    public void assertAfterTest() throws IOException {
-        assertAllSearchersClosed();
-        assertAllFilesClosed();
-        ensureEstimatedStats();
-    }
-
     private void resetClients() {
         final Collection<NodeAndClient> nodesAndClients = nodes.values();
         for (NodeAndClient nodeAndClient : nodesAndClients) {
@@ -991,6 +791,7 @@ public final class TestCluster implements Iterable<Client> {
     /**
      * Returns the number of nodes in the cluster.
      */
+    @Override
     public synchronized int size() {
         return this.nodes.size();
     }
@@ -1234,7 +1035,7 @@ public final class TestCluster implements Iterable<Client> {
     }
 
     public void closeNonSharedNodes(boolean wipeData) {
-        reset(random, wipeData, transportClientRatio);
+        reset(wipeData);
     }
 
     public int dataNodes() {

src/test/java/org/elasticsearch/test/engine/MockInternalEngine.java

@@ -46,7 +46,7 @@ import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.warmer.IndicesWarmer;
-import org.elasticsearch.test.TestCluster;
+import org.elasticsearch.test.ImmutableTestCluster;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.lang.reflect.Constructor;
@@ -72,7 +72,7 @@ public final class MockInternalEngine extends InternalEngine implements Engine {
                               CodecService codecService) throws EngineException {
         super(shardId, indexSettings, threadPool, indexSettingsService, indexingService, warmer, store,
                 deletionPolicy, translog, mergePolicyProvider, mergeScheduler, analysisService, similarityService, codecService);
-        final long seed = indexSettings.getAsLong(TestCluster.SETTING_INDEX_SEED, 0l);
+        final long seed = indexSettings.getAsLong(ImmutableTestCluster.SETTING_INDEX_SEED, 0l);
         random = new Random(seed);
         final double ratio = indexSettings.getAsDouble(WRAP_READER_RATIO, 0.0d); // DISABLED by default - AssertingDR is crazy slow
         wrapper = indexSettings.getAsClass(READER_WRAPPER_TYPE, AssertingDirectoryReader.class);

src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java

@@ -22,6 +22,7 @@ import com.google.common.base.Predicate;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchIllegalStateException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionRequest;
@@ -469,7 +470,7 @@ public class ElasticsearchAssertions {
         }
     }
 
-    public static void assertAllFilesClosed() throws IOException {
+    public static void assertAllFilesClosed() {
         try {
             for (final MockDirectoryHelper.ElasticsearchMockDirectoryWrapper w : MockDirectoryHelper.wrappers) {
                 try {
@@ -484,7 +485,11 @@ public class ElasticsearchAssertions {
                 }
                 if (!w.successfullyClosed()) {
                     if (w.closeException() == null) {
+                        try {
                             w.close();
+                        } catch (IOException e) {
+                            throw new ElasticsearchIllegalStateException("directory close threw IOException", e);
+                        }
                         if (w.closeException() != null) {
                             throw w.closeException();
                         }

src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java

@@ -39,7 +39,7 @@ import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.store.fs.FsDirectoryService;
 import org.elasticsearch.indices.IndicesLifecycle;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.test.TestCluster;
+import org.elasticsearch.test.ImmutableTestCluster;
 
 import java.io.File;
 import java.io.IOException;
@@ -56,7 +56,7 @@ public class MockFSDirectoryService extends FsDirectoryService {
     @Inject
     public MockFSDirectoryService(final ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, final IndicesService service) {
         super(shardId, indexSettings, indexStore);
-        final long seed = indexSettings.getAsLong(TestCluster.SETTING_INDEX_SEED, 0l);
+        final long seed = indexSettings.getAsLong(ImmutableTestCluster.SETTING_INDEX_SEED, 0l);
         Random random = new Random(seed);
         helper = new MockDirectoryHelper(shardId, indexSettings, logger, random, seed);
         checkIndexOnClose = indexSettings.getAsBoolean(CHECK_INDEX_ON_CLOSE, random.nextDouble() < 0.1);

src/test/java/org/elasticsearch/test/store/MockRamDirectoryService.java

@@ -25,7 +25,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.AbstractIndexShardComponent;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.DirectoryService;
-import org.elasticsearch.test.TestCluster;
+import org.elasticsearch.test.ImmutableTestCluster;
 
 import java.io.IOException;
 import java.util.Random;
@@ -38,7 +38,7 @@ public class MockRamDirectoryService extends AbstractIndexShardComponent impleme
     @Inject
     public MockRamDirectoryService(ShardId shardId, Settings indexSettings) {
         super(shardId, indexSettings);
-        final long seed = indexSettings.getAsLong(TestCluster.SETTING_INDEX_SEED, 0l);
+        final long seed = indexSettings.getAsLong(ImmutableTestCluster.SETTING_INDEX_SEED, 0l);
         Random random = new Random(seed);
         helper = new MockDirectoryHelper(shardId, indexSettings, logger, random, seed);
         delegateService = helper.randomRamDirectoryService();

src/test/java/org/elasticsearch/test/transport/AssertingLocalTransport.java

@@ -24,7 +24,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ElasticsearchTestCase;
-import org.elasticsearch.test.TestCluster;
+import org.elasticsearch.test.ImmutableTestCluster;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.*;
@@ -42,7 +42,7 @@ public class AssertingLocalTransport extends LocalTransport {
     @Inject
     public AssertingLocalTransport(Settings settings, ThreadPool threadPool, Version version) {
         super(settings, threadPool, version);
-        final long seed = settings.getAsLong(TestCluster.SETTING_INDEX_SEED, 0l);
+        final long seed = settings.getAsLong(ImmutableTestCluster.SETTING_INDEX_SEED, 0l);
         random = new Random(seed);
     }