diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index 5b9427a4b75..270059b09dc 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -371,7 +371,7 @@ public class TasksIT extends ESIntegTestCase { logger.info("--> started test tasks"); // Wait for the task to start on all nodes - assertBusy(() -> assertEquals(internalCluster().numDataAndMasterNodes(), + assertBusy(() -> assertEquals(internalCluster().size(), client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size())); logger.info("--> cancelling the main test task"); @@ -391,7 +391,7 @@ public class TasksIT extends ESIntegTestCase { ListenableActionFuture future = TestTaskPlugin.TestTaskAction.INSTANCE.newRequestBuilder(client()) .execute(); // Wait for the task to start on all nodes - assertBusy(() -> assertEquals(internalCluster().numDataAndMasterNodes(), + assertBusy(() -> assertEquals(internalCluster().size(), client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size())); TestTaskPlugin.UnblockTestTasksAction.INSTANCE.newRequestBuilder(client()).get(); @@ -409,7 +409,7 @@ public class TasksIT extends ESIntegTestCase { ListenableActionFuture waitResponseFuture; try { // Wait for the task to start on all nodes - assertBusy(() -> assertEquals(internalCluster().numDataAndMasterNodes(), client().admin().cluster().prepareListTasks() + assertBusy(() -> assertEquals(internalCluster().size(), client().admin().cluster().prepareListTasks() .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size())); // Spin up a request to wait for that task to finish @@ -437,7 +437,7 @@ public class TasksIT extends ESIntegTestCase { .execute(); try { // Wait for the task to start on all nodes - assertBusy(() -> assertEquals(internalCluster().numDataAndMasterNodes(), client().admin().cluster().prepareListTasks() + assertBusy(() -> assertEquals(internalCluster().size(), client().admin().cluster().prepareListTasks() .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size())); // Spin up a request that should wait for those tasks to finish diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java index 09f3d49e0a7..9874bdd44b1 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java @@ -276,7 +276,7 @@ public class TestTaskPlugin extends Plugin { protected String[] filterNodeIds(DiscoveryNodes nodes, String[] nodesIds) { List list = new ArrayList<>(); for (String node : nodesIds) { - if (nodes.getDataNodes().containsKey(node)) { + if (nodes.nodeExists(node)) { list.add(node); } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java index 1be7c626502..0a63dd46095 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.index.store.Store; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -37,12 +36,12 @@ import org.hamcrest.Matchers; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -@ClusterScope(scope = Scope.SUITE, numDataNodes = 1, numClientNodes = 0) +@ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class ClusterStatsIT extends ESIntegTestCase { private void assertCounts(ClusterStatsNodes.Counts counts, int total, Map roles) { @@ -58,12 +57,14 @@ public class ClusterStatsIT extends ESIntegTestCase { public void testNodeCounts() { int total = 1; + internalCluster().startNode(); Map expectedCounts = new HashMap<>(); expectedCounts.put(DiscoveryNode.Role.DATA.getRoleName(), 1); expectedCounts.put(DiscoveryNode.Role.MASTER.getRoleName(), 1); expectedCounts.put(DiscoveryNode.Role.INGEST.getRoleName(), 1); expectedCounts.put(ClusterStatsNodes.Counts.COORDINATING_ONLY, 0); int numNodes = randomIntBetween(1, 5); + ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); assertCounts(response.getNodesStats().getCounts(), total, expectedCounts); @@ -112,7 +113,9 @@ public class ClusterStatsIT extends ESIntegTestCase { assertThat(stats.getReplication(), Matchers.equalTo(replicationFactor)); } - public void testIndicesShardStats() { + public void testIndicesShardStats() throws ExecutionException, InterruptedException { + internalCluster().startNode(); + ensureGreen(); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); @@ -155,10 +158,8 @@ public class ClusterStatsIT extends ESIntegTestCase { } - public void testValuesSmokeScreen() throws IOException { - internalCluster().ensureAtMostNumDataNodes(5); - internalCluster().ensureAtLeastNumDataNodes(1); - assertAcked(prepareCreate("test1").setSettings(Settings.builder().put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0).build())); + public void testValuesSmokeScreen() throws IOException, ExecutionException, InterruptedException { + internalCluster().startNodesAsync(randomIntBetween(1, 3)).get(); index("test1", "type", "1", "f", "f"); /* * Ensure at least one shard is allocated otherwise the FS stats might @@ -188,9 +189,6 @@ public class ClusterStatsIT extends ESIntegTestCase { } public void testAllocatedProcessors() throws Exception { - // stop all other nodes - internalCluster().ensureAtMostNumDataNodes(0); - // start one node with 7 processors. internalCluster().startNodesAsync(Settings.builder().put(EsExecutors.PROCESSORS_SETTING.getKey(), 7).build()).get(); waitForNodes(1); @@ -200,14 +198,15 @@ public class ClusterStatsIT extends ESIntegTestCase { } public void testClusterStatusWhenStateNotRecovered() throws Exception { - // stop all other nodes - internalCluster().ensureAtMostNumDataNodes(0); - - internalCluster().startNode(Settings.builder().put("gateway.recover_after_nodes", 2).build()); + internalCluster().startMasterOnlyNode(Settings.builder().put("gateway.recover_after_nodes", 2).build()); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); assertThat(response.getStatus(), equalTo(ClusterHealthStatus.RED)); - internalCluster().ensureAtLeastNumDataNodes(3); + if (randomBoolean()) { + internalCluster().startMasterOnlyNode(Settings.EMPTY); + } else { + internalCluster().startDataOnlyNode(Settings.EMPTY); + } // wait for the cluster status to settle ensureGreen(); response = client().admin().cluster().prepareClusterStats().get(); diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java index 4ec1f66df57..ed9136851b4 100644 --- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java +++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.env.Environment; import org.elasticsearch.node.Node; -import org.elasticsearch.node.internal.InternalSettingsPreparer; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -41,7 +40,7 @@ import java.util.concurrent.ExecutionException; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -@ClusterScope(scope = Scope.TEST, numClientNodes = 0) +@ClusterScope(scope = Scope.TEST, numClientNodes = 0, supportsDedicatedMasters = false) public class TransportClientRetryIT extends ESIntegTestCase { public void testRetry() throws IOException, ExecutionException, InterruptedException { Iterable instances = internalCluster().getInstances(TransportService.class); diff --git a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java index e61bbc5f719..194b82620de 100644 --- a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java @@ -37,7 +37,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; -@ClusterScope(scope = SUITE, numDataNodes = 1) +@ClusterScope(scope = SUITE, supportsDedicatedMasters = false, numDataNodes = 1) public class SettingsFilteringIT extends ESIntegTestCase { @Override diff --git a/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java b/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java index 84cd23a7f09..5e8a99d82fe 100644 --- a/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java @@ -43,7 +43,7 @@ public class ClusterSearchShardsIT extends ESIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { - switch(nodeOrdinal) { + switch(nodeOrdinal % 2) { case 1: return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put("node.attr.tag", "B").build(); case 0: diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java index eeda96743df..ba6b4438aaa 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java +++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java @@ -41,7 +41,7 @@ import static org.hamcrest.Matchers.hasSize; /** * */ -@ClusterScope(scope = Scope.TEST, numDataNodes = 1) +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) public class NettyHttpRequestSizeLimitIT extends ESIntegTestCase { private static final ByteSizeValue LIMIT = new ByteSizeValue(2, ByteSizeUnit.KB); diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningDisabledIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningDisabledIT.java index f048b5526d5..576456c0647 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningDisabledIT.java +++ b/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningDisabledIT.java @@ -40,7 +40,7 @@ import static org.hamcrest.Matchers.hasSize; /** * */ -@ClusterScope(scope = Scope.TEST, numDataNodes = 1) +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) public class NettyPipeliningDisabledIT extends ESIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningEnabledIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningEnabledIT.java index fd1c493ac36..7ca714ece93 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningEnabledIT.java +++ b/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningEnabledIT.java @@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; -@ClusterScope(scope = Scope.TEST, numDataNodes = 1) +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) public class NettyPipeliningEnabledIT extends ESIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { diff --git a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java index e9e8dcfc007..07afef1d8ae 100644 --- a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java +++ b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java @@ -34,7 +34,7 @@ import java.util.Collections; import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -@ClusterScope(scope = SUITE, numDataNodes = 1, numClientNodes = 0) +@ClusterScope(scope = SUITE, supportsDedicatedMasters = false, numDataNodes = 1, numClientNodes = 0) public class SettingsListenerIT extends ESIntegTestCase { @Override @@ -109,13 +109,13 @@ public class SettingsListenerIT extends ESIntegTestCase { .put("index.test.new.setting", 21) .build()).get()); - for (SettingsTestingService instance : internalCluster().getInstances(SettingsTestingService.class)) { + for (SettingsTestingService instance : internalCluster().getDataNodeInstances(SettingsTestingService.class)) { assertEquals(21, instance.value); } client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder() .put("index.test.new.setting", 42)).get(); - for (SettingsTestingService instance : internalCluster().getInstances(SettingsTestingService.class)) { + for (SettingsTestingService instance : internalCluster().getDataNodeInstances(SettingsTestingService.class)) { assertEquals(42, instance.value); } @@ -123,14 +123,14 @@ public class SettingsListenerIT extends ESIntegTestCase { .put("index.test.new.setting", 21) .build()).get()); - for (SettingsTestingService instance : internalCluster().getInstances(SettingsTestingService.class)) { + for (SettingsTestingService instance : internalCluster().getDataNodeInstances(SettingsTestingService.class)) { assertEquals(42, instance.value); } client().admin().indices().prepareUpdateSettings("other").setSettings(Settings.builder() .put("index.test.new.setting", 84)).get(); - for (SettingsTestingService instance : internalCluster().getInstances(SettingsTestingService.class)) { + for (SettingsTestingService instance : internalCluster().getDataNodeInstances(SettingsTestingService.class)) { assertEquals(42, instance.value); } diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java index cf56f41c83a..110e34d59a9 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java @@ -37,7 +37,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -@ClusterScope(numDataNodes = 1, scope = Scope.SUITE) +@ClusterScope(supportsDedicatedMasters = false, numDataNodes = 1, scope = Scope.SUITE) public class InternalEngineMergeIT extends ESIntegTestCase { @TestLogging("_root:DEBUG") diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java index df8736b44ef..c3323919c82 100644 --- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java +++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.store; +import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.IndexFileNames; @@ -26,6 +27,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Requests; @@ -48,6 +50,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.PrimaryShardAllocator; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; @@ -65,7 +68,6 @@ import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.CorruptionUtils; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.MockIndexEventListener; import org.elasticsearch.test.store.MockFSIndexStore; import org.elasticsearch.test.transport.MockTransportService; @@ -84,9 +86,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.CopyOnWriteArrayList; @@ -113,13 +113,13 @@ public class CorruptedFileIT extends ESIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() - // we really need local GW here since this also checks for corruption etc. - // and we need to make sure primaries are not just trashed if we don't have replicas - .put(super.nodeSettings(nodeOrdinal)) - // speed up recoveries - .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), 5) - .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), 5) - .build(); + // we really need local GW here since this also checks for corruption etc. + // and we need to make sure primaries are not just trashed if we don't have replicas + .put(super.nodeSettings(nodeOrdinal)) + // speed up recoveries + .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), 5) + .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), 5) + .build(); } @Override @@ -142,11 +142,11 @@ public class CorruptedFileIT extends ESIntegTestCase { assertThat(cluster().numDataNodes(), greaterThanOrEqualTo(3)); assertAcked(prepareCreate("test").setSettings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1") - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) - .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1") + .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files )); ensureGreen(); disableAllocation("test"); @@ -171,9 +171,9 @@ public class CorruptedFileIT extends ESIntegTestCase { Settings build = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "2").build(); client().admin().indices().prepareUpdateSettings("test").setSettings(build).get(); ClusterHealthResponse health = client().admin().cluster() - .health(Requests.clusterHealthRequest("test").waitForGreenStatus() - .timeout("5m") // sometimes due to cluster rebalacing and random settings default timeout is just not enough. - .waitForRelocatingShards(0)).actionGet(); + .health(Requests.clusterHealthRequest("test").waitForGreenStatus() + .timeout("5m") // sometimes due to cluster rebalacing and random settings default timeout is just not enough. + .waitForRelocatingShards(0)).actionGet(); if (health.isTimedOut()) { logger.info("cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint()); assertThat("timed out waiting for green state", health.isTimedOut(), equalTo(false)); @@ -247,10 +247,10 @@ public class CorruptedFileIT extends ESIntegTestCase { internalCluster().ensureAtLeastNumDataNodes(2); assertAcked(prepareCreate("test").setSettings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) - .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") + .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files )); ensureGreen(); IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; @@ -273,12 +273,12 @@ public class CorruptedFileIT extends ESIntegTestCase { client().admin().cluster().prepareReroute().get(); boolean didClusterTurnRed = awaitBusy(() -> { - ClusterHealthStatus test = client().admin().cluster() - .health(Requests.clusterHealthRequest("test")).actionGet().getStatus(); - return test == ClusterHealthStatus.RED; + ClusterHealthStatus test = client().admin().cluster() + .health(Requests.clusterHealthRequest("test")).actionGet().getStatus(); + return test == ClusterHealthStatus.RED; }, 5, TimeUnit.MINUTES);// sometimes on slow nodes the replication / recovery is just dead slow final ClusterHealthResponse response = client().admin().cluster() - .health(Requests.clusterHealthRequest("test")).get(); + .health(Requests.clusterHealthRequest("test")).get(); if (response.getStatus() != ClusterHealthStatus.RED) { logger.info("Cluster turned red in busy loop: {}", didClusterTurnRed); logger.info("cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint()); @@ -328,10 +328,10 @@ public class CorruptedFileIT extends ESIntegTestCase { NodeStats primariesNode = dataNodeStats.get(0); NodeStats unluckyNode = dataNodeStats.get(1); assertAcked(prepareCreate("test").setSettings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put("index.routing.allocation.include._name", primariesNode.getNode().getName()) - .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.routing.allocation.include._name", primariesNode.getNode().getName()) + .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE) )); ensureGreen(); // allocated with empty commit @@ -356,8 +356,8 @@ public class CorruptedFileIT extends ESIntegTestCase { } Settings build = Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1") - .put("index.routing.allocation.include._name", primariesNode.getNode().getName() + "," + unluckyNode.getNode().getName()).build(); + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1") + .put("index.routing.allocation.include._name", primariesNode.getNode().getName() + "," + unluckyNode.getNode().getName()).build(); client().admin().indices().prepareUpdateSettings("test").setSettings(build).get(); client().admin().cluster().prepareReroute().get(); hasCorrupted.await(); @@ -390,12 +390,12 @@ public class CorruptedFileIT extends ESIntegTestCase { assertAcked(prepareCreate("test").setSettings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(1, 4)) // don't go crazy here it must recovery fast - // This does corrupt files on the replica, so we can't check: - .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) - .put("index.routing.allocation.include._name", primariesNode.getNode().getName()) - .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(1, 4)) // don't go crazy here it must recovery fast + // This does corrupt files on the replica, so we can't check: + .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) + .put("index.routing.allocation.include._name", primariesNode.getNode().getName()) + .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE) )); ensureGreen(); IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; @@ -432,12 +432,12 @@ public class CorruptedFileIT extends ESIntegTestCase { } Settings build = Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1") - .put("index.routing.allocation.include._name", "*").build(); + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1") + .put("index.routing.allocation.include._name", "*").build(); client().admin().indices().prepareUpdateSettings("test").setSettings(build).get(); client().admin().cluster().prepareReroute().get(); ClusterHealthResponse actionGet = client().admin().cluster() - .health(Requests.clusterHealthRequest("test").waitForGreenStatus()).actionGet(); + .health(Requests.clusterHealthRequest("test").waitForGreenStatus()).actionGet(); if (actionGet.isTimedOut()) { logger.info("ensureGreen timed out, cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint()); assertThat("timed out waiting for green state", actionGet.isTimedOut(), equalTo(false)); @@ -525,11 +525,12 @@ public class CorruptedFileIT extends ESIntegTestCase { internalCluster().ensureAtLeastNumDataNodes(2); assertAcked(prepareCreate("test").setSettings(Settings.builder() - .put(PrimaryShardAllocator.INDEX_RECOVERY_INITIAL_SHARDS_SETTING.getKey(), "one") - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, cluster().numDataNodes() - 1) - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) - .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files + .put(PrimaryShardAllocator.INDEX_RECOVERY_INITIAL_SHARDS_SETTING.getKey(), "one") + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, cluster().numDataNodes() - 1) + .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), + new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files )); ensureGreen(); IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; @@ -543,22 +544,41 @@ public class CorruptedFileIT extends ESIntegTestCase { SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); - final Map> filesToCorrupt = findFilesToCorruptForReplica(); - internalCluster().fullRestart(new InternalTestCluster.RestartCallback() { - @Override - public Settings onNodeStopped(String nodeName) throws Exception { - List paths = filesToCorrupt.get(nodeName); - if (paths != null) { - for (Path path : paths) { + // disable allocations of replicas post restart (the restart will change replicas to primaries, so we have + // to capture replicas post restart) + assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings( + Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "primaries") + )); + + internalCluster().fullRestart(); + + ensureYellow(); + + final Index index = resolveIndex("test"); + + final IndicesShardStoresResponse stores = client().admin().indices().prepareShardStores(index.getName()).get(); + + + for (IntObjectCursor> shards : + stores.getStoreStatuses().get(index.getName())) { + for (IndicesShardStoresResponse.StoreStatus store : shards.value) { + final ShardId shardId = new ShardId(index, shards.key); + if (store.getAllocationStatus().equals(IndicesShardStoresResponse.StoreStatus.AllocationStatus.UNUSED)) { + for (Path path : findFilesToCorruptOnNode(store.getNode().getName(), shardId)) { try (OutputStream os = Files.newOutputStream(path)) { os.write(0); } - logger.info("corrupting file {} on node {}", path, nodeName); + logger.info("corrupting file {} on node {}", path, store.getNode().getName()); } } - return null; } - }); + } + + // enable allocation + assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings( + Settings.builder().putNull(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey()) + )); + ensureGreen(); } @@ -568,34 +588,21 @@ public class CorruptedFileIT extends ESIntegTestCase { return shardIterators.size(); } - private Map> findFilesToCorruptForReplica() throws IOException { - Map> filesToNodes = new HashMap<>(); - ClusterState state = client().admin().cluster().prepareState().get().getState(); - Index test = state.metaData().index("test").getIndex(); - for (ShardRouting shardRouting : state.getRoutingTable().allShards("test")) { - if (shardRouting.primary()) { - continue; - } - assertTrue(shardRouting.assignedToNode()); - NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(shardRouting.currentNodeId()).setFs(true).get(); - NodeStats nodeStats = nodeStatses.getNodes().get(0); - List files = new ArrayList<>(); - filesToNodes.put(nodeStats.getNode().getName(), files); - for (FsInfo.Path info : nodeStats.getFs()) { - String path = info.getPath(); - Path file = PathUtils.get(path).resolve("indices").resolve(test.getUUID()).resolve(Integer.toString(shardRouting.getId())).resolve("index"); - if (Files.exists(file)) { // multi data path might only have one path in use - try (DirectoryStream stream = Files.newDirectoryStream(file)) { - for (Path item : stream) { - if (item.getFileName().toString().startsWith("segments_")) { - files.add(item); - } + private List findFilesToCorruptOnNode(final String nodeName, final ShardId shardId) throws IOException { + List files = new ArrayList<>(); + for (Path path : internalCluster().getInstance(NodeEnvironment.class, nodeName).availableShardPaths(shardId)) { + path = path.resolve("index"); + if (Files.exists(path)) { // multi data path might only have one path in use + try (DirectoryStream stream = Files.newDirectoryStream(path)) { + for (Path item : stream) { + if (item.getFileName().toString().startsWith("segments_")) { + files.add(item); } } } } } - return filesToNodes; + return files; } private ShardRouting corruptRandomPrimaryFile() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index 92c3260aeb0..66aed327f70 100644 --- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -39,16 +39,16 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.MergePolicyConfig; +import org.elasticsearch.index.MergeSchedulerConfig; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.cache.query.QueryCacheStats; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.MergePolicyConfig; -import org.elasticsearch.index.MergeSchedulerConfig; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.translog.Translog; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.IndicesRequestCache; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -103,7 +103,7 @@ public class IndexStatsIT extends ESIntegTestCase { client().prepareIndex("test", "type", "2").setSource("field", "value2", "field2", "value2").execute().actionGet(); client().admin().indices().prepareRefresh().execute().actionGet(); - NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet(); + NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet(); assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L)); IndicesStatsResponse indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet(); assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L)); @@ -112,7 +112,7 @@ public class IndexStatsIT extends ESIntegTestCase { client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet(); client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet(); - nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet(); + nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet(); assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L)); indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet(); assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0L)); @@ -122,7 +122,7 @@ public class IndexStatsIT extends ESIntegTestCase { client().prepareSearch().addSort("field2", SortOrder.ASC).execute().actionGet(); // now check the per field stats - nodesStats = client().admin().cluster().prepareNodesStats().setIndices(new CommonStatsFlags().set(CommonStatsFlags.Flag.FieldData, true).fieldDataFields("*")).execute().actionGet(); + nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(new CommonStatsFlags().set(CommonStatsFlags.Flag.FieldData, true).fieldDataFields("*")).execute().actionGet(); assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L)); assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes().get(1).getIndices().getFieldData().getFields().get("field"), greaterThan(0L)); assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes().get(1).getIndices().getFieldData().getFields().get("field"), lessThan(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes())); @@ -133,7 +133,7 @@ public class IndexStatsIT extends ESIntegTestCase { assertThat(indicesStats.getTotal().getFieldData().getFields().get("field"), lessThan(indicesStats.getTotal().getFieldData().getMemorySizeInBytes())); client().admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet(); - nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet(); + nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet(); assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L)); indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet(); assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L)); @@ -150,7 +150,7 @@ public class IndexStatsIT extends ESIntegTestCase { client().prepareIndex("test", "type", "2").setSource("field", "value2").execute().actionGet(); client().admin().indices().prepareRefresh().execute().actionGet(); - NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true) + NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true) .execute().actionGet(); assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L)); assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L)); @@ -171,7 +171,7 @@ public class IndexStatsIT extends ESIntegTestCase { .addSort("field", SortOrder.ASC) .execute().actionGet(); - nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true) + nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true) .execute().actionGet(); assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L)); assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), greaterThan(0L)); @@ -184,7 +184,7 @@ public class IndexStatsIT extends ESIntegTestCase { client().admin().indices().prepareClearCache().execute().actionGet(); Thread.sleep(100); // Make sure the filter cache entries have been removed... - nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true) + nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true) .execute().actionGet(); assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L)); assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L)); diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java index f839cd2583f..683ae71a112 100644 --- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java +++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java @@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.is; /** * Tests that when disabling detailed errors, a request with the error_trace parameter returns a HTTP 400 */ -@ClusterScope(scope = Scope.TEST, numDataNodes = 1) +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) public class DetailedErrorsDisabledIT extends ESIntegTestCase { // Build our cluster settings @Override diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java index 6b9d78e5468..d98db83dddc 100644 --- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java +++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java @@ -23,7 +23,6 @@ import org.apache.http.impl.client.HttpClients; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.http.HttpServerTransport; -import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -37,7 +36,7 @@ import static org.hamcrest.Matchers.not; /** * Tests that by default the error_trace parameter can be used to show stacktraces */ -@ClusterScope(scope = Scope.TEST, numDataNodes = 1) +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) public class DetailedErrorsEnabledIT extends ESIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { diff --git a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java index 85d83e69b75..ac6d8fddd89 100644 --- a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java +++ b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java @@ -35,7 +35,7 @@ import static org.hamcrest.Matchers.equalTo; /** * Test a rest action that sets special response headers */ -@ClusterScope(scope = Scope.SUITE, numDataNodes = 1) +@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) public class ResponseHeaderPluginIT extends ESIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { @@ -60,4 +60,4 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase { assertThat(authResponse, hasStatus(OK)); assertThat(authResponse.getHeaders().get("Secret"), equalTo("granted")); } -} \ No newline at end of file +} diff --git a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java index 9e043c660f4..dfdd88d1987 100644 --- a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java +++ b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java @@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.not; /** * */ -@ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 1) +@ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) public class CorsNotSetIT extends ESIntegTestCase { @Override diff --git a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java index 837b0523072..18351dcb294 100644 --- a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java +++ b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java @@ -39,7 +39,7 @@ import static org.hamcrest.Matchers.not; /** * Test CORS where the allow origin value is a regular expression. */ -@ClusterScope(scope = Scope.SUITE, numDataNodes = 1) +@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) public class CorsRegexIT extends ESIntegTestCase { protected static final ESLogger logger = Loggers.getLogger(CorsRegexIT.class); diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java index ea76dbf7d6c..cf8830aa7ef 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java @@ -58,7 +58,7 @@ import static org.hamcrest.CoreMatchers.equalTo; /** * */ -@ClusterScope(scope = Scope.SUITE, numDataNodes = 1) +@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) public class FetchSubPhasePluginIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java index 489c6bf908f..9999f96e0b1 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java @@ -56,7 +56,7 @@ import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -@ClusterScope(scope = Scope.SUITE, numDataNodes = 1) +@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) public class ExplainableScriptIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java index 2d3c593c895..7fdeca4680e 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java @@ -53,7 +53,7 @@ import static org.hamcrest.Matchers.equalTo; /** * */ -@ClusterScope(scope = Scope.SUITE, numDataNodes = 1) +@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) public class FunctionScorePluginIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { diff --git a/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchIT.java index 07045cc9176..4b5153b2417 100644 --- a/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchIT.java @@ -37,7 +37,7 @@ import static org.hamcrest.Matchers.equalTo; /** * */ -@ClusterScope(scope = Scope.SUITE, numDataNodes = 1) +@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) public class CustomHighlighterSearchIT extends ESIntegTestCase { @Override diff --git a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 7e0bb173e40..102173626f3 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -71,7 +71,8 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase { public static long getFailureCount(String repository) { long failureCount = 0; - for (RepositoriesService repositoriesService : internalCluster().getDataNodeInstances(RepositoriesService.class)) { + for (RepositoriesService repositoriesService : + internalCluster().getDataOrMasterNodeInstances(RepositoriesService.class)) { MockRepository mockRepository = (MockRepository) repositoriesService.repository(repository); failureCount += mockRepository.getFailureCount(); } diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 7ca30132a4a..e9d10d31a04 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -563,6 +563,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } } } catch (Exception ex) { + logger.info("--> caught a top level exception, asserting what's expected", ex); assertThat(getFailureCount("test-repo"), greaterThan(0L)); assertThat(ExceptionsHelper.detailedMessage(ex), containsString("IOException")); } diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java index 10e20928376..f01a4ca5e92 100644 --- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java +++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java @@ -53,7 +53,7 @@ import static org.hamcrest.Matchers.is; /** * */ -@ClusterScope(scope = Scope.TEST, numDataNodes = 1) +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) public class NettyTransportIT extends ESIntegTestCase { // static so we can use it in anonymous classes private static String channelProfileName = null; diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java index 7f673e00fee..0189b73799a 100644 --- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java @@ -47,7 +47,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; -@ClusterScope(scope = Scope.SUITE, numDataNodes = 1, numClientNodes = 0) +@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1, numClientNodes = 0) public class NettyTransportMultiPortIntegrationIT extends ESIntegTestCase { private static int randomPort = -1; diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java index 52827491fe4..6ea466d3cb3 100644 --- a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java +++ b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java @@ -94,7 +94,7 @@ public class TribeIT extends ESIntegTestCase { } }; - cluster2 = new InternalTestCluster(InternalTestCluster.configuredNodeMode(), randomLong(), createTempDir(), 2, 2, + cluster2 = new InternalTestCluster(InternalTestCluster.configuredNodeMode(), randomLong(), createTempDir(), true, 2, 2, UUIDs.randomBase64UUID(random()), nodeConfigurationSource, 0, false, SECOND_CLUSTER_NODE_PREFIX, Collections.emptyList(), Function.identity()); cluster2.beforeTest(random(), 0.1); diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java index f216f998305..f6a74faac39 100644 --- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java +++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java @@ -49,7 +49,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -@ClusterScope(scope= Scope.SUITE, numDataNodes = 1) +@ClusterScope(scope= Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) public class SimpleTTLIT extends ESIntegTestCase { static private final long PURGE_INTERVAL = 200; diff --git a/docs/reference/testing/testing-framework.asciidoc b/docs/reference/testing/testing-framework.asciidoc index e0b27733441..fe7daf8cca4 100644 --- a/docs/reference/testing/testing-framework.asciidoc +++ b/docs/reference/testing/testing-framework.asciidoc @@ -148,13 +148,19 @@ You can use the `@ClusterScope` annotation at class level to configure this beha [source,java] ----------------------------------------- -@ClusterScope(scope=TEST, numNodes=1) +@ClusterScope(scope=TEST, numDataNodes=1) public class CustomSuggesterSearchTests extends ESIntegTestCase { // ... tests go here } ----------------------------------------- -The above sample configures the test to use a new cluster for each test method. The default scope is `SUITE` (one cluster for all test methods in the test). The `numNodes` settings allows you to only start a certain number of nodes, which can speed up test execution, as starting a new node is a costly and time consuming operation and might not be needed for this test. +The above sample configures the test to use a new cluster for each test method. The default scope is `SUITE` (one cluster for all +test methods in the test). The `numDataNodes` settings allows you to only start a certain number of data nodes, which can speed up test +execution, as starting a new node is a costly and time consuming operation and might not be needed for this test. + +By default, the testing infrastructure will randomly start dedicated master nodes. If you want to disable dedicated masters +you can set `supportsDedicatedMasters=false` in a similar fashion to the `numDataNodes` setting. If dedicated master nodes are not used, +data nodes will be allowed to become masters as well. [[changing-node-configuration]] diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java index 6c2bec3ba43..489f7043b83 100644 --- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java +++ b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java @@ -277,9 +277,9 @@ public class AzureDiscoveryClusterFormationTests extends ESIntegTestCase { public void testJoin() throws ExecutionException, InterruptedException { // only wait for the cluster to form - assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(2)).get()); + ensureClusterSizeConsistency(); // add one more node and wait for it to join internalCluster().startDataOnlyNodeAsync().get(); - assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(3)).get()); + ensureClusterSizeConsistency(); } } diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java index 6d1104ce6e2..0271f522943 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java @@ -57,7 +57,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTi import static org.hamcrest.Matchers.equalTo; @ESIntegTestCase.SuppressLocalMode -@ESIntegTestCase.ClusterScope(numDataNodes = 2, numClientNodes = 0) +@ESIntegTestCase.ClusterScope(supportsDedicatedMasters = false, numDataNodes = 2, numClientNodes = 0) @SuppressForbidden(reason = "use http server") // TODO this should be a IT but currently all ITs in this project run against a real cluster public class Ec2DiscoveryClusterFormationTests extends ESIntegTestCase { diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java index dbedbe1a6a9..f6a850d1d23 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java @@ -57,7 +57,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTi @ESIntegTestCase.SuppressLocalMode -@ESIntegTestCase.ClusterScope(numDataNodes = 2, numClientNodes = 0) +@ESIntegTestCase.ClusterScope(supportsDedicatedMasters = false, numDataNodes = 2, numClientNodes = 0) @SuppressForbidden(reason = "use http server") // TODO this should be a IT but currently all ITs in this project run against a real cluster public class GceDiscoverTests extends ESIntegTestCase { diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java index 2c50deafb59..9ac15cae5a2 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java @@ -36,6 +36,7 @@ import static org.hamcrest.Matchers.greaterThan; @ESIntegTestCase.ClusterScope( scope = ESIntegTestCase.Scope.SUITE, + supportsDedicatedMasters = false, numDataNodes = 1, numClientNodes = 0, transportClientRatio = 0.0) diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java index 79b41a355b7..c062e765c32 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java @@ -62,7 +62,7 @@ import static org.hamcrest.Matchers.greaterThan; */ @ClusterScope( scope = ESIntegTestCase.Scope.SUITE, - numDataNodes = 1, + supportsDedicatedMasters = false, numDataNodes = 1, transportClientRatio = 0.0) public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyIntegTestCase { private String getRepositoryPath() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 8f62617ee7b..2a3eecf4cc8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -108,8 +108,8 @@ import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.index.translog.Translog; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.IndicesRequestCache; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.node.NodeMocksPlugin; import org.elasticsearch.plugins.Plugin; @@ -1026,7 +1026,7 @@ public abstract class ESIntegTestCase extends ESTestCase { logger.info("memory: {}", XContentHelper.toString(client().admin().cluster().prepareNodesStats().clear().setJvm(true).get())); } - void ensureClusterSizeConsistency() { + protected void ensureClusterSizeConsistency() { if (cluster() != null) { // if static init fails the cluster can be null logger.trace("Check consistency for [{}] nodes", cluster().size()); assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(cluster().size())).get()); @@ -1478,17 +1478,24 @@ public abstract class ESIntegTestCase extends ESTestCase { int numDataNodes() default -1; /** - * Returns the minimum number of nodes in the cluster. Default is -1. + * Returns the minimum number of data nodes in the cluster. Default is -1. * Ignored when {@link ClusterScope#numDataNodes()} is set. */ int minNumDataNodes() default -1; /** - * Returns the maximum number of nodes in the cluster. Default is -1. + * Returns the maximum number of data nodes in the cluster. Default is -1. * Ignored when {@link ClusterScope#numDataNodes()} is set. */ int maxNumDataNodes() default -1; + /** + * Indicates whether the cluster can have dedicated master nodes. If false means data nodes will serve as master nodes + * and there will be no dedicated master (and data) nodes. Default is true which means + * dedicated master nodes will be randomly used. + */ + boolean supportsDedicatedMasters() default true; + /** * Returns the number of client nodes in the cluster. Default is {@link InternalTestCluster#DEFAULT_NUM_CLIENT_NODES}, a * negative value means that the number of client nodes will be randomized. @@ -1571,7 +1578,6 @@ public abstract class ESIntegTestCase extends ESTestCase { return getAnnotation(clazz.getSuperclass(), annotationClass); } - private Scope getCurrentClusterScope() { return getCurrentClusterScope(this.getClass()); } @@ -1582,6 +1588,11 @@ public abstract class ESIntegTestCase extends ESTestCase { return annotation == null ? Scope.SUITE : annotation.scope(); } + private boolean getSupportsDedicatedMasters() { + ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class); + return annotation == null ? true : annotation.supportsDedicatedMasters(); + } + private int getNumDataNodes() { ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class); return annotation == null ? -1 : annotation.numDataNodes(); @@ -1717,6 +1728,7 @@ public abstract class ESIntegTestCase extends ESTestCase { } }; + boolean supportsDedicatedMasters = getSupportsDedicatedMasters(); int numDataNodes = getNumDataNodes(); int minNumDataNodes; int maxNumDataNodes; @@ -1739,7 +1751,7 @@ public abstract class ESIntegTestCase extends ESTestCase { Collection> mockPlugins = getMockPlugins(); - return new InternalTestCluster(nodeMode, seed, createTempDir(), minNumDataNodes, maxNumDataNodes, + return new InternalTestCluster(nodeMode, seed, createTempDir(), supportsDedicatedMasters, minNumDataNodes, maxNumDataNodes, InternalTestCluster.clusterName(scope.name(), seed) + "-cluster", nodeConfigurationSource, getNumClientNodes(), InternalTestCluster.DEFAULT_ENABLE_HTTP_PIPELINING, nodePrefix, mockPlugins, getClientWrapper()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index aa1d0aa29ef..c67ffc07a2b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -161,7 +161,9 @@ public final class InternalTestCluster extends TestCluster { private static final int JVM_ORDINAL = Integer.parseInt(System.getProperty(SysGlobals.CHILDVM_SYSPROP_JVM_ID, "0")); - /** a per-JVM unique offset to be used for calculating unique port ranges. */ + /** + * a per-JVM unique offset to be used for calculating unique port ranges. + */ public static final int JVM_BASE_PORT_OFFSET = PORTS_PER_JVM * (JVM_ORDINAL + 1); private static final AtomicInteger clusterOrdinal = new AtomicInteger(); @@ -171,6 +173,9 @@ public final class InternalTestCluster extends TestCluster { public final int HTTP_BASE_PORT = GLOBAL_HTTP_BASE_PORT + CLUSTER_BASE_PORT_OFFSET; + static final int DEFAULT_LOW_NUM_MASTER_NODES = 1; + static final int DEFAULT_HIGH_NUM_MASTER_NODES = 3; + static final int DEFAULT_MIN_NUM_DATA_NODES = 1; static final int DEFAULT_MAX_NUM_DATA_NODES = TEST_NIGHTLY ? 6 : 3; @@ -198,9 +203,13 @@ public final class InternalTestCluster extends TestCluster { * fully shared cluster to be more reproducible */ private final long[] sharedNodesSeeds; - private final int numSharedAllRolesNodes; - private final int numShareCoordOnlyNodes; + // if set to 0, data nodes will also assume the master role + private final int numSharedDedicatedMasterNodes; + + private final int numSharedDataNodes; + + private final int numSharedCoordOnlyNodes; private final NodeConfigurationSource nodeConfigurationSource; @@ -219,6 +228,7 @@ public final class InternalTestCluster extends TestCluster { private Function clientWrapper; public InternalTestCluster(String nodeMode, long clusterSeed, Path baseDir, + boolean randomlyAddDedicatedMasters, int minNumDataNodes, int maxNumDataNodes, String clusterName, NodeConfigurationSource nodeConfigurationSource, int numClientNodes, boolean enableHttpPipelining, String nodePrefix, Collection> mockPlugins, Function clientWrapper) { super(clusterSeed); @@ -239,39 +249,47 @@ public final class InternalTestCluster extends TestCluster { Random random = new Random(clusterSeed); - this.numSharedAllRolesNodes = RandomInts.randomIntBetween(random, minNumDataNodes, maxNumDataNodes); - assert this.numSharedAllRolesNodes >= 0; + boolean useDedicatedMasterNodes = randomlyAddDedicatedMasters ? random.nextBoolean() : false; - //for now all shared data nodes are also master eligible - if (numSharedAllRolesNodes == 0) { - this.numShareCoordOnlyNodes = 0; + this.numSharedDataNodes = RandomInts.randomIntBetween(random, minNumDataNodes, maxNumDataNodes); + assert this.numSharedDataNodes >= 0; + + if (numSharedDataNodes == 0) { + this.numSharedCoordOnlyNodes = 0; + this.numSharedDedicatedMasterNodes = 0; } else { - if (numClientNodes < 0) { - this.numShareCoordOnlyNodes = RandomInts.randomIntBetween(random, DEFAULT_MIN_NUM_CLIENT_NODES, DEFAULT_MAX_NUM_CLIENT_NODES); + if (useDedicatedMasterNodes) { + if (random.nextBoolean()) { + // use a dedicated master, but only low number to reduce overhead to tests + this.numSharedDedicatedMasterNodes = DEFAULT_LOW_NUM_MASTER_NODES; + } else { + this.numSharedDedicatedMasterNodes = DEFAULT_HIGH_NUM_MASTER_NODES; + } } else { - this.numShareCoordOnlyNodes = numClientNodes; + this.numSharedDedicatedMasterNodes = 0; + } + if (numClientNodes < 0) { + this.numSharedCoordOnlyNodes = RandomInts.randomIntBetween(random, DEFAULT_MIN_NUM_CLIENT_NODES, DEFAULT_MAX_NUM_CLIENT_NODES); + } else { + this.numSharedCoordOnlyNodes = numClientNodes; } } - assert this.numShareCoordOnlyNodes >= 0; + assert this.numSharedCoordOnlyNodes >= 0; this.nodePrefix = nodePrefix; assert nodePrefix != null; this.mockPlugins = mockPlugins; - /* - * TODO - * - we might want start some master only nodes? - * - we could add a flag that returns a client to the master all the time? - * - we could add a flag that never returns a client to the master - * - along those lines use a dedicated node that is master eligible and let all other nodes be only data nodes - */ - sharedNodesSeeds = new long[numSharedAllRolesNodes + numShareCoordOnlyNodes]; + sharedNodesSeeds = new long[numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes]; for (int i = 0; i < sharedNodesSeeds.length; i++) { sharedNodesSeeds[i] = random.nextLong(); } - logger.info("Setup InternalTestCluster [{}] with seed [{}] using [{}] data nodes and [{}] client nodes", clusterName, SeedUtils.formatSeed(clusterSeed), numSharedAllRolesNodes, numShareCoordOnlyNodes); + logger.info("Setup InternalTestCluster [{}] with seed [{}] using [{}] dedicated masters, " + + "[{}] (data) nodes and [{}] coord only nodes", + clusterName, SeedUtils.formatSeed(clusterSeed), + numSharedDedicatedMasterNodes, numSharedDataNodes, numSharedCoordOnlyNodes); this.nodeConfigurationSource = nodeConfigurationSource; Builder builder = Settings.builder(); if (random.nextInt(5) == 0) { // sometimes set this @@ -348,7 +366,16 @@ public final class InternalTestCluster extends TestCluster { private Settings getSettings(int nodeOrdinal, long nodeSeed, Settings others) { Builder builder = Settings.builder().put(defaultSettings) - .put(getRandomNodeSettings(nodeSeed)); + .put(getRandomNodeSettings(nodeSeed)); + Settings interimSettings = builder.build(); + final String dataSuffix = getRoleSuffix(interimSettings); + if (dataSuffix.isEmpty() == false) { + // to make sure that a master node will not pick up on the data folder of a data only node + // once restarted we append the role suffix to each path. + String[] dataPath = Environment.PATH_DATA_SETTING.get(interimSettings).stream() + .map(path -> path + dataSuffix).toArray(String[]::new); + builder.putArray(Environment.PATH_DATA_SETTING.getKey(), dataPath); + } Settings settings = nodeConfigurationSource.nodeSettings(nodeOrdinal); if (settings != null) { if (settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey()) != null) { @@ -506,7 +533,11 @@ public final class InternalTestCluster extends TestCluster { int size = numDataNodes(); for (int i = size; i < n; i++) { logger.info("increasing cluster size from {} to {}", size, n); - asyncs.add(startNodeAsync()); + if (numSharedDedicatedMasterNodes > 0) { + asyncs.add(startDataOnlyNodeAsync()); + } else { + asyncs.add(startNodeAsync()); + } } } try { @@ -535,10 +566,10 @@ public final class InternalTestCluster extends TestCluster { } // prevent killing the master if possible and client nodes final Stream collection = - n == 0 ? nodes.values().stream() : nodes.values().stream().filter(new DataNodePredicate().and(new MasterNodePredicate(getMasterName()).negate())); + n == 0 ? nodes.values().stream() : nodes.values().stream().filter(new DataNodePredicate().and(new MasterNodePredicate(getMasterName()).negate())); final Iterator values = collection.iterator(); - logger.info("changing cluster size from {} to {}, {} data nodes", size(), n + numShareCoordOnlyNodes, n); + logger.info("changing cluster size from {} to {}, {} data nodes", size(), n + numSharedCoordOnlyNodes, n); Set nodesToRemove = new HashSet<>(); int numNodesAndClients = 0; while (values.hasNext() && numNodesAndClients++ < size - n) { @@ -557,33 +588,59 @@ public final class InternalTestCluster extends TestCluster { private NodeAndClient buildNode(Settings settings, Version version) { int ord = nextNodeId.getAndIncrement(); - return buildNode(ord, random.nextLong(), settings, version); + return buildNode(ord, random.nextLong(), settings, version, false); } private NodeAndClient buildNode() { int ord = nextNodeId.getAndIncrement(); - return buildNode(ord, random.nextLong(), null, Version.CURRENT); + return buildNode(ord, random.nextLong(), null, Version.CURRENT, false); } - private NodeAndClient buildNode(int nodeId, long seed, Settings settings, Version version) { + private NodeAndClient buildNode(int nodeId, long seed, Settings settings, Version version, boolean reuseExisting) { assert Thread.holdsLock(this); ensureOpen(); settings = getSettings(nodeId, seed, settings); Collection> plugins = getPlugins(); - String name = buildNodeName(nodeId); - assert !nodes.containsKey(name); + String name = buildNodeName(nodeId, settings); + if (reuseExisting && nodes.containsKey(name)) { + return nodes.get(name); + } else { + assert reuseExisting == true || nodes.containsKey(name) == false : + "node name [" + name + "] already exists but not allowed to use it"; + } Settings finalSettings = Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) // allow overriding path.home - .put(settings) - .put("node.name", name) - .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), seed) - .build(); + .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) // allow overriding path.home + .put(settings) + .put("node.name", name) + .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), seed) + .build(); MockNode node = new MockNode(finalSettings, version, plugins); return new NodeAndClient(name, node); } - private String buildNodeName(int id) { - return nodePrefix + id; + private String buildNodeName(int id, Settings settings) { + String prefix = nodePrefix; + prefix = prefix + getRoleSuffix(settings); + return prefix + id; + } + + /** + * returns a suffix string based on the node role. If no explicit role is defined, the suffix will be empty + */ + private String getRoleSuffix(Settings settings) { + String suffix = ""; + if (Node.NODE_MASTER_SETTING.exists(settings) && Node.NODE_MASTER_SETTING.get(settings)) { + suffix = suffix + DiscoveryNode.Role.MASTER.getAbbreviation(); + } + if (Node.NODE_DATA_SETTING.exists(settings) && Node.NODE_DATA_SETTING.get(settings)) { + suffix = suffix + DiscoveryNode.Role.DATA.getAbbreviation(); + } + if (Node.NODE_MASTER_SETTING.exists(settings) && Node.NODE_MASTER_SETTING.get(settings) == false && + Node.NODE_DATA_SETTING.exists(settings) && Node.NODE_DATA_SETTING.get(settings) == false + ) { + suffix = suffix + "c"; + } + return suffix; } /** @@ -873,14 +930,14 @@ public final class InternalTestCluster extends TestCluster { TransportAddress addr = node.injector().getInstance(TransportService.class).boundAddress().publishAddress(); Settings nodeSettings = node.settings(); Builder builder = Settings.builder() - .put("client.transport.nodes_sampler_interval", "1s") - .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) - .put("node.name", TRANSPORT_CLIENT_PREFIX + node.settings().get("node.name")) - .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName).put("client.transport.sniff", sniff) - .put(Node.NODE_MODE_SETTING.getKey(), Node.NODE_MODE_SETTING.exists(nodeSettings) ? Node.NODE_MODE_SETTING.get(nodeSettings) : nodeMode) - .put("logger.prefix", nodeSettings.get("logger.prefix", "")) - .put("logger.level", nodeSettings.get("logger.level", "INFO")) - .put(settings); + .put("client.transport.nodes_sampler_interval", "1s") + .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) + .put("node.name", TRANSPORT_CLIENT_PREFIX + node.settings().get("node.name")) + .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName).put("client.transport.sniff", sniff) + .put(Node.NODE_MODE_SETTING.getKey(), Node.NODE_MODE_SETTING.exists(nodeSettings) ? Node.NODE_MODE_SETTING.get(nodeSettings) : nodeMode) + .put("logger.prefix", nodeSettings.get("logger.prefix", "")) + .put("logger.level", nodeSettings.get("logger.level", "INFO")) + .put(settings); if (Node.NODE_LOCAL_SETTING.exists(nodeSettings)) { builder.put(Node.NODE_LOCAL_SETTING.getKey(), Node.NODE_LOCAL_SETTING.get(nodeSettings)); @@ -924,39 +981,35 @@ public final class InternalTestCluster extends TestCluster { Set sharedNodes = new HashSet<>(); - assert sharedNodesSeeds.length == numSharedAllRolesNodes + numShareCoordOnlyNodes; - boolean changed = false; - for (int i = 0; i < numSharedAllRolesNodes; i++) { - String buildNodeName = buildNodeName(i); - NodeAndClient nodeAndClient = nodes.get(buildNodeName); - if (nodeAndClient == null) { - changed = true; - nodeAndClient = buildNode(i, sharedNodesSeeds[i], null, Version.CURRENT); - nodeAndClient.node.start(); - logger.info("Start Shared Node [{}] not shared", nodeAndClient.name); - } + assert sharedNodesSeeds.length == numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; + for (int i = 0; i < numSharedDedicatedMasterNodes; i++) { + final Settings.Builder settings = Settings.builder(); + settings.put(Node.NODE_MASTER_SETTING.getKey(), true).build(); + settings.put(Node.NODE_DATA_SETTING.getKey(), false).build(); + NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), Version.CURRENT, true); + nodeAndClient.node().start(); sharedNodes.add(nodeAndClient); } - for (int i = numSharedAllRolesNodes; i < numSharedAllRolesNodes + numShareCoordOnlyNodes; i++) { - String buildNodeName = buildNodeName(i); - NodeAndClient nodeAndClient = nodes.get(buildNodeName); - if (nodeAndClient == null) { - changed = true; - Builder clientSettingsBuilder = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false) - .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false); - nodeAndClient = buildNode(i, sharedNodesSeeds[i], clientSettingsBuilder.build(), Version.CURRENT); - nodeAndClient.node.start(); - logger.info("Start Shared Node [{}] not shared", nodeAndClient.name); + for (int i = numSharedDedicatedMasterNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes; i++) { + final Settings.Builder settings = Settings.builder(); + if (numSharedDedicatedMasterNodes > 0) { + // if we don't have dedicated master nodes, keep things default + settings.put(Node.NODE_MASTER_SETTING.getKey(), false).build(); + settings.put(Node.NODE_DATA_SETTING.getKey(), true).build(); } + NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), Version.CURRENT, true); + nodeAndClient.node().start(); sharedNodes.add(nodeAndClient); } - if (!changed && sharedNodes.size() == nodes.size()) { - logger.debug("Cluster is consistent - moving out - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), sharedNodesSeeds.length); - if (size() > 0) { - client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(sharedNodesSeeds.length)).get(); - } - return; // we are consistent - return + for (int i = numSharedDedicatedMasterNodes + numSharedDataNodes; + i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) { + final Builder settings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false) + .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false); + NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), Version.CURRENT, true); + nodeAndClient.node().start(); + sharedNodes.add(nodeAndClient); } + for (NodeAndClient nodeAndClient : sharedNodes) { nodes.remove(nodeAndClient.name); } @@ -1092,6 +1145,14 @@ public final class InternalTestCluster extends TestCluster { return getInstances(clazz, new DataNodePredicate()); } + /** + * Returns an Iterable to all instances for the given class >T< across all data and master nodes + * in the cluster. + */ + public synchronized Iterable getDataOrMasterNodeInstances(Class clazz) { + return getInstances(clazz, new DataOrMasterNodePredicate()); + } + private synchronized Iterable getInstances(Class clazz, Predicate predicate) { Iterable filteredNodes = nodes.values().stream().filter(predicate)::iterator; List instances = new ArrayList<>(); @@ -1296,7 +1357,19 @@ public final class InternalTestCluster extends TestCluster { } nodeAndClient.closeNode(); } - for (NodeAndClient nodeAndClient : nodes.values()) { + + // starting master nodes first, for now so restart will be quick. If we'll start + // the data nodes first, they will wait for 30s for a master + List discoveryNodes = new ArrayList<>(); + for (ClusterService clusterService : getInstances(ClusterService.class)) { + discoveryNodes.add(clusterService.localNode()); + } + + discoveryNodes.sort((n1, n2) -> Boolean.compare(n1.isMasterNode() == false, n2.isMasterNode() == false)); + + + for (DiscoveryNode node : discoveryNodes) { + NodeAndClient nodeAndClient = nodes.get(node.getName()); logger.info("Starting node [{}] ", nodeAndClient.name); if (activeDisruptionScheme != null) { activeDisruptionScheme.removeFromNode(nodeAndClient.name, this); @@ -1375,11 +1448,11 @@ public final class InternalTestCluster extends TestCluster { private synchronized Set nRandomDataNodes(int numNodes) { assert size() >= numNodes; Map dataNodes = - nodes - .entrySet() - .stream() - .filter(new EntryNodePredicate(new DataNodePredicate())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + nodes + .entrySet() + .stream() + .filter(new EntryNodePredicate(new DataNodePredicate())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); final HashSet set = new HashSet<>(); final Iterator iterator = dataNodes.keySet().iterator(); for (int i = 0; i < numNodes; i++) { @@ -1632,10 +1705,10 @@ public final class InternalTestCluster extends TestCluster { private synchronized Collection filterNodes(Map map, Predicate predicate) { return map - .values() - .stream() - .filter(predicate) - .collect(Collectors.toCollection(ArrayList::new)); + .values() + .stream() + .filter(predicate) + .collect(Collectors.toCollection(ArrayList::new)); } private static final class DataNodePredicate implements Predicate { @@ -1649,7 +1722,7 @@ public final class InternalTestCluster extends TestCluster { @Override public boolean test(NodeAndClient nodeAndClient) { return DiscoveryNode.isDataNode(nodeAndClient.node.settings()) || - DiscoveryNode.isMasterNode(nodeAndClient.node.settings()); + DiscoveryNode.isMasterNode(nodeAndClient.node.settings()); } } @@ -1887,6 +1960,7 @@ public final class InternalTestCluster extends TestCluster { /** * Simple interface that allows to wait for an async operation to finish + * * @param the result of the async execution */ public interface Async { diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index 2c7e35fa4ea..ae774220ccf 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -49,6 +49,7 @@ public class InternalTestClusterTests extends ESTestCase { public void testInitializiationIsConsistent() { long clusterSeed = randomLong(); + boolean masterNodes = randomBoolean(); int minNumDataNodes = randomIntBetween(0, 9); int maxNumDataNodes = randomIntBetween(minNumDataNodes, 10); String clusterName = randomRealisticUnicodeOfCodepointLengthBetween(1, 10); @@ -58,8 +59,12 @@ public class InternalTestClusterTests extends ESTestCase { String nodePrefix = randomRealisticUnicodeOfCodepointLengthBetween(1, 10); Path baseDir = createTempDir(); - InternalTestCluster cluster0 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, nodeConfigurationSource, numClientNodes, enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity()); - InternalTestCluster cluster1 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, nodeConfigurationSource, numClientNodes, enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity()); + InternalTestCluster cluster0 = new InternalTestCluster("local", clusterSeed, baseDir, masterNodes, + minNumDataNodes, maxNumDataNodes, clusterName, nodeConfigurationSource, numClientNodes, + enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity()); + InternalTestCluster cluster1 = new InternalTestCluster("local", clusterSeed, baseDir, masterNodes, + minNumDataNodes, maxNumDataNodes, clusterName, nodeConfigurationSource, numClientNodes, + enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity()); // TODO: this is not ideal - we should have a way to make sure ports are initialized in the same way assertClusters(cluster0, cluster1, false); @@ -101,6 +106,7 @@ public class InternalTestClusterTests extends ESTestCase { public void testBeforeTest() throws Exception { long clusterSeed = randomLong(); + boolean masterNodes = randomBoolean(); int minNumDataNodes = randomIntBetween(0, 3); int maxNumDataNodes = randomIntBetween(minNumDataNodes, 4); final String clusterName1 = "shared1";//clusterName("shared1", clusterSeed); @@ -115,8 +121,12 @@ public class InternalTestClusterTests extends ESTestCase { String nodePrefix = "foobar"; Path baseDir = createTempDir(); - InternalTestCluster cluster0 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName1, nodeConfigurationSource, numClientNodes, enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity()); - InternalTestCluster cluster1 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName2, nodeConfigurationSource, numClientNodes, enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity()); + InternalTestCluster cluster0 = new InternalTestCluster("local", clusterSeed, baseDir, masterNodes, + minNumDataNodes, maxNumDataNodes, clusterName1, nodeConfigurationSource, numClientNodes, + enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity()); + InternalTestCluster cluster1 = new InternalTestCluster("local", clusterSeed, baseDir, masterNodes, + minNumDataNodes, maxNumDataNodes, clusterName2, nodeConfigurationSource, numClientNodes, + enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity()); assertClusters(cluster0, cluster1, false); long seed = randomLong();