diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index 6af6e6696e0..a1549c5e217 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus; import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision; import org.elasticsearch.cluster.routing.allocation.AllocationDecision; @@ -115,7 +116,7 @@ public class BalancedShardsAllocator implements ShardsAllocator { @Override public void allocate(RoutingAllocation allocation) { if (allocation.routingNodes().size() == 0) { - /* with no nodes this is pointless */ + failAllocationOfNewPrimaries(allocation); return; } final Balancer balancer = new Balancer(logger, allocation, weightFunction, threshold); @@ -141,6 +142,22 @@ public class BalancedShardsAllocator implements ShardsAllocator { return new ShardAllocationDecision(allocateUnassignedDecision, moveDecision); } + private void failAllocationOfNewPrimaries(RoutingAllocation allocation) { + RoutingNodes routingNodes = allocation.routingNodes(); + assert routingNodes.size() == 0 : routingNodes; + final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = routingNodes.unassigned().iterator(); + while (unassignedIterator.hasNext()) { + final ShardRouting shardRouting = unassignedIterator.next(); + final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo(); + if (shardRouting.primary() && unassignedInfo.getLastAllocationStatus() == AllocationStatus.NO_ATTEMPT) { + unassignedIterator.updateUnassigned(new UnassignedInfo(unassignedInfo.getReason(), unassignedInfo.getMessage(), + unassignedInfo.getFailure(), unassignedInfo.getNumFailedAllocations(), unassignedInfo.getUnassignedTimeInNanos(), + unassignedInfo.getUnassignedTimeInMillis(), unassignedInfo.isDelayed(), AllocationStatus.DECIDERS_NO), + shardRouting.recoverySource(), allocation.changes()); + } + } + } + /** * Returns the currently configured delta threshold */ diff --git a/server/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java index 82744aa86f9..1bd6439c2fc 100644 --- a/server/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java @@ -85,7 +85,7 @@ public class SimpleDataNodesIT extends ESIntegTestCase { final ClusterHealthResponse healthResponse1 = client().admin().cluster().prepareHealth() .setWaitForEvents(Priority.LANGUID).execute().actionGet(); assertThat(healthResponse1.isTimedOut(), equalTo(false)); - assertThat(healthResponse1.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); // TODO should be RED, see #41073 + assertThat(healthResponse1.getStatus(), equalTo(ClusterHealthStatus.RED)); assertThat(healthResponse1.getActiveShards(), equalTo(0)); internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), true).build()); @@ -104,7 +104,7 @@ public class SimpleDataNodesIT extends ESIntegTestCase { final ClusterHealthResponse healthResponse1 = client().admin().cluster().prepareHealth() .setWaitForEvents(Priority.LANGUID).execute().actionGet(); assertThat(healthResponse1.isTimedOut(), equalTo(false)); - assertThat(healthResponse1.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); // TODO should be RED, see #41073 + assertThat(healthResponse1.getStatus(), equalTo(ClusterHealthStatus.RED)); assertThat(healthResponse1.getActiveShards(), equalTo(0)); internalCluster().startNode(); diff --git a/server/src/test/java/org/elasticsearch/cluster/health/ClusterHealthAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/health/ClusterHealthAllocationTests.java new file mode 100644 index 00000000000..52adcd503da --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/health/ClusterHealthAllocationTests.java @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.health; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ESAllocationTestCase; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.allocation.AllocationService; + +import java.util.Collections; + +public class ClusterHealthAllocationTests extends ESAllocationTestCase { + + public void testClusterHealth() { + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).build(); + if (randomBoolean()) { + clusterState = addNode(clusterState, "node_m", true); + } + assertEquals(ClusterHealthStatus.GREEN, getClusterHealthStatus(clusterState)); + + MetaData metaData = MetaData.builder() + .put(IndexMetaData.builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(2) + .numberOfReplicas(1)) + .build(); + RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build(); + clusterState = ClusterState.builder(clusterState).metaData(metaData).routingTable(routingTable).build(); + MockAllocationService allocation = createAllocationService(); + clusterState = applyStartedShardsUntilNoChange(clusterState, allocation); + assertEquals(0, clusterState.nodes().getDataNodes().size()); + assertEquals(ClusterHealthStatus.RED, getClusterHealthStatus(clusterState)); + + clusterState = addNode(clusterState, "node_d1", false); + assertEquals(1, clusterState.nodes().getDataNodes().size()); + clusterState = applyStartedShardsUntilNoChange(clusterState, allocation); + assertEquals(ClusterHealthStatus.YELLOW, getClusterHealthStatus(clusterState)); + + clusterState = addNode(clusterState, "node_d2", false); + clusterState = applyStartedShardsUntilNoChange(clusterState, allocation); + assertEquals(ClusterHealthStatus.GREEN, getClusterHealthStatus(clusterState)); + + clusterState = removeNode(clusterState, "node_d1", allocation); + assertEquals(ClusterHealthStatus.YELLOW, getClusterHealthStatus(clusterState)); + + clusterState = removeNode(clusterState, "node_d2", allocation); + assertEquals(ClusterHealthStatus.RED, getClusterHealthStatus(clusterState)); + + routingTable = RoutingTable.builder(routingTable).remove("test").build(); + metaData = MetaData.builder(clusterState.metaData()).remove("test").build(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).metaData(metaData).build(); + assertEquals(0, clusterState.nodes().getDataNodes().size()); + assertEquals(ClusterHealthStatus.GREEN, getClusterHealthStatus(clusterState)); + } + + private ClusterState addNode(ClusterState clusterState, String nodeName, boolean isMaster) { + DiscoveryNodes.Builder nodeBuilder = DiscoveryNodes.builder(clusterState.getNodes()); + nodeBuilder.add(newNode(nodeName, Collections.singleton(isMaster ? DiscoveryNodeRole.MASTER_ROLE : DiscoveryNodeRole.DATA_ROLE))); + return ClusterState.builder(clusterState).nodes(nodeBuilder).build(); + } + + private ClusterState removeNode(ClusterState clusterState, String nodeName, AllocationService allocationService) { + return allocationService.disassociateDeadNodes(ClusterState.builder(clusterState) + .nodes(DiscoveryNodes.builder(clusterState.getNodes()).remove(nodeName)).build(), true, "reroute"); + } + + private ClusterHealthStatus getClusterHealthStatus(ClusterState clusterState) { + return new ClusterStateHealth(clusterState).getStatus(); + } + +}