From 7924115b907c55fa689f6e37d3d58f2098f5dc05 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 17 Sep 2012 16:00:55 +0200 Subject: [PATCH] Disable allocation: New indices allocation not to be disabled by default When setting cluster.routing.allocation.disable_allocation, it causes new indices primary shards to not be allocated. By default, new indices created should allow to, at the very least, allocate primary shards so they become operations. A new setting, cluster.routing.allocation.disable_new_allocation, allows to also disable "new" allocations. closes #2258. --- .../metadata/MetaDataCreateIndexService.java | 2 +- .../metadata/MetaDataStateIndexService.java | 2 +- .../cluster/routing/IndexRoutingTable.java | 46 ++++++++++++------- .../routing/IndexShardRoutingTable.java | 28 +++++------ .../cluster/routing/RoutingTable.java | 21 ++++++--- .../decider/DisableAllocationDecider.java | 14 ++++++ .../elasticsearch/gateway/GatewayService.java | 2 +- .../gateway/local/LocalGatewayAllocator.java | 2 +- .../meta/LocalAllocateDangledIndices.java | 2 +- .../cluster/IndicesClusterStateService.java | 2 +- .../allocation/ClusterRerouteTests.java | 2 + .../allocation/AllocatePostApiFlagTests.java | 9 ++-- .../allocation/AllocationCommandsTests.java | 9 ++-- .../allocation/AwarenessAllocationTests.java | 25 +++++----- .../ClusterRebalanceRoutingTests.java | 25 +++++----- .../ConcurrentRebalanceRoutingTests.java | 3 +- .../allocation/DeadNodesAllocationTests.java | 7 ++- .../allocation/DisableAllocationTests.java | 8 ++-- ...ReplicaAsPrimaryDuringRelocationTests.java | 3 +- .../allocation/FailedNodeRoutingTests.java | 10 ++-- .../allocation/FailedShardsRoutingTests.java | 11 ++--- .../allocation/FilterRoutingTests.java | 7 ++- .../PrimaryElectionRoutingTests.java | 5 +- ...yNotRelocatedWhileBeingRecoveredTests.java | 5 +- .../allocation/RebalanceAfterActiveTests.java | 5 +- .../ReplicaAllocatedAfterPrimaryTests.java | 5 +- .../allocation/SameShardRoutingTests.java | 3 +- .../allocation/ShardVersioningTests.java | 7 ++- .../ShardsLimitAllocationTests.java | 7 ++- .../SingleShardNoReplicasRoutingTests.java | 10 ++-- .../SingleShardOneReplicaRoutingTests.java | 5 +- .../TenShardsOneReplicaRoutingTests.java | 5 +- .../allocation/ThrottlingAllocationTests.java | 6 +-- .../UpdateNumberOfReplicasTests.java | 5 +- .../ClusterSerializationTests.java | 5 +- .../structure/RoutingIteratorTests.java | 15 +++--- 36 files changed, 170 insertions(+), 158 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 7ba52fdfd3d..80ac07f3ef1 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -319,7 +319,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { if (request.state == State.OPEN) { RoutingTable.Builder routingTableBuilder = RoutingTable.builder().routingTable(updatedState.routingTable()) - .add(updatedState.metaData().index(request.index), true); + .addAsNew(updatedState.metaData().index(request.index)); RoutingAllocation.Result routingResult = allocationService.reroute(newClusterStateBuilder().state(updatedState).routingTable(routingTableBuilder).build()); updatedState = newClusterStateBuilder().state(updatedState).routingResult(routingResult).build(); } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataStateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataStateIndexService.java index 438033e6081..eb6c1e405a5 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataStateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataStateIndexService.java @@ -127,7 +127,7 @@ public class MetaDataStateIndexService extends AbstractComponent { ClusterState updatedState = ClusterState.builder().state(currentState).metaData(mdBuilder).blocks(blocks).build(); RoutingTable.Builder rtBuilder = RoutingTable.builder().routingTable(updatedState.routingTable()) - .add(updatedState.metaData().index(request.index), false); + .addAsRecovery(updatedState.metaData().index(request.index)); RoutingAllocation.Result routingResult = allocationService.reroute(newClusterStateBuilder().state(updatedState).routingTable(rtBuilder).build()); diff --git a/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index 51bdaff0fc9..082781670a8 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -23,10 +23,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import com.google.common.collect.UnmodifiableIterator; +import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.shard.ShardId; import java.io.IOException; import java.util.*; @@ -270,18 +272,30 @@ public class IndexRoutingTable implements Iterable { /** * Initializes a new empty index, as if it was created from an API. */ - public Builder initializeEmpty(IndexMetaData indexMetaData) { + public Builder initializeAsNew(IndexMetaData indexMetaData) { return initializeEmpty(indexMetaData, true); } + /** + * Initializes a new empty index, as if it was created from an API. + */ + public Builder initializeAsRecovery(IndexMetaData indexMetaData) { + return initializeEmpty(indexMetaData, false); + } + /** * Initializes a new empty index, with an option to control if its from an API or not. */ - public Builder initializeEmpty(IndexMetaData indexMetaData, boolean fromApi) { + private Builder initializeEmpty(IndexMetaData indexMetaData, boolean asNew) { + if (!shards.isEmpty()) { + throw new ElasticSearchIllegalStateException("trying to initialize an index with fresh shards, but already has shards created"); + } for (int shardId = 0; shardId < indexMetaData.numberOfShards(); shardId++) { + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.index(), shardId), asNew ? false : true); for (int i = 0; i <= indexMetaData.numberOfReplicas(); i++) { - addShard(shardId, null, i == 0, ShardRoutingState.UNASSIGNED, 0, fromApi); + indexShardRoutingBuilder.addShard(new ImmutableShardRouting(index, shardId, null, i == 0, ShardRoutingState.UNASSIGNED, 0)); } + shards.put(shardId, indexShardRoutingBuilder.build()); } return this; } @@ -289,7 +303,10 @@ public class IndexRoutingTable implements Iterable { public Builder addReplica() { for (int shardId : shards.keySet()) { // version 0, will get updated when reroute will happen - addShard(shardId, null, false, ShardRoutingState.UNASSIGNED, 0, false); + ImmutableShardRouting shard = new ImmutableShardRouting(index, shardId, null, false, ShardRoutingState.UNASSIGNED, 0); + shards.put(shardId, + new IndexShardRoutingTable.Builder(shards.get(shard.id())).addShard(shard).build() + ); } return this; } @@ -302,7 +319,7 @@ public class IndexRoutingTable implements Iterable { return this; } // re-add all the current ones - IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(indexShard.shardId(), indexShard.allocatedPostApi()); + IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(indexShard.shardId(), indexShard.primaryAllocatedPostApi()); for (ShardRouting shardRouting : indexShard) { builder.addShard(new ImmutableShardRouting(shardRouting)); } @@ -334,21 +351,16 @@ public class IndexRoutingTable implements Iterable { return this; } - public Builder addShard(ShardRouting shard, boolean fromApi) { - return internalAddShard(new ImmutableShardRouting(shard), fromApi); - } - - private Builder addShard(int shardId, String nodeId, boolean primary, ShardRoutingState state, long version, boolean fromApi) { - ImmutableShardRouting shard = new ImmutableShardRouting(index, shardId, nodeId, primary, state, version); - return internalAddShard(shard, fromApi); - } - - private Builder internalAddShard(ImmutableShardRouting shard, boolean fromApi) { + /** + * Adds a new shard routing (makes a copy of it), with reference data used from the index shard routing table + * if it needs to be created. + */ + public Builder addShard(IndexShardRoutingTable refData, ShardRouting shard) { IndexShardRoutingTable indexShard = shards.get(shard.id()); if (indexShard == null) { - indexShard = new IndexShardRoutingTable.Builder(shard.shardId(), fromApi ? false : true).addShard(shard).build(); + indexShard = new IndexShardRoutingTable.Builder(refData.shardId(), refData.primaryAllocatedPostApi()).addShard(new ImmutableShardRouting(shard)).build(); } else { - indexShard = new IndexShardRoutingTable.Builder(indexShard).addShard(shard).build(); + indexShard = new IndexShardRoutingTable.Builder(indexShard).addShard(new ImmutableShardRouting(shard)).build(); } shards.put(indexShard.shardId().id(), indexShard); return this; diff --git a/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java index 02961bda7cb..0c7807d992f 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java @@ -51,12 +51,12 @@ public class IndexShardRoutingTable implements Iterable { final AtomicInteger counter; - final boolean allocatedPostApi; + final boolean primaryAllocatedPostApi; - IndexShardRoutingTable(ShardId shardId, ImmutableList shards, boolean allocatedPostApi) { + IndexShardRoutingTable(ShardId shardId, ImmutableList shards, boolean primaryAllocatedPostApi) { this.shardId = shardId; this.shards = shards; - this.allocatedPostApi = allocatedPostApi; + this.primaryAllocatedPostApi = primaryAllocatedPostApi; this.counter = new AtomicInteger(ThreadLocalRandom.current().nextInt(shards.size())); ShardRouting primary = null; @@ -120,15 +120,15 @@ public class IndexShardRoutingTable implements Iterable { shardRoutings.add(new ImmutableShardRouting(shards.get(i), highestVersion)); } } - return new IndexShardRoutingTable(shardId, ImmutableList.copyOf(shardRoutings), allocatedPostApi); + return new IndexShardRoutingTable(shardId, ImmutableList.copyOf(shardRoutings), primaryAllocatedPostApi); } /** * Has this shard group primary shard been allocated post API creation. Will be set to * true if it was created because of recovery action. */ - public boolean allocatedPostApi() { - return allocatedPostApi; + public boolean primaryAllocatedPostApi() { + return primaryAllocatedPostApi; } public ShardId shardId() { @@ -404,18 +404,18 @@ public class IndexShardRoutingTable implements Iterable { private final List shards; - private boolean allocatedPostApi; + private boolean primaryAllocatedPostApi; public Builder(IndexShardRoutingTable indexShard) { this.shardId = indexShard.shardId; this.shards = newArrayList(indexShard.shards); - this.allocatedPostApi = indexShard.allocatedPostApi(); + this.primaryAllocatedPostApi = indexShard.primaryAllocatedPostApi(); } - public Builder(ShardId shardId, boolean allocatedPostApi) { + public Builder(ShardId shardId, boolean primaryAllocatedPostApi) { this.shardId = shardId; this.shards = newArrayList(); - this.allocatedPostApi = allocatedPostApi; + this.primaryAllocatedPostApi = primaryAllocatedPostApi; } public Builder addShard(ImmutableShardRouting shardEntry) { @@ -438,14 +438,14 @@ public class IndexShardRoutingTable implements Iterable { public IndexShardRoutingTable build() { // we can automatically set allocatedPostApi to true if the primary is active - if (!allocatedPostApi) { + if (!primaryAllocatedPostApi) { for (ShardRouting shardRouting : shards) { if (shardRouting.primary() && shardRouting.active()) { - allocatedPostApi = true; + primaryAllocatedPostApi = true; } } } - return new IndexShardRoutingTable(shardId, ImmutableList.copyOf(shards), allocatedPostApi); + return new IndexShardRoutingTable(shardId, ImmutableList.copyOf(shards), primaryAllocatedPostApi); } public static IndexShardRoutingTable readFrom(StreamInput in) throws IOException { @@ -474,7 +474,7 @@ public class IndexShardRoutingTable implements Iterable { public static void writeToThin(IndexShardRoutingTable indexShard, StreamOutput out) throws IOException { out.writeVInt(indexShard.shardId.id()); - out.writeBoolean(indexShard.allocatedPostApi()); + out.writeBoolean(indexShard.primaryAllocatedPostApi()); out.writeVInt(indexShard.shards.size()); for (ShardRouting entry : indexShard) { diff --git a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index 98292845988..ef330a93e21 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -285,8 +285,8 @@ public class RoutingTable implements Iterable { indexRoutingTableBuilders.put(index, indexBuilder); } - boolean allocatedPostApi = routingNodes.routingTable().index(shardRoutingEntry.index()).shard(shardRoutingEntry.id()).allocatedPostApi(); - indexBuilder.addShard(new ImmutableShardRouting(shardRoutingEntry), !allocatedPostApi); + IndexShardRoutingTable refData = routingNodes.routingTable().index(shardRoutingEntry.index()).shard(shardRoutingEntry.id()); + indexBuilder.addShard(refData, shardRoutingEntry); } } for (MutableShardRouting shardRoutingEntry : Iterables.concat(routingNodes.unassigned(), routingNodes.ignoredUnassigned())) { @@ -296,8 +296,8 @@ public class RoutingTable implements Iterable { indexBuilder = new IndexRoutingTable.Builder(index); indexRoutingTableBuilders.put(index, indexBuilder); } - boolean allocatedPostApi = routingNodes.routingTable().index(shardRoutingEntry.index()).shard(shardRoutingEntry.id()).allocatedPostApi(); - indexBuilder.addShard(new ImmutableShardRouting(shardRoutingEntry), !allocatedPostApi); + IndexShardRoutingTable refData = routingNodes.routingTable().index(shardRoutingEntry.index()).shard(shardRoutingEntry.id()); + indexBuilder.addShard(refData, shardRoutingEntry); } for (IndexRoutingTable.Builder indexBuilder : indexRoutingTableBuilders.values()) { add(indexBuilder); @@ -341,10 +341,19 @@ public class RoutingTable implements Iterable { return this; } - public Builder add(IndexMetaData indexMetaData, boolean fromApi) { + public Builder addAsNew(IndexMetaData indexMetaData) { if (indexMetaData.state() == IndexMetaData.State.OPEN) { IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.index()) - .initializeEmpty(indexMetaData, fromApi); + .initializeAsNew(indexMetaData); + add(indexRoutingBuilder); + } + return this; + } + + public Builder addAsRecovery(IndexMetaData indexMetaData) { + if (indexMetaData.state() == IndexMetaData.State.OPEN) { + IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.index()) + .initializeAsRecovery(indexMetaData); add(indexRoutingBuilder); } return this; diff --git a/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DisableAllocationDecider.java b/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DisableAllocationDecider.java index d442c51f2f2..1d923d884c6 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DisableAllocationDecider.java +++ b/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DisableAllocationDecider.java @@ -33,6 +33,7 @@ public class DisableAllocationDecider extends AllocationDecider { static { MetaData.addDynamicSettings( + "cluster.routing.allocation.disable_new_allocation", "cluster.routing.allocation.disable_allocation", "cluster.routing.allocation.disable_replica_allocation" ); @@ -41,6 +42,12 @@ public class DisableAllocationDecider extends AllocationDecider { class ApplySettings implements NodeSettingsService.Listener { @Override public void onRefreshSettings(Settings settings) { + boolean disableNewAllocation = settings.getAsBoolean("cluster.routing.allocation.disable_new_allocation", DisableAllocationDecider.this.disableNewAllocation); + if (disableNewAllocation != DisableAllocationDecider.this.disableNewAllocation) { + logger.info("updating [cluster.routing.allocation.disable_new_allocation] from [{}] to [{}]", DisableAllocationDecider.this.disableNewAllocation, disableNewAllocation); + DisableAllocationDecider.this.disableNewAllocation = disableNewAllocation; + } + boolean disableAllocation = settings.getAsBoolean("cluster.routing.allocation.disable_allocation", DisableAllocationDecider.this.disableAllocation); if (disableAllocation != DisableAllocationDecider.this.disableAllocation) { logger.info("updating [cluster.routing.allocation.disable_allocation] from [{}] to [{}]", DisableAllocationDecider.this.disableAllocation, disableAllocation); @@ -55,12 +62,14 @@ public class DisableAllocationDecider extends AllocationDecider { } } + private volatile boolean disableNewAllocation; private volatile boolean disableAllocation; private volatile boolean disableReplicaAllocation; @Inject public DisableAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) { super(settings); + this.disableNewAllocation = settings.getAsBoolean("cluster.routing.allocation.disable_new_allocation", false); this.disableAllocation = settings.getAsBoolean("cluster.routing.allocation.disable_allocation", false); this.disableReplicaAllocation = settings.getAsBoolean("cluster.routing.allocation.disable_replica_allocation", false); @@ -69,6 +78,11 @@ public class DisableAllocationDecider extends AllocationDecider { @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + if (shardRouting.primary() && !allocation.routingNodes().routingTable().index(shardRouting.index()).shard(shardRouting.id()).primaryAllocatedPostApi()) { + // if its primary, and it hasn't been allocated post API (meaning its a "fresh newly created shard"), only disable allocation + // on a special disable allocation flag + return allocation.ignoreDisable() ? Decision.YES : disableNewAllocation ? Decision.NO : Decision.YES; + } if (disableAllocation) { return allocation.ignoreDisable() ? Decision.YES : Decision.NO; } diff --git a/src/main/java/org/elasticsearch/gateway/GatewayService.java b/src/main/java/org/elasticsearch/gateway/GatewayService.java index cf16cc1553c..edfe64f869d 100644 --- a/src/main/java/org/elasticsearch/gateway/GatewayService.java +++ b/src/main/java/org/elasticsearch/gateway/GatewayService.java @@ -269,7 +269,7 @@ public class GatewayService extends AbstractLifecycleComponent i // initialize all index routing tables as empty RoutingTable.Builder routingTableBuilder = RoutingTable.builder().routingTable(updatedState.routingTable()); for (IndexMetaData indexMetaData : updatedState.metaData().indices().values()) { - routingTableBuilder.add(indexMetaData, false /* not from API */); + routingTableBuilder.addAsRecovery(indexMetaData); } // start with 0 based versions for routing table routingTableBuilder.version(0); diff --git a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java index 7422b87b095..20e67f052d2 100644 --- a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java +++ b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java @@ -118,7 +118,7 @@ public class LocalGatewayAllocator extends AbstractComponent implements GatewayA } // this is an API allocation, ignore since we know there is no data... - if (!routingNodes.routingTable().index(shard.index()).shard(shard.id()).allocatedPostApi()) { + if (!routingNodes.routingTable().index(shard.index()).shard(shard.id()).primaryAllocatedPostApi()) { continue; } diff --git a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalAllocateDangledIndices.java b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalAllocateDangledIndices.java index 79b6d7a416b..754d5b7dafd 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalAllocateDangledIndices.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalAllocateDangledIndices.java @@ -140,7 +140,7 @@ public class LocalAllocateDangledIndices extends AbstractComponent { importNeeded = true; metaData.put(indexMetaData, false); blocks.addBlocks(indexMetaData); - routingTableBuilder.add(indexMetaData, false); + routingTableBuilder.addAsRecovery(indexMetaData); sb.append("[").append(indexMetaData.index()).append("/").append(indexMetaData.state()).append("]"); } if (!importNeeded) { diff --git a/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index f8c066e1a3c..88454597155 100644 --- a/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -616,7 +616,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent