From cd7096dfc2b098e68cbf5615ca7d9bf2ceb46165 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 21 Jul 2015 13:27:52 +0200 Subject: [PATCH] Replace primaryPostAllocated flag and use UnassignedInfo There is no need to maintain additional state as to if a primary was allocated post api creation on the index routing table, we hold all this information already in the UnassignedInfo class. closes #12374 --- .../metadata/MetaDataIndexStateService.java | 7 +- .../cluster/routing/IndexRoutingTable.java | 28 +++----- .../routing/IndexShardRoutingTable.java | 38 ++-------- .../cluster/routing/RoutingNodes.java | 21 ------ .../cluster/routing/RoutingTable.java | 7 -- .../cluster/routing/ShardRouting.java | 21 ++++++ .../cluster/routing/UnassignedInfo.java | 6 +- .../command/AllocateAllocationCommand.java | 18 ++--- .../decider/DisableAllocationDecider.java | 2 +- .../decider/DiskThresholdDecider.java | 2 +- .../decider/EnableAllocationDecider.java | 2 +- .../gateway/PrimaryShardAllocator.java | 7 +- .../elasticsearch/index/shard/IndexShard.java | 8 +-- .../cluster/IndicesClusterStateService.java | 3 +- .../replication/ShardReplicationTests.java | 2 +- .../cluster/ClusterHealthResponsesTests.java | 2 +- .../cluster/ClusterStateDiffTests.java | 2 +- .../cluster/routing/UnassignedInfoTests.java | 3 +- .../allocation/AllocatePostApiFlagTests.java | 69 ------------------- .../allocation/CatAllocationTestBase.java | 2 +- .../allocation/StartedShardsRoutingTests.java | 6 +- .../decider/DiskThresholdDeciderTests.java | 8 +-- .../gateway/PrimaryShardAllocatorTests.java | 8 ++- .../store/IndicesStoreIntegrationTests.java | 2 +- .../indices/store/IndicesStoreTests.java | 12 ++-- 25 files changed, 92 insertions(+), 194 deletions(-) delete mode 100644 core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocatePostApiFlagTests.java diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java index f680ac1d5e5..b5b3cb6e1e4 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.common.Priority; @@ -92,8 +93,10 @@ public class MetaDataIndexStateService extends AbstractComponent { if (indexMetaData.state() != IndexMetaData.State.CLOSE) { IndexRoutingTable indexRoutingTable = currentState.routingTable().index(index); for (IndexShardRoutingTable shard : indexRoutingTable) { - if (!shard.primaryAllocatedPostApi()) { - throw new IndexPrimaryShardNotAllocatedException(new Index(index)); + for (ShardRouting shardRouting : shard) { + if (shardRouting.primary() == true && shardRouting.allocatedPostIndexCreate() == false) { + throw new IndexPrimaryShardNotAllocatedException(new Index(index)); + } } } indicesToClose.add(index); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index e4fa1800546..ac5a6483a95 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -383,28 +383,28 @@ public class IndexRoutingTable extends AbstractDiffable imple * Initializes a new empty index, as if it was created from an API. */ public Builder initializeAsNew(IndexMetaData indexMetaData) { - return initializeEmpty(indexMetaData, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); } /** * Initializes a new empty index, as if it was created from an API. */ public Builder initializeAsRecovery(IndexMetaData indexMetaData) { - return initializeEmpty(indexMetaData, false, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)); + return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)); } /** * Initializes a new index caused by dangling index imported. */ public Builder initializeAsFromDangling(IndexMetaData indexMetaData) { - return initializeEmpty(indexMetaData, false, new UnassignedInfo(UnassignedInfo.Reason.DANGLING_INDEX_IMPORTED, null)); + return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.DANGLING_INDEX_IMPORTED, null)); } /** * Initializes a new empty index, as as a result of opening a closed index. */ public Builder initializeAsFromCloseToOpen(IndexMetaData indexMetaData) { - return initializeEmpty(indexMetaData, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, null)); + return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, null)); } /** @@ -429,7 +429,7 @@ public class IndexRoutingTable extends AbstractDiffable imple throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created"); } for (int shardId = 0; shardId < indexMetaData.numberOfShards(); shardId++) { - IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.index(), shardId), asNew ? false : true); + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.index(), shardId)); for (int i = 0; i <= indexMetaData.numberOfReplicas(); i++) { if (asNew && ignoreShards.contains(shardId)) { // This shards wasn't completely snapshotted - restore it as new shard @@ -446,12 +446,12 @@ public class IndexRoutingTable extends AbstractDiffable imple /** * Initializes a new empty index, with an option to control if its from an API or not. */ - private Builder initializeEmpty(IndexMetaData indexMetaData, boolean asNew, UnassignedInfo unassignedInfo) { + private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unassignedInfo) { if (!shards.isEmpty()) { throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created"); } for (int shardId = 0; shardId < indexMetaData.numberOfShards(); shardId++) { - IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.index(), shardId), asNew ? false : true); + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.index(), shardId)); for (int i = 0; i <= indexMetaData.numberOfReplicas(); i++) { indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo)); } @@ -481,7 +481,7 @@ public class IndexRoutingTable extends AbstractDiffable imple return this; } // re-add all the current ones - IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(indexShard.shardId(), indexShard.primaryAllocatedPostApi()); + IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(indexShard.shardId()); for (ShardRouting shardRouting : indexShard) { builder.addShard(new ShardRouting(shardRouting)); } @@ -513,16 +513,6 @@ public class IndexRoutingTable extends AbstractDiffable imple return this; } - /** - * Clears the post allocation flag for the specified shard - */ - public Builder clearPostAllocationFlag(ShardId shardId) { - assert this.index.equals(shardId.index().name()); - IndexShardRoutingTable indexShard = shards.get(shardId.id()); - shards.put(indexShard.shardId().id(), new IndexShardRoutingTable(indexShard.shardId(), indexShard.shards(), false)); - return this; - } - /** * Adds a new shard routing (makes a copy of it), with reference data used from the index shard routing table * if it needs to be created. @@ -530,7 +520,7 @@ public class IndexRoutingTable extends AbstractDiffable imple public Builder addShard(IndexShardRoutingTable refData, ShardRouting shard) { IndexShardRoutingTable indexShard = shards.get(shard.id()); if (indexShard == null) { - indexShard = new IndexShardRoutingTable.Builder(refData.shardId(), refData.primaryAllocatedPostApi()).addShard(new ShardRouting(shard)).build(); + indexShard = new IndexShardRoutingTable.Builder(refData.shardId()).addShard(new ShardRouting(shard)).build(); } else { indexShard = new IndexShardRoutingTable.Builder(indexShard).addShard(new ShardRouting(shard)).build(); } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java index ef5011f4dbb..bc13e081563 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java @@ -62,13 +62,10 @@ public class IndexShardRoutingTable implements Iterable { */ final ImmutableList allInitializingShards; - final boolean primaryAllocatedPostApi; - - IndexShardRoutingTable(ShardId shardId, List shards, boolean primaryAllocatedPostApi) { + IndexShardRoutingTable(ShardId shardId, List shards) { this.shardId = shardId; this.shuffler = new RotationShardShuffler(ThreadLocalRandom.current().nextInt()); this.shards = ImmutableList.copyOf(shards); - this.primaryAllocatedPostApi = primaryAllocatedPostApi; ShardRouting primary = null; ImmutableList.Builder replicas = ImmutableList.builder(); @@ -144,15 +141,7 @@ public class IndexShardRoutingTable implements Iterable { shardRoutings.add(new ShardRouting(shards.get(i), highestVersion)); } } - return new IndexShardRoutingTable(shardId, ImmutableList.copyOf(shardRoutings), primaryAllocatedPostApi); - } - - /** - * Has this shard group primary shard been allocated post API creation. Will be set to - * true if it was created because of recovery action. - */ - public boolean primaryAllocatedPostApi() { - return primaryAllocatedPostApi; + return new IndexShardRoutingTable(shardId, ImmutableList.copyOf(shardRoutings)); } /** @@ -434,7 +423,6 @@ public class IndexShardRoutingTable implements Iterable { IndexShardRoutingTable that = (IndexShardRoutingTable) o; - if (primaryAllocatedPostApi != that.primaryAllocatedPostApi) return false; if (!shardId.equals(that.shardId)) return false; if (!shards.equals(that.shards)) return false; @@ -445,7 +433,6 @@ public class IndexShardRoutingTable implements Iterable { public int hashCode() { int result = shardId.hashCode(); result = 31 * result + shards.hashCode(); - result = 31 * result + (primaryAllocatedPostApi ? 1 : 0); return result; } @@ -594,21 +581,16 @@ public class IndexShardRoutingTable implements Iterable { public static class Builder { private ShardId shardId; - private final List shards; - private boolean primaryAllocatedPostApi; - public Builder(IndexShardRoutingTable indexShard) { this.shardId = indexShard.shardId; this.shards = newArrayList(indexShard.shards); - this.primaryAllocatedPostApi = indexShard.primaryAllocatedPostApi(); } - public Builder(ShardId shardId, boolean primaryAllocatedPostApi) { + public Builder(ShardId shardId) { this.shardId = shardId; this.shards = newArrayList(); - this.primaryAllocatedPostApi = primaryAllocatedPostApi; } public Builder addShard(ShardRouting shardEntry) { @@ -630,15 +612,7 @@ public class IndexShardRoutingTable implements Iterable { } public IndexShardRoutingTable build() { - // we can automatically set allocatedPostApi to true if the primary is active - if (!primaryAllocatedPostApi) { - for (ShardRouting shardRouting : shards) { - if (shardRouting.primary() && shardRouting.active()) { - primaryAllocatedPostApi = true; - } - } - } - return new IndexShardRoutingTable(shardId, ImmutableList.copyOf(shards), primaryAllocatedPostApi); + return new IndexShardRoutingTable(shardId, ImmutableList.copyOf(shards)); } public static IndexShardRoutingTable readFrom(StreamInput in) throws IOException { @@ -648,8 +622,7 @@ public class IndexShardRoutingTable implements Iterable { public static IndexShardRoutingTable readFromThin(StreamInput in, String index) throws IOException { int iShardId = in.readVInt(); - boolean allocatedPostApi = in.readBoolean(); - Builder builder = new Builder(new ShardId(index, iShardId), allocatedPostApi); + Builder builder = new Builder(new ShardId(index, iShardId)); int size = in.readVInt(); for (int i = 0; i < size; i++) { @@ -667,7 +640,6 @@ public class IndexShardRoutingTable implements Iterable { public static void writeToThin(IndexShardRoutingTable indexShard, StreamOutput out) throws IOException { out.writeVInt(indexShard.shardId.id()); - out.writeBoolean(indexShard.primaryAllocatedPostApi()); out.writeVInt(indexShard.shards.size()); for (ShardRouting entry : indexShard) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java index 4306be8b24e..dcb44e50ffe 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java @@ -64,8 +64,6 @@ public class RoutingNodes implements Iterable { private int relocatingShards = 0; - private Set clearPostAllocationFlag; - private final Map> nodesPerAttributeNames = new HashMap<>(); public RoutingNodes(ClusterState clusterState) { @@ -191,25 +189,6 @@ public class RoutingNodes implements Iterable { return new RoutingNodesIterator(nodesToShards.values().iterator()); } - /** - * Clears the post allocation flag for the provided shard id. NOTE: this should be used cautiously - * since it will lead to data loss of the primary shard is not allocated, as it will allocate - * the primary shard on a node and *not* expect it to have an existing valid index there. - */ - public void addClearPostAllocationFlag(ShardId shardId) { - if (clearPostAllocationFlag == null) { - clearPostAllocationFlag = Sets.newHashSet(); - } - clearPostAllocationFlag.add(shardId); - } - - public Iterable getShardsToClearPostAllocationFlag() { - if (clearPostAllocationFlag == null) { - return ImmutableSet.of(); - } - return clearPostAllocationFlag; - } - public RoutingNode node(String nodeId) { return nodesToShards.get(nodeId); } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index 156f54112f2..8b4f75dcbc4 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -373,13 +373,6 @@ public class RoutingTable implements Iterable, Diffable { /** * Unassigned as a result of explicit cancel reroute command. */ - REROUTE_CANCELLED; + REROUTE_CANCELLED, + /** + * When a shard moves from started back to initializing, for example, during shadow replica + */ + REINITIALIZED; } private final Reason reason; diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java index c7598700341..5e9f44b92b0 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java @@ -21,9 +21,10 @@ package org.elasticsearch.cluster.routing.allocation.command; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.RerouteExplanation; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.decider.Decision; @@ -35,7 +36,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.shard.ShardId; import java.io.IOException; -import java.util.Iterator; /** * Allocates an unassigned shard to a specific node. Note, primary allocation will "force" @@ -221,15 +221,17 @@ public class AllocateAllocationCommand implements AllocationCommand { } // go over and remove it from the unassigned for (RoutingNodes.UnassignedShards.UnassignedIterator it = routingNodes.unassigned().iterator(); it.hasNext(); ) { - if (it.next() != shardRouting) { + ShardRouting unassigned = it.next(); + if (unassigned != shardRouting) { continue; } - it.initialize(routingNode.nodeId()); - if (shardRouting.primary()) { - // we need to clear the post allocation flag, since its an explicit allocation of the primary shard - // and we want to force allocate it (and create a new index for it) - routingNodes.addClearPostAllocationFlag(shardRouting.shardId()); + // if we force allocation of a primary, we need to move the unassigned info back to treat it as if + // it was index creation + if (unassigned.primary() && unassigned.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED) { + unassigned.updateUnassignedInfo(new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, + "force allocation from previous reason " + unassigned.unassignedInfo().getReason() + ", " + unassigned.unassignedInfo().getMessage(), unassigned.unassignedInfo().getFailure())); } + it.initialize(routingNode.nodeId()); break; } return new RerouteExplanation(this, decision); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DisableAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DisableAllocationDecider.java index 26a0beec0c2..bf3c833b7f8 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DisableAllocationDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DisableAllocationDecider.java @@ -109,7 +109,7 @@ public class DisableAllocationDecider extends AllocationDecider { return allocation.decision(Decision.YES, NAME, "allocation disabling is ignored"); } Settings indexSettings = allocation.routingNodes().metaData().index(shardRouting.index()).settings(); - if (shardRouting.primary() && !allocation.routingNodes().routingTable().index(shardRouting.index()).shard(shardRouting.id()).primaryAllocatedPostApi()) { + if (shardRouting.primary() && shardRouting.allocatedPostIndexCreate() == false) { // if its primary, and it hasn't been allocated post API (meaning its a "fresh newly created shard"), only disable allocation // on a special disable allocation flag if (indexSettings.getAsBoolean(INDEX_ROUTING_ALLOCATION_DISABLE_NEW_ALLOCATION, disableNewAllocation)) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index 8519360375d..87378df1538 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -311,7 +311,7 @@ public class DiskThresholdDecider extends AllocationDecider { } // a flag for whether the primary shard has been previously allocated - boolean primaryHasBeenAllocated = allocation.routingTable().index(shardRouting.index()).shard(shardRouting.id()).primaryAllocatedPostApi(); + boolean primaryHasBeenAllocated = shardRouting.primary() && shardRouting.allocatedPostIndexCreate(); // checks for exact byte comparisons if (freeBytes < freeBytesThresholdLow.bytes()) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java index 7546482d87a..de4bb4cbee5 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java @@ -98,7 +98,7 @@ public class EnableAllocationDecider extends AllocationDecider implements NodeSe case NONE: return allocation.decision(Decision.NO, NAME, "no allocations are allowed"); case NEW_PRIMARIES: - if (shardRouting.primary() && !allocation.routingNodes().routingTable().index(shardRouting.index()).shard(shardRouting.id()).primaryAllocatedPostApi()) { + if (shardRouting.primary() && shardRouting.allocatedPostIndexCreate() == false) { return allocation.decision(Decision.YES, NAME, "new primary allocations are allowed"); } else { return allocation.decision(Decision.NO, NAME, "non-new primary allocations are forbidden"); diff --git a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java index 6b23a0c96e9..715ba681bbc 100644 --- a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java +++ b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java @@ -25,7 +25,6 @@ import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; @@ -62,7 +61,7 @@ public abstract class PrimaryShardAllocator extends AbstractComponent { while (unassignedIterator.hasNext()) { ShardRouting shard = unassignedIterator.next(); - if (needToFindPrimaryCopy(shard, routingNodes.routingTable().index(shard.index()).shard(shard.id())) == false) { + if (needToFindPrimaryCopy(shard) == false) { continue; } @@ -113,13 +112,13 @@ public abstract class PrimaryShardAllocator extends AbstractComponent { /** * Does the shard need to find a primary copy? */ - boolean needToFindPrimaryCopy(ShardRouting shard, IndexShardRoutingTable indexShardRoutingTable) { + boolean needToFindPrimaryCopy(ShardRouting shard) { if (shard.primary() == false) { return false; } // this is an API allocation, ignore since we know there is no data... - if (indexShardRoutingTable.primaryAllocatedPostApi() == false) { + if (shard.allocatedPostIndexCreate() == false) { return false; } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index dd4b6bc7dc9..900603bd7a1 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -24,13 +24,11 @@ import com.google.common.base.Preconditions; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.CheckIndex; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.UsageTrackingQueryCachingPolicy; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.ThreadInterruptedException; -import org.elasticsearch.ElasticsearchCorruptionException; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.flush.FlushRequest; @@ -39,7 +37,6 @@ import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RestoreSource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; @@ -1039,10 +1036,11 @@ public class IndexShard extends AbstractIndexShardComponent { return path; } - public void recoverFromStore(IndexShardRoutingTable shardRoutingTable, StoreRecoveryService.RecoveryListener recoveryListener) { + public void recoverFromStore(ShardRouting shard, StoreRecoveryService.RecoveryListener recoveryListener) { // we are the first primary, recover from the gateway // if its post api allocation, the index should exists - final boolean shouldExist = shardRoutingTable.primaryAllocatedPostApi(); + assert shard.primary() : "recover from store only makes sense if the shard is a primary shard"; + final boolean shouldExist = shard.allocatedPostIndexCreate(); storeRecoveryService.recover(this, shouldExist, recoveryListener); } diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 8be847fa97b..b17c639ae06 100644 --- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -676,8 +676,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent shardIdToRouting = new HashMap<>(); for (ShardRouting r : idx.routing) { - IndexShardRoutingTable refData = new IndexShardRoutingTable.Builder(new ShardId(idx.name, r.id()), true).addShard(r).build(); + IndexShardRoutingTable refData = new IndexShardRoutingTable.Builder(new ShardId(idx.name, r.id())).addShard(r).build(); if (shardIdToRouting.containsKey(r.getId())) { refData = new IndexShardRoutingTable.Builder(shardIdToRouting.get(r.getId())).addShard(r).build(); } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java index 506d3d29e0f..92847cb5be6 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java @@ -54,9 +54,9 @@ public class StartedShardsRoutingTests extends ElasticsearchAllocationTestCase { final ShardRouting startedShard = TestShardRouting.newShardRouting("test", 1, "node2", randomBoolean(), ShardRoutingState.STARTED, 1); final ShardRouting relocatingShard = TestShardRouting.newShardRouting("test", 2, "node1", "node2", randomBoolean(), ShardRoutingState.RELOCATING, 1); stateBuilder.routingTable(RoutingTable.builder().add(IndexRoutingTable.builder("test") - .addIndexShard(new IndexShardRoutingTable.Builder(initShard.shardId(), true).addShard(initShard).build()) - .addIndexShard(new IndexShardRoutingTable.Builder(startedShard.shardId(), true).addShard(startedShard).build()) - .addIndexShard(new IndexShardRoutingTable.Builder(relocatingShard.shardId(), true).addShard(relocatingShard).build()))); + .addIndexShard(new IndexShardRoutingTable.Builder(initShard.shardId()).addShard(initShard).build()) + .addIndexShard(new IndexShardRoutingTable.Builder(startedShard.shardId()).addShard(startedShard).build()) + .addIndexShard(new IndexShardRoutingTable.Builder(relocatingShard.shardId()).addShard(relocatingShard).build()))); ClusterState state = stateBuilder.build(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index e92a4fde2e0..cc5628f605a 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -834,11 +834,11 @@ public class DiskThresholdDeciderTests extends ElasticsearchAllocationTestCase { RoutingNode firstRoutingNode = new RoutingNode("node1", discoveryNode1, Arrays.asList(firstRouting, secondRouting)); RoutingTable.Builder builder = RoutingTable.builder().add( IndexRoutingTable.builder("test") - .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId("test", 0), false) + .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId("test", 0)) .addShard(firstRouting) .build() ) - .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId("test", 1), false) + .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId("test", 1)) .addShard(secondRouting) .build() ) @@ -854,11 +854,11 @@ public class DiskThresholdDeciderTests extends ElasticsearchAllocationTestCase { firstRoutingNode = new RoutingNode("node1", discoveryNode1, Arrays.asList(firstRouting, secondRouting)); builder = RoutingTable.builder().add( IndexRoutingTable.builder("test") - .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId("test", 0), false) + .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId("test", 0)) .addShard(firstRouting) .build() ) - .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId("test", 1), false) + .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId("test", 1)) .addShard(secondRouting) .build() ) diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java index c5d41d9acd1..5ffeb40cf41 100644 --- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java @@ -66,7 +66,13 @@ public class PrimaryShardAllocatorTests extends ElasticsearchAllocationTestCase @Test public void testNoProcessReplica() { ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, false, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)); - assertThat(testAllocator.needToFindPrimaryCopy(shard, null), equalTo(false)); + assertThat(testAllocator.needToFindPrimaryCopy(shard), equalTo(false)); + } + + @Test + public void testNoProcessPrimayNotAllcoatedBefore() { + ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, true, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + assertThat(testAllocator.needToFindPrimaryCopy(shard), equalTo(false)); } /** diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java index dd357971f03..9a4783a2fa2 100644 --- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java +++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java @@ -248,7 +248,7 @@ public class IndicesStoreIntegrationTests extends ElasticsearchIntegrationTest { IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder("test"); for (int i = 0; i < numShards; i++) { indexRoutingTableBuilder.addIndexShard( - new IndexShardRoutingTable.Builder(new ShardId("test", i), false) + new IndexShardRoutingTable.Builder(new ShardId("test", i)) .addShard(TestShardRouting.newShardRouting("test", i, masterId, true, ShardRoutingState.STARTED, shardVersions[shardIds[i]])) .build() ); diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java index 2d0c7e9d7a4..bf34648d648 100644 --- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java +++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java @@ -70,7 +70,7 @@ public class IndicesStoreTests extends ElasticsearchTestCase { ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test")); clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas))); - IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1), false); + IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1)); assertFalse(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build())); } @@ -82,7 +82,7 @@ public class IndicesStoreTests extends ElasticsearchTestCase { ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test")); clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas))); - IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1), false); + IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1)); for (int i = 0; i < numShards; i++) { int unStartedShard = randomInt(numReplicas); @@ -111,7 +111,7 @@ public class IndicesStoreTests extends ElasticsearchTestCase { ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test")); clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas))); clusterState.nodes(DiscoveryNodes.builder().localNodeId(localNode.id()).put(localNode).put(new DiscoveryNode("xyz", new LocalTransportAddress("xyz"), Version.CURRENT))); - IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1), false); + IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1)); int localShardId = randomInt(numShards - 1); for (int i = 0; i < numShards; i++) { String nodeId = i == localShardId ? localNode.getId() : randomBoolean() ? "abc" : "xyz"; @@ -134,7 +134,7 @@ public class IndicesStoreTests extends ElasticsearchTestCase { ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test")); clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas))); clusterState.nodes(DiscoveryNodes.builder().localNodeId(localNode.id()).put(localNode)); - IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1), false); + IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1)); for (int i = 0; i < numShards; i++) { String relocatingNodeId = randomBoolean() ? null : "def"; routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", relocatingNodeId, true, ShardRoutingState.STARTED, 0)); @@ -157,7 +157,7 @@ public class IndicesStoreTests extends ElasticsearchTestCase { ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test")); clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas))); clusterState.nodes(DiscoveryNodes.builder().localNodeId(localNode.id()).put(localNode).put(new DiscoveryNode("xyz", new LocalTransportAddress("xyz"), nodeVersion))); - IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1), false); + IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1)); for (int i = 0; i < numShards; i++) { routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", null, true, ShardRoutingState.STARTED, 0)); for (int j = 0; j < numReplicas; j++) { @@ -183,7 +183,7 @@ public class IndicesStoreTests extends ElasticsearchTestCase { .put(new DiscoveryNode("xyz", new LocalTransportAddress("xyz"), Version.CURRENT)) .put(new DiscoveryNode("def", new LocalTransportAddress("def"), nodeVersion) // <-- only set relocating, since we're testing that in this test )); - IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1), false); + IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1)); for (int i = 0; i < numShards; i++) { routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", "def", true, ShardRoutingState.STARTED, 0)); for (int j = 0; j < numReplicas; j++) {