Add recovery source to ShardRouting (#19516)

Adds an explicit recoverySource field to ShardRouting that characterizes the type of recovery to perform:

- fresh empty shard copy
- existing local shard copy
- recover from peer (primary)
- recover from snapshot
- recover from other local shards on same node (shrink index action)

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -460,7 +460,7 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoveriesCollection.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoveryFailedException.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoverySettings.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoverySource.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]PeerRecoverySourceService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoveryState.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]StartRecoveryRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStore.java" checks="LineLength" />

core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java

@@ -37,6 +37,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -168,31 +169,35 @@ public class TransportClusterAllocationExplainAction
         if (node.getId().equals(assignedNodeId)) {
             finalDecision = ClusterAllocationExplanation.FinalDecision.ALREADY_ASSIGNED;
             finalExplanation = "the shard is already assigned to this node";
-        } else if (hasPendingAsyncFetch &&
-                shard.primary() == false &&
-                shard.unassigned() &&
-                shard.allocatedPostIndexCreate(indexMetaData) &&
-                nodeDecision.type() != Decision.Type.YES) {
+        } else if (shard.unassigned() && shard.primary() == false &&
+                shard.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED && nodeDecision.type() != Decision.Type.YES) {
             finalExplanation = "the shard cannot be assigned because allocation deciders return a " + nodeDecision.type().name() +
-                    " decision and the shard's state is still being fetched";
+                    " decision";
             finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (hasPendingAsyncFetch &&
-                shard.unassigned() &&
-                shard.allocatedPostIndexCreate(indexMetaData)) {
+        } else if (shard.unassigned() && shard.primary() == false &&
+                shard.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED && hasPendingAsyncFetch) {
             finalExplanation = "the shard's state is still being fetched so it cannot be allocated";
             finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() && shard.allocatedPostIndexCreate(indexMetaData) &&
+        } else if (shard.primary() && shard.unassigned() &&
+                (shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE ||
+                    shard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT)
+                && hasPendingAsyncFetch) {
+            finalExplanation = "the shard's state is still being fetched so it cannot be allocated";
+            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
+        } else if (shard.primary() && shard.unassigned() && shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE &&
                 storeCopy == ClusterAllocationExplanation.StoreCopy.STALE) {
             finalExplanation = "the copy of the shard is stale, allocation ids do not match";
             finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() && shard.allocatedPostIndexCreate(indexMetaData) &&
+        } else if (shard.primary() && shard.unassigned() && shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE &&
                 storeCopy == ClusterAllocationExplanation.StoreCopy.NONE) {
             finalExplanation = "there is no copy of the shard available";
             finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() && storeCopy == ClusterAllocationExplanation.StoreCopy.CORRUPT) {
+        } else if (shard.primary() && shard.unassigned() && shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE &&
+                storeCopy == ClusterAllocationExplanation.StoreCopy.CORRUPT) {
             finalExplanation = "the copy of the shard is corrupt";
             finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() && storeCopy == ClusterAllocationExplanation.StoreCopy.IO_ERROR) {
+        } else if (shard.primary() && shard.unassigned() && shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE &&
+                storeCopy == ClusterAllocationExplanation.StoreCopy.IO_ERROR) {
             finalExplanation = "the copy of the shard cannot be read";
             finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
         } else {

core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java

@@ -94,14 +94,13 @@ public class TransportIndicesShardStoresAction extends TransportMasterNodeReadAc
         logger.trace("using cluster state version [{}] to determine shards", state.version());
         // collect relevant shard ids of the requested indices for fetching store infos
         for (String index : concreteIndices) {
-            IndexMetaData indexMetaData = state.metaData().index(index);
             IndexRoutingTable indexShardRoutingTables = routingTables.index(index);
             if (indexShardRoutingTables == null) {
                 continue;
             }
             for (IndexShardRoutingTable routing : indexShardRoutingTables) {
                 final int shardId = routing.shardId().id();
-                ClusterShardHealth shardHealth = new ClusterShardHealth(shardId, routing, indexMetaData);
+                ClusterShardHealth shardHealth = new ClusterShardHealth(shardId, routing);
                 if (request.shardStatuses().contains(shardHealth.getStatus())) {
                     shardIdsToFetch.add(routing.shardId());
                 }

core/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java

@@ -54,7 +54,7 @@ public final class ClusterIndexHealth implements Iterable<ClusterShardHealth>, W
 
         for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) {
             int shardId = shardRoutingTable.shardId().id();
-            shards.put(shardId, new ClusterShardHealth(shardId, shardRoutingTable, indexMetaData));
+            shards.put(shardId, new ClusterShardHealth(shardId, shardRoutingTable));
         }
 
         // update the index status

core/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java

@@ -19,12 +19,11 @@
 
 package org.elasticsearch.cluster.health;
 
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
-import org.elasticsearch.cluster.routing.UnassignedInfo.Reason;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -41,7 +40,7 @@ public final class ClusterShardHealth implements Writeable {
     private final int unassignedShards;
     private final boolean primaryActive;
 
-    public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardRoutingTable, final IndexMetaData indexMetaData) {
+    public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardRoutingTable) {
         this.shardId = shardId;
         int computeActiveShards = 0;
         int computeRelocatingShards = 0;
@@ -69,7 +68,7 @@ public final class ClusterShardHealth implements Writeable {
                 computeStatus = ClusterHealthStatus.YELLOW;
             }
         } else {
-            computeStatus = getInactivePrimaryHealth(primaryRouting, indexMetaData);
+            computeStatus = getInactivePrimaryHealth(primaryRouting);
         }
         this.status = computeStatus;
         this.activeShards = computeActiveShards;
@@ -131,28 +130,25 @@ public final class ClusterShardHealth implements Writeable {
     /**
      * Checks if an inactive primary shard should cause the cluster health to go RED.
      *
-     * Normally, an inactive primary shard in an index should cause the cluster health to be RED.  However,
-     * there are exceptions where a health status of RED is inappropriate, namely in these scenarios:
-     *   1. Index Creation.  When an index is first created, the primary shards are in the initializing state, so
-     *      there is a small window where the cluster health is RED due to the primaries not being activated yet.
-     *      However, this leads to a false sense that the cluster is in an unhealthy state, when in reality, its
-     *      simply a case of needing to wait for the primaries to initialize.
-     *   2. When a cluster is in the recovery state, and the shard never had any allocation ids assigned to it,
-     *      which indicates the index was created and before allocation of the primary occurred for this shard,
-     *      a cluster restart happened.
-     *
-     * Here, we check for these scenarios and set the cluster health to YELLOW if any are applicable.
+     * An inactive primary shard in an index should cause the cluster health to be RED to make it visible that some of the existing data is
+     * unavailable. In case of index creation, snapshot restore or index shrinking, which are unexceptional events in the cluster lifecycle,
+     * cluster health should not turn RED for the time where primaries are still in the initializing state but go to YELLOW instead.
+     * However, in case of exceptional events, for example when the primary shard cannot be assigned to a node or initialization fails at
+     * some point, cluster health should still turn RED.
      *
      * NB: this method should *not* be called on active shards nor on non-primary shards.
      */
-    public static ClusterHealthStatus getInactivePrimaryHealth(final ShardRouting shardRouting, final IndexMetaData indexMetaData) {
+    public static ClusterHealthStatus getInactivePrimaryHealth(final ShardRouting shardRouting) {
         assert shardRouting.primary() : "cannot invoke on a replica shard: " + shardRouting;
         assert shardRouting.active() == false : "cannot invoke on an active shard: " + shardRouting;
         assert shardRouting.unassignedInfo() != null : "cannot invoke on a shard with no UnassignedInfo: " + shardRouting;
+        assert shardRouting.recoverySource() != null : "cannot invoke on a shard that has no recovery source" + shardRouting;
         final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
-        if (unassignedInfo.getLastAllocationStatus() != AllocationStatus.DECIDERS_NO
-                && shardRouting.allocatedPostIndexCreate(indexMetaData) == false
-                && (unassignedInfo.getReason() == Reason.INDEX_CREATED || unassignedInfo.getReason() == Reason.CLUSTER_RECOVERED)) {
+        RecoverySource.Type recoveryType = shardRouting.recoverySource().getType();
+        if (unassignedInfo.getLastAllocationStatus() != AllocationStatus.DECIDERS_NO && unassignedInfo.getNumFailedAllocations() == 0
+                && (recoveryType == RecoverySource.Type.EMPTY_STORE
+                    || recoveryType == RecoverySource.Type.LOCAL_SHARDS
+                    || recoveryType == RecoverySource.Type.SNAPSHOT)) {
             return ClusterHealthStatus.YELLOW;
         } else {
             return ClusterHealthStatus.RED;

core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java

@@ -23,9 +23,15 @@ import com.carrotsearch.hppc.IntSet;
 import com.carrotsearch.hppc.cursors.IntCursor;
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -353,54 +359,57 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
          * Initializes a new empty index, as if it was created from an API.
          */
         public Builder initializeAsNew(IndexMetaData indexMetaData) {
-            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+            RecoverySource primaryRecoverySource = indexMetaData.getMergeSourceIndex() != null ?
+                LocalShardsRecoverySource.INSTANCE :
+                StoreRecoverySource.EMPTY_STORE_INSTANCE;
+            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null), primaryRecoverySource);
         }
 
         /**
-         * Initializes a new empty index, as if it was created from an API.
+         * Initializes an existing index.
          */
         public Builder initializeAsRecovery(IndexMetaData indexMetaData) {
-            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null));
+            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null), null);
         }
 
         /**
          * Initializes a new index caused by dangling index imported.
          */
         public Builder initializeAsFromDangling(IndexMetaData indexMetaData) {
-            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.DANGLING_INDEX_IMPORTED, null));
+            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.DANGLING_INDEX_IMPORTED, null), null);
         }
 
         /**
          * Initializes a new empty index, as as a result of opening a closed index.
          */
         public Builder initializeAsFromCloseToOpen(IndexMetaData indexMetaData) {
-            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, null));
+            return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, null), null);
         }
 
         /**
          * Initializes a new empty index, to be restored from a snapshot
          */
-        public Builder initializeAsNewRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards) {
+        public Builder initializeAsNewRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards) {
             final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NEW_INDEX_RESTORED,
-                                                                     "restore_source[" + restoreSource.snapshot().getRepository() + "/" +
-                                                                         restoreSource.snapshot().getSnapshotId().getName() + "]");
-            return initializeAsRestore(indexMetaData, restoreSource, ignoreShards, true, unassignedInfo);
+                                                                     "restore_source[" + recoverySource.snapshot().getRepository() + "/" +
+                                                                         recoverySource.snapshot().getSnapshotId().getName() + "]");
+            return initializeAsRestore(indexMetaData, recoverySource, ignoreShards, true, unassignedInfo);
         }
 
         /**
          * Initializes an existing index, to be restored from a snapshot
          */
-        public Builder initializeAsRestore(IndexMetaData indexMetaData, RestoreSource restoreSource) {
+        public Builder initializeAsRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource) {
             final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED,
-                                                                     "restore_source[" + restoreSource.snapshot().getRepository() + "/" +
-                                                                         restoreSource.snapshot().getSnapshotId().getName() + "]");
-            return initializeAsRestore(indexMetaData, restoreSource, null, false, unassignedInfo);
+                                                                     "restore_source[" + recoverySource.snapshot().getRepository() + "/" +
+                                                                         recoverySource.snapshot().getSnapshotId().getName() + "]");
+            return initializeAsRestore(indexMetaData, recoverySource, null, false, unassignedInfo);
         }
 
         /**
          * Initializes an index, to be restored from snapshot
          */
-        private Builder initializeAsRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards, boolean asNew, UnassignedInfo unassignedInfo) {
+        private Builder initializeAsRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards, boolean asNew, UnassignedInfo unassignedInfo) {
             assert indexMetaData.getIndex().equals(index);
             if (!shards.isEmpty()) {
                 throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
@@ -409,11 +418,14 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
                 ShardId shardId = new ShardId(index, shardNumber);
                 IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
                 for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
+                    boolean primary = i == 0;
                     if (asNew && ignoreShards.contains(shardNumber)) {
                         // This shards wasn't completely snapshotted - restore it as new shard
-                        indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, null, i == 0, unassignedInfo));
+                        indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary,
+                            primary ? StoreRecoverySource.EMPTY_STORE_INSTANCE : PeerRecoverySource.INSTANCE, unassignedInfo));
                     } else {
-                        indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, i == 0 ? restoreSource : null, i == 0, unassignedInfo));
+                        indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary,
+                            primary ? recoverySource : PeerRecoverySource.INSTANCE, unassignedInfo));
                     }
                 }
                 shards.put(shardNumber, indexShardRoutingBuilder.build());
@@ -423,17 +435,28 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
 
         /**
          * Initializes a new empty index, with an option to control if its from an API or not.
+         *
+         * @param primaryRecoverySource recovery source for primary shards. If null, it is automatically determined based on active
+         *                              allocation ids
          */
-        private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unassignedInfo) {
+        private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unassignedInfo, @Nullable RecoverySource primaryRecoverySource) {
             assert indexMetaData.getIndex().equals(index);
             if (!shards.isEmpty()) {
                 throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
             }
             for (int shardNumber = 0; shardNumber < indexMetaData.getNumberOfShards(); shardNumber++) {
                 ShardId shardId = new ShardId(index, shardNumber);
+                if (primaryRecoverySource == null) {
+                    if (indexMetaData.inSyncAllocationIds(shardNumber).isEmpty() && indexMetaData.getCreationVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
+                        primaryRecoverySource = indexMetaData.getMergeSourceIndex() != null ? LocalShardsRecoverySource.INSTANCE : StoreRecoverySource.EMPTY_STORE_INSTANCE;
+                    } else {
+                        primaryRecoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE;
+                    }
+                }
                 IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
                 for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
-                    indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, null, i == 0, unassignedInfo));
+                    boolean primary = i == 0;
+                    indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary, primary ? primaryRecoverySource : PeerRecoverySource.INSTANCE, unassignedInfo));
                 }
                 shards.put(shardNumber, indexShardRoutingBuilder.build());
             }
@@ -445,7 +468,7 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
                 int shardNumber = cursor.value;
                 ShardId shardId = new ShardId(index, shardNumber);
                 // version 0, will get updated when reroute will happen
-                ShardRouting shard = ShardRouting.newUnassigned(shardId, null, false, new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null));
+                ShardRouting shard = ShardRouting.newUnassigned(shardId, false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null));
                 shards.put(shardNumber,
                         new IndexShardRoutingTable.Builder(shards.get(shard.id())).addShard(shard).build()
                 );

core/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java

@@ -0,0 +1,250 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.snapshots.Snapshot;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Represents the recovery source of a shard. Available recovery types are:
+ *
+ * - {@link StoreRecoverySource} recovery from the local store (empty or with existing data)
+ * - {@link PeerRecoverySource} recovery from a primary on another node
+ * - {@link SnapshotRecoverySource} recovery from a snapshot
+ * - {@link LocalShardsRecoverySource} recovery from other shards of another index on the same node
+ */
+public abstract class RecoverySource implements Writeable, ToXContent {
+
+    @Override
+    public final XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
+        builder.field("type", getType());
+        addAdditionalFields(builder, params);
+        return builder.endObject();
+    }
+
+    /**
+     * to be overridden by subclasses
+     */
+    public void addAdditionalFields(XContentBuilder builder, ToXContent.Params params) throws IOException {
+
+    }
+
+    public static RecoverySource readFrom(StreamInput in) throws IOException {
+        Type type = Type.values()[in.readByte()];
+        switch (type) {
+            case EMPTY_STORE: return StoreRecoverySource.EMPTY_STORE_INSTANCE;
+            case EXISTING_STORE: return StoreRecoverySource.EXISTING_STORE_INSTANCE;
+            case PEER: return PeerRecoverySource.INSTANCE;
+            case SNAPSHOT: return new SnapshotRecoverySource(in);
+            case LOCAL_SHARDS: return LocalShardsRecoverySource.INSTANCE;
+            default: throw new IllegalArgumentException("unknown recovery type: " + type.name());
+        }
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeByte((byte) getType().ordinal());
+        writeAdditionalFields(out);
+    }
+
+    /**
+     * to be overridden by subclasses
+     */
+    protected void writeAdditionalFields(StreamOutput out) throws IOException {
+
+    }
+
+    public enum Type {
+        EMPTY_STORE,
+        EXISTING_STORE,
+        PEER,
+        SNAPSHOT,
+        LOCAL_SHARDS
+    }
+
+    public abstract Type getType();
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        RecoverySource that = (RecoverySource) o;
+
+        return getType() == that.getType();
+    }
+
+    @Override
+    public int hashCode() {
+        return getType().hashCode();
+    }
+
+    /**
+     * recovery from an existing on-disk store or a fresh copy
+     */
+    public abstract static class StoreRecoverySource extends RecoverySource {
+        public static final StoreRecoverySource EMPTY_STORE_INSTANCE = new StoreRecoverySource() {
+            @Override
+            public Type getType() {
+                return Type.EMPTY_STORE;
+            }
+        };
+        public static final StoreRecoverySource EXISTING_STORE_INSTANCE = new StoreRecoverySource() {
+            @Override
+            public Type getType() {
+                return Type.EXISTING_STORE;
+            }
+        };
+
+        @Override
+        public String toString() {
+            return getType() == Type.EMPTY_STORE ? "new shard recovery" : "existing recovery";
+        }
+    }
+
+    /**
+     * recovery from other shards on same node (shrink index action)
+     */
+    public static class LocalShardsRecoverySource extends RecoverySource {
+
+        public static final LocalShardsRecoverySource INSTANCE = new LocalShardsRecoverySource();
+
+        private LocalShardsRecoverySource() {
+        }
+
+        @Override
+        public Type getType() {
+            return Type.LOCAL_SHARDS;
+        }
+
+        @Override
+        public String toString() {
+            return "local shards recovery";
+        }
+
+    }
+
+    /**
+     * recovery from a snapshot
+     */
+    public static class SnapshotRecoverySource extends RecoverySource {
+        private final Snapshot snapshot;
+        private final String index;
+        private final Version version;
+
+        public SnapshotRecoverySource(Snapshot snapshot, Version version, String index) {
+            this.snapshot = Objects.requireNonNull(snapshot);
+            this.version = Objects.requireNonNull(version);
+            this.index = Objects.requireNonNull(index);
+        }
+
+        SnapshotRecoverySource(StreamInput in) throws IOException {
+            snapshot = new Snapshot(in);
+            version = Version.readVersion(in);
+            index = in.readString();
+        }
+
+        public Snapshot snapshot() {
+            return snapshot;
+        }
+
+        public String index() {
+            return index;
+        }
+
+        public Version version() {
+            return version;
+        }
+
+        @Override
+        protected void writeAdditionalFields(StreamOutput out) throws IOException {
+            snapshot.writeTo(out);
+            Version.writeVersion(version, out);
+            out.writeString(index);
+        }
+
+        @Override
+        public Type getType() {
+            return Type.SNAPSHOT;
+        }
+
+        @Override
+        public void addAdditionalFields(XContentBuilder builder, ToXContent.Params params) throws IOException {
+            builder.field("repository", snapshot.getRepository())
+                .field("snapshot", snapshot.getSnapshotId().getName())
+                .field("version", version.toString())
+                .field("index", index);
+        }
+
+        @Override
+        public String toString() {
+            return "snapshot recovery from " + snapshot.toString();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+
+            @SuppressWarnings("unchecked") SnapshotRecoverySource that = (SnapshotRecoverySource) o;
+            return snapshot.equals(that.snapshot) && index.equals(that.index) && version.equals(that.version);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(snapshot, index, version);
+        }
+
+    }
+
+    /**
+     * peer recovery from a primary shard
+     */
+    public static class PeerRecoverySource extends RecoverySource {
+
+        public static final PeerRecoverySource INSTANCE = new PeerRecoverySource();
+
+        private PeerRecoverySource() {
+        }
+
+        @Override
+        public Type getType() {
+            return Type.PEER;
+        }
+
+        @Override
+        public String toString() {
+            return "peer recovery";
+        }
+    }
+}

core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java

@@ -1,115 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.routing;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.snapshots.Snapshot;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Objects;
-
-/**
- * Represents snapshot and index from which a recovering index should be restored
- */
-public class RestoreSource implements Streamable, ToXContent {
-
-    private Snapshot snapshot;
-
-    private String index;
-
-    private Version version;
-
-    RestoreSource() {
-    }
-
-    public RestoreSource(Snapshot snapshot, Version version, String index) {
-        this.snapshot = Objects.requireNonNull(snapshot);
-        this.version = Objects.requireNonNull(version);
-        this.index = Objects.requireNonNull(index);
-    }
-
-    public Snapshot snapshot() {
-        return snapshot;
-    }
-
-    public String index() {
-        return index;
-    }
-
-    public Version version() {
-        return version;
-    }
-
-    public static RestoreSource readOptionalRestoreSource(StreamInput in) throws IOException {
-        return in.readOptionalStreamable(RestoreSource::new);
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        snapshot = new Snapshot(in);
-        version = Version.readVersion(in);
-        index = in.readString();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        snapshot.writeTo(out);
-        Version.writeVersion(version, out);
-        out.writeString(index);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return builder.startObject()
-                .field("repository", snapshot.getRepository())
-                .field("snapshot", snapshot.getSnapshotId().getName())
-                .field("version", version.toString())
-                .field("index", index)
-                .endObject();
-    }
-
-    @Override
-    public String toString() {
-        return snapshot.toString();
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-
-        @SuppressWarnings("unchecked") RestoreSource that = (RestoreSource) o;
-        return snapshot.equals(that.snapshot) && index.equals(that.index);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(snapshot, index);
-    }
-}

core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java

@@ -166,7 +166,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
 
         Recoveries.getOrAdd(recoveriesPerNode, routing.currentNodeId()).addIncoming(howMany);
 
-        if (routing.isPeerRecovery()) {
+        if (routing.recoverySource().getType() == RecoverySource.Type.PEER) {
             // add/remove corresponding outgoing recovery on node with primary shard
             if (primary == null) {
                 throw new IllegalStateException("shard is peer recovering but primary is unassigned");
@@ -177,7 +177,8 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                 // primary is done relocating, move non-primary recoveries from old primary to new primary
                 int numRecoveringReplicas = 0;
                 for (ShardRouting assigned : assignedShards(routing.shardId())) {
-                    if (assigned.primary() == false && assigned.isPeerRecovery()) {
+                    if (assigned.primary() == false && assigned.initializing() &&
+                        assigned.recoverySource().getType() == RecoverySource.Type.PEER) {
                         numRecoveringReplicas++;
                     }
                 }
@@ -198,7 +199,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
     @Nullable
     private ShardRouting findAssignedPrimaryIfPeerRecovery(ShardRouting routing) {
         ShardRouting primary = null;
-        if (routing.isPeerRecovery()) {
+        if (routing.recoverySource() != null && routing.recoverySource().getType() == RecoverySource.Type.PEER) {
             List<ShardRouting> shardRoutings = assignedShards.get(routing.shardId());
             if (shardRoutings != null) {
                 for (ShardRouting shardRouting : shardRoutings) {
@@ -627,7 +628,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
     private ShardRouting promoteActiveReplicaShardToPrimary(ShardRouting replicaShard) {
         assert replicaShard.active() : "non-active shard cannot be promoted to primary: " + replicaShard;
         assert replicaShard.primary() == false : "primary shard cannot be promoted to primary: " + replicaShard;
-        ShardRouting primaryShard = replicaShard.moveToPrimary();
+        ShardRouting primaryShard = replicaShard.moveActiveReplicaToPrimary();
         updateAssigned(replicaShard, primaryShard);
         return primaryShard;
     }
@@ -701,7 +702,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         if (candidate.relocating()) {
             cancelRelocation(candidate);
         }
-        ShardRouting reinitializedShard = candidate.reinitializeShard();
+        ShardRouting reinitializedShard = candidate.reinitializePrimaryShard();
         updateAssigned(candidate, reinitializedShard);
         inactivePrimaryCount++;
         inactiveShardCount++;
@@ -738,7 +739,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         assert shard.unassigned() == false : "only assigned shards can be moved to unassigned (" + shard + ")";
         assert shard.primary() : "only primary can be demoted to replica (" + shard + ")";
         remove(shard);
-        ShardRouting unassigned = shard.moveToUnassigned(unassignedInfo).moveFromPrimary();
+        ShardRouting unassigned = shard.moveToUnassigned(unassignedInfo).moveUnassignedFromPrimary();
         unassignedShards.add(unassigned);
         return unassigned;
     }
@@ -832,7 +833,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                                                                 currInfo.getNumFailedAllocations(), currInfo.getUnassignedTimeInNanos(),
                                                                 currInfo.getUnassignedTimeInMillis(), currInfo.isDelayed(),
                                                                 allocationStatus);
-                    ShardRouting updatedShard = shard.updateUnassignedInfo(newInfo);
+                    ShardRouting updatedShard = shard.updateUnassigned(newInfo, shard.recoverySource());
                     changes.unassignedInfoUpdated(shard, newInfo);
                     shard = updatedShard;
                 }
@@ -891,14 +892,16 @@ public class RoutingNodes implements Iterable<RoutingNode> {
             }
 
             /**
-             * updates the unassigned info on the current unassigned shard
+             * updates the unassigned info and recovery source on the current unassigned shard
              *
              * @param  unassignedInfo the new unassigned info to use
+             * @param  recoverySource the new recovery source to use
              * @return the shard with unassigned info updated
              */
-            public ShardRouting updateUnassignedInfo(UnassignedInfo unassignedInfo, RoutingChangesObserver changes) {
+            public ShardRouting updateUnassigned(UnassignedInfo unassignedInfo, RecoverySource recoverySource,
+                                                 RoutingChangesObserver changes) {
                 nodes.ensureMutable();
-                ShardRouting updatedShardRouting = current.updateUnassignedInfo(unassignedInfo);
+                ShardRouting updatedShardRouting = current.updateUnassigned(unassignedInfo, recoverySource);
                 changes.unassignedInfoUpdated(current, unassignedInfo);
                 updateShardRouting(updatedShardRouting);
                 return updatedShardRouting;
@@ -1040,9 +1043,9 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                     if (routing.initializing()) {
                         incoming++;
                     }
-                    if (routing.primary() && routing.isPeerRecovery() == false) {
+                    if (routing.primary() && routing.isRelocationTarget() == false) {
                         for (ShardRouting assigned : routingNodes.assignedShards.get(routing.shardId())) {
-                            if (assigned.isPeerRecovery()) {
+                            if (assigned.initializing() && assigned.recoverySource().getType() == RecoverySource.Type.PEER) {
                                 outgoing++;
                             }
                         }

core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java

@@ -28,6 +28,7 @@ import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -540,16 +541,16 @@ public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<Routi
             return this;
         }
 
-        public Builder addAsRestore(IndexMetaData indexMetaData, RestoreSource restoreSource) {
+        public Builder addAsRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource) {
             IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex())
-                    .initializeAsRestore(indexMetaData, restoreSource);
+                    .initializeAsRestore(indexMetaData, recoverySource);
             add(indexRoutingBuilder);
             return this;
         }
 
-        public Builder addAsNewRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards) {
+        public Builder addAsNewRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards) {
             IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex())
-                    .initializeAsNewRestore(indexMetaData, restoreSource, ignoreShards);
+                    .initializeAsNewRestore(indexMetaData, recoverySource, ignoreShards);
             add(indexRoutingBuilder);
             return this;
         }

core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java

@@ -19,8 +19,8 @@
 
 package org.elasticsearch.cluster.routing;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -51,7 +51,7 @@ public final class ShardRouting implements Writeable, ToXContent {
     private final String relocatingNodeId;
     private final boolean primary;
     private final ShardRoutingState state;
-    private final RestoreSource restoreSource;
+    private final RecoverySource recoverySource;
     private final UnassignedInfo unassignedInfo;
     private final AllocationId allocationId;
     private final transient List<ShardRouting> asList;
@@ -64,29 +64,31 @@ public final class ShardRouting implements Writeable, ToXContent {
      * by either this class or tests. Visible for testing.
      */
     ShardRouting(ShardId shardId, String currentNodeId,
-                 String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state,
+                 String relocatingNodeId, boolean primary, ShardRoutingState state, RecoverySource recoverySource,
                  UnassignedInfo unassignedInfo, AllocationId allocationId, long expectedShardSize) {
         this.shardId = shardId;
         this.currentNodeId = currentNodeId;
         this.relocatingNodeId = relocatingNodeId;
         this.primary = primary;
         this.state = state;
-        this.asList = Collections.singletonList(this);
-        this.restoreSource = restoreSource;
+        this.recoverySource = recoverySource;
         this.unassignedInfo = unassignedInfo;
         this.allocationId = allocationId;
         this.expectedShardSize = expectedShardSize;
         this.targetRelocatingShard = initializeTargetRelocatingShard();
+        this.asList = Collections.singletonList(this);
         assert expectedShardSize == UNAVAILABLE_EXPECTED_SHARD_SIZE || state == ShardRoutingState.INITIALIZING || state == ShardRoutingState.RELOCATING : expectedShardSize + " state: " + state;
         assert expectedShardSize >= 0 || state != ShardRoutingState.INITIALIZING || state != ShardRoutingState.RELOCATING : expectedShardSize + " state: " + state;
         assert !(state == ShardRoutingState.UNASSIGNED && unassignedInfo == null) : "unassigned shard must be created with meta";
+        assert (state == ShardRoutingState.UNASSIGNED || state == ShardRoutingState.INITIALIZING) == (recoverySource != null) : "recovery source only available on unassigned or initializing shard but was " + state;
+        assert recoverySource == null || recoverySource == PeerRecoverySource.INSTANCE || primary : "replica shards always recover from primary";
     }
 
     @Nullable
     private ShardRouting initializeTargetRelocatingShard() {
         if (state == ShardRoutingState.RELOCATING) {
-            return new ShardRouting(shardId, relocatingNodeId, currentNodeId, restoreSource, primary,
-                ShardRoutingState.INITIALIZING, unassignedInfo, AllocationId.newTargetRelocation(allocationId), expectedShardSize);
+            return new ShardRouting(shardId, relocatingNodeId, currentNodeId, primary, ShardRoutingState.INITIALIZING,
+                PeerRecoverySource.INSTANCE, unassignedInfo, AllocationId.newTargetRelocation(allocationId), expectedShardSize);
         } else {
             return null;
         }
@@ -95,8 +97,8 @@ public final class ShardRouting implements Writeable, ToXContent {
     /**
      * Creates a new unassigned shard.
      */
-    public static ShardRouting newUnassigned(ShardId shardId, RestoreSource restoreSource, boolean primary, UnassignedInfo unassignedInfo) {
-        return new ShardRouting(shardId, null, null, restoreSource, primary, ShardRoutingState.UNASSIGNED, unassignedInfo, null, UNAVAILABLE_EXPECTED_SHARD_SIZE);
+    public static ShardRouting newUnassigned(ShardId shardId, boolean primary, RecoverySource recoverySource, UnassignedInfo unassignedInfo) {
+        return new ShardRouting(shardId, null, null, primary, ShardRoutingState.UNASSIGNED, recoverySource, unassignedInfo, null, UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
     public Index index() {
@@ -199,13 +201,6 @@ public final class ShardRouting implements Writeable, ToXContent {
         return targetRelocatingShard;
     }
 
-    /**
-     * Snapshot id and repository where this shard is being restored from
-     */
-    public RestoreSource restoreSource() {
-        return restoreSource;
-    }
-
     /**
      * Additional metadata on why the shard is/was unassigned. The metadata is kept around
      * until the shard moves to STARTED.
@@ -244,32 +239,6 @@ public final class ShardRouting implements Writeable, ToXContent {
         return shardId;
     }
 
-    public boolean allocatedPostIndexCreate(IndexMetaData indexMetaData) {
-        if (active()) {
-            return true;
-        }
-
-        // initializing replica might not have unassignedInfo
-        assert unassignedInfo != null || (primary == false && state == ShardRoutingState.INITIALIZING);
-        if (unassignedInfo != null && unassignedInfo.getReason() == UnassignedInfo.Reason.INDEX_CREATED) {
-            return false;
-        }
-
-        if (indexMetaData.inSyncAllocationIds(id()).isEmpty() && indexMetaData.getCreationVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
-            // when no shards with this id have ever been active for this index
-            return false;
-        }
-
-        return true;
-    }
-
-    /**
-     * returns true for initializing shards that recover their data from another shard copy
-     */
-    public boolean isPeerRecovery() {
-        return state == ShardRoutingState.INITIALIZING && (primary() == false || relocatingNodeId != null);
-    }
-
     /**
      * A shard iterator with just this shard in it.
      */
@@ -283,7 +252,11 @@ public final class ShardRouting implements Writeable, ToXContent {
         relocatingNodeId = in.readOptionalString();
         primary = in.readBoolean();
         state = ShardRoutingState.fromValue(in.readByte());
-        restoreSource = RestoreSource.readOptionalRestoreSource(in);
+        if (state == ShardRoutingState.UNASSIGNED || state == ShardRoutingState.INITIALIZING) {
+            recoverySource = RecoverySource.readFrom(in);
+        } else {
+            recoverySource = null;
+        }
         unassignedInfo = in.readOptionalWriteable(UnassignedInfo::new);
         allocationId = in.readOptionalWriteable(AllocationId::new);
         final long shardSize;
@@ -312,7 +285,9 @@ public final class ShardRouting implements Writeable, ToXContent {
         out.writeOptionalString(relocatingNodeId);
         out.writeBoolean(primary);
         out.writeByte(state.value());
-        out.writeOptionalStreamable(restoreSource);
+        if (state == ShardRoutingState.UNASSIGNED || state == ShardRoutingState.INITIALIZING) {
+            recoverySource.writeTo(out);
+        }
         out.writeOptionalWriteable(unassignedInfo);
         out.writeOptionalWriteable(allocationId);
         if (state == ShardRoutingState.RELOCATING || state == ShardRoutingState.INITIALIZING) {
@@ -326,10 +301,10 @@ public final class ShardRouting implements Writeable, ToXContent {
         writeToThin(out);
     }
 
-    public ShardRouting updateUnassignedInfo(UnassignedInfo unassignedInfo) {
+    public ShardRouting updateUnassigned(UnassignedInfo unassignedInfo, RecoverySource recoverySource) {
         assert this.unassignedInfo != null : "can only update unassign info if they are already set";
         assert this.unassignedInfo.isDelayed() || (unassignedInfo.isDelayed() == false) : "cannot transition from non-delayed to delayed";
-        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state,
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, primary, state, recoverySource,
             unassignedInfo, allocationId, expectedShardSize);
     }
 
@@ -338,7 +313,17 @@ public final class ShardRouting implements Writeable, ToXContent {
      */
     public ShardRouting moveToUnassigned(UnassignedInfo unassignedInfo) {
         assert state != ShardRoutingState.UNASSIGNED : this;
-        return new ShardRouting(shardId, null, null, restoreSource, primary, ShardRoutingState.UNASSIGNED,
+        final RecoverySource recoverySource;
+        if (active()) {
+            if (primary()) {
+                recoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE;
+            } else {
+                recoverySource = PeerRecoverySource.INSTANCE;
+            }
+        } else {
+            recoverySource = recoverySource();
+        }
+        return new ShardRouting(shardId, null, null, primary, ShardRoutingState.UNASSIGNED, recoverySource,
             unassignedInfo, null, UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
@@ -356,7 +341,7 @@ public final class ShardRouting implements Writeable, ToXContent {
         } else {
             allocationId = AllocationId.newInitializing(existingAllocationId);
         }
-        return new ShardRouting(shardId, nodeId, null, restoreSource, primary, ShardRoutingState.INITIALIZING,
+        return new ShardRouting(shardId, nodeId, null, primary, ShardRoutingState.INITIALIZING, recoverySource,
             unassignedInfo, allocationId, expectedShardSize);
     }
 
@@ -367,7 +352,7 @@ public final class ShardRouting implements Writeable, ToXContent {
      */
     public ShardRouting relocate(String relocatingNodeId, long expectedShardSize) {
         assert state == ShardRoutingState.STARTED : "current shard has to be started in order to be relocated " + this;
-        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, primary, ShardRoutingState.RELOCATING,
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, primary, ShardRoutingState.RELOCATING, recoverySource,
             null, AllocationId.newRelocation(allocationId), expectedShardSize);
     }
 
@@ -379,7 +364,7 @@ public final class ShardRouting implements Writeable, ToXContent {
         assert state == ShardRoutingState.RELOCATING : this;
         assert assignedToNode() : this;
         assert relocatingNodeId != null : this;
-        return new ShardRouting(shardId, currentNodeId, null, restoreSource, primary, ShardRoutingState.STARTED,
+        return new ShardRouting(shardId, currentNodeId, null, primary, ShardRoutingState.STARTED, recoverySource,
             null, AllocationId.cancelRelocation(allocationId), UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
@@ -393,17 +378,19 @@ public final class ShardRouting implements Writeable, ToXContent {
         assert state == ShardRoutingState.INITIALIZING : this;
         assert assignedToNode() : this;
         assert relocatingNodeId != null : this;
-        return new ShardRouting(shardId, currentNodeId, null, restoreSource, primary, state, unassignedInfo,
+        return new ShardRouting(shardId, currentNodeId, null, primary, state, recoverySource, unassignedInfo,
             AllocationId.finishRelocation(allocationId), expectedShardSize);
     }
 
     /**
-     * Moves the shard from started to initializing
+     * Moves the primary shard from started to initializing
      */
-    public ShardRouting reinitializeShard() {
-        assert state == ShardRoutingState.STARTED;
-        return new ShardRouting(shardId, currentNodeId, null, restoreSource, primary, ShardRoutingState.INITIALIZING,
-            new UnassignedInfo(UnassignedInfo.Reason.REINITIALIZED, null), AllocationId.newInitializing(), UNAVAILABLE_EXPECTED_SHARD_SIZE);
+    public ShardRouting reinitializePrimaryShard() {
+        assert state == ShardRoutingState.STARTED : this;
+        assert primary : this;
+        return new ShardRouting(shardId, currentNodeId, null, primary, ShardRoutingState.INITIALIZING,
+            StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.REINITIALIZED, null),
+            AllocationId.newInitializing(), UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
     /**
@@ -418,39 +405,36 @@ public final class ShardRouting implements Writeable, ToXContent {
             // relocation target
             allocationId = AllocationId.finishRelocation(allocationId);
         }
-        return new ShardRouting(shardId, currentNodeId, null, restoreSource, primary, ShardRoutingState.STARTED, null, allocationId,
+        return new ShardRouting(shardId, currentNodeId, null, primary, ShardRoutingState.STARTED, null, null, allocationId,
             UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
     /**
-     * Make the shard primary unless it's not Primary
+     * Make the active shard primary unless it's not primary
      *
      * @throws IllegalShardRoutingStateException if shard is already a primary
      */
-    public ShardRouting moveToPrimary() {
+    public ShardRouting moveActiveReplicaToPrimary() {
+        assert active(): "expected an active shard " + this;
         if (primary) {
             throw new IllegalShardRoutingStateException(this, "Already primary, can't move to primary");
         }
-        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, true, state, unassignedInfo, allocationId,
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, true, state, recoverySource, unassignedInfo, allocationId,
             expectedShardSize);
     }
 
     /**
-     * Set the primary shard to non-primary
+     * Set the unassigned primary shard to non-primary
      *
      * @throws IllegalShardRoutingStateException if shard is already a replica
      */
-    public ShardRouting moveFromPrimary() {
+    public ShardRouting moveUnassignedFromPrimary() {
+        assert state == ShardRoutingState.UNASSIGNED : "expected an unassigned shard " + this;
         if (!primary) {
             throw new IllegalShardRoutingStateException(this, "Not primary, can't move to replica");
         }
-        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, false, state, unassignedInfo, allocationId,
-            expectedShardSize);
-    }
-
-    /** returns true if this routing has the same shardId as another */
-    public boolean isSameShard(ShardRouting other) {
-        return getIndexName().equals(other.getIndexName()) && id() == other.id();
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, false, state, PeerRecoverySource.INSTANCE, unassignedInfo,
+            allocationId, expectedShardSize);
     }
 
     /**
@@ -490,8 +474,8 @@ public final class ShardRouting implements Writeable, ToXContent {
         assert b == false || this.currentNodeId().equals(other.relocatingNodeId) :
             "ShardRouting is a relocation target but current node id isn't equal to source relocating node. This [" + this + "], other [" + other + "]";
 
-        assert b == false || isSameShard(other) :
-            "ShardRouting is a relocation target but both routings are not of the same shard. This [" + this + "], other [" + other + "]";
+        assert b == false || this.shardId.equals(other.shardId) :
+            "ShardRouting is a relocation target but both routings are not of the same shard id. This [" + this + "], other [" + other + "]";
 
         assert b == false || this.primary == other.primary :
             "ShardRouting is a relocation target but primary flag is different. This [" + this + "], target [" + other + "]";
@@ -517,7 +501,7 @@ public final class ShardRouting implements Writeable, ToXContent {
         assert b == false || other.currentNodeId().equals(this.relocatingNodeId) :
             "ShardRouting is a relocation source but relocating node isn't equal to other's current node. This [" + this + "], other [" + other + "]";
 
-        assert b == false || isSameShard(other) :
+        assert b == false || this.shardId.equals(other.shardId) :
             "ShardRouting is a relocation source but both routings are not of the same shard. This [" + this + "], target [" + other + "]";
 
         assert b == false || this.primary == other.primary :
@@ -526,7 +510,7 @@ public final class ShardRouting implements Writeable, ToXContent {
         return b;
     }
 
-    /** returns true if the current routing is identical to the other routing in all but meta fields, i.e., version and unassigned info */
+    /** returns true if the current routing is identical to the other routing in all but meta fields, i.e., unassigned info */
     public boolean equalsIgnoringMetaData(ShardRouting other) {
         if (primary != other.primary) {
             return false;
@@ -546,7 +530,7 @@ public final class ShardRouting implements Writeable, ToXContent {
         if (state != other.state) {
             return false;
         }
-        if (restoreSource != null ? !restoreSource.equals(other.restoreSource) : other.restoreSource != null) {
+        if (recoverySource != null ? !recoverySource.equals(other.recoverySource) : other.recoverySource != null) {
             return false;
         }
         return true;
@@ -582,7 +566,7 @@ public final class ShardRouting implements Writeable, ToXContent {
             h = 31 * h + (relocatingNodeId != null ? relocatingNodeId.hashCode() : 0);
             h = 31 * h + (primary ? 1 : 0);
             h = 31 * h + (state != null ? state.hashCode() : 0);
-            h = 31 * h + (restoreSource != null ? restoreSource.hashCode() : 0);
+            h = 31 * h + (recoverySource != null ? recoverySource.hashCode() : 0);
             h = 31 * h + (allocationId != null ? allocationId.hashCode() : 0);
             h = 31 * h + (unassignedInfo != null ? unassignedInfo.hashCode() : 0);
             hashCode = h;
@@ -610,8 +594,8 @@ public final class ShardRouting implements Writeable, ToXContent {
         } else {
             sb.append("[R]");
         }
-        if (this.restoreSource != null) {
-            sb.append(", restoring[" + restoreSource + "]");
+        if (recoverySource != null) {
+            sb.append(", recovery_source[").append(recoverySource).append("]");
         }
         sb.append(", s[").append(state).append("]");
         if (allocationId != null) {
@@ -638,9 +622,8 @@ public final class ShardRouting implements Writeable, ToXContent {
         if (expectedShardSize != UNAVAILABLE_EXPECTED_SHARD_SIZE) {
             builder.field("expected_shard_size_in_bytes", expectedShardSize);
         }
-        if (restoreSource() != null) {
-            builder.field("restore_source");
-            restoreSource().toXContent(builder, params);
+        if (recoverySource != null) {
+            builder.field("recovery_source", recoverySource);
         }
         if (allocationId != null) {
             builder.field("allocation_id");
@@ -659,4 +642,14 @@ public final class ShardRouting implements Writeable, ToXContent {
     public long getExpectedShardSize() {
         return expectedShardSize;
     }
+
+    /**
+     * Returns recovery source for the given shard. Replica shards always recover from the primary {@link PeerRecoverySource}.
+     *
+     * @return recovery source or null if shard is {@link #active()}
+     */
+    @Nullable
+    public RecoverySource recoverySource() {
+        return recoverySource;
+    }
 }

core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java

@@ -108,7 +108,11 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         /**
          * Unassigned as a result of a failed primary while the replica was initializing.
          */
-        PRIMARY_FAILED
+        PRIMARY_FAILED,
+        /**
+         * Unassigned after forcing an empty primary
+         */
+        FORCED_EMPTY_PRIMARY
     }
 
     /**

core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java

@@ -218,9 +218,10 @@ public class AllocationService extends AbstractComponent {
                 final long newComputedLeftDelayNanos = unassignedInfo.getRemainingDelay(allocation.getCurrentNanoTime(),
                     metaData.getIndexSafe(shardRouting.index()).getSettings());
                 if (newComputedLeftDelayNanos == 0) {
-                    unassignedIterator.updateUnassignedInfo(new UnassignedInfo(unassignedInfo.getReason(), unassignedInfo.getMessage(),
+                    unassignedIterator.updateUnassigned(new UnassignedInfo(unassignedInfo.getReason(), unassignedInfo.getMessage(),
                         unassignedInfo.getFailure(), unassignedInfo.getNumFailedAllocations(), unassignedInfo.getUnassignedTimeInNanos(),
-                        unassignedInfo.getUnassignedTimeInMillis(), false, unassignedInfo.getLastAllocationStatus()), allocation.changes());
+                        unassignedInfo.getUnassignedTimeInMillis(), false, unassignedInfo.getLastAllocationStatus()),
+                        shardRouting.recoverySource(), allocation.changes());
                 }
             }
         }

core/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetaDataUpdater.java

@@ -22,6 +22,7 @@ package org.elasticsearch.cluster.routing.allocation;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingChangesObserver;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -154,7 +155,8 @@ public class IndexMetaDataUpdater extends RoutingChangesObserver.AbstractRouting
             oldInSyncAllocationIds.contains(updates.initializedPrimary.allocationId().getId()) == false) {
             // we're not reusing an existing in-sync allocation id to initialize a primary, which means that we're either force-allocating
             // an empty or a stale primary (see AllocateEmptyPrimaryAllocationCommand or AllocateStalePrimaryAllocationCommand).
-            boolean emptyPrimary = updates.initializedPrimary.unassignedInfo().getReason() == UnassignedInfo.Reason.INDEX_CREATED;
+            RecoverySource.Type recoverySourceType = updates.initializedPrimary.recoverySource().getType();
+            boolean emptyPrimary = recoverySourceType == RecoverySource.Type.EMPTY_STORE;
             assert updates.addedAllocationIds.isEmpty() : (emptyPrimary ? "empty" : "stale") +
                 " primary is not force-initialized in same allocation round where shards are started";
 

core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster.routing.allocation.command;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -187,7 +188,7 @@ public abstract class AbstractAllocateAllocationCommand implements AllocationCom
      * @param shardRouting the shard routing that is to be matched in unassigned shards
      */
     protected void initializeUnassignedShard(RoutingAllocation allocation, RoutingNodes routingNodes, RoutingNode routingNode, ShardRouting shardRouting) {
-        initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, null);
+        initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, null, null);
     }
 
     /**
@@ -198,16 +199,19 @@ public abstract class AbstractAllocateAllocationCommand implements AllocationCom
      * @param routingNode the node to initialize it to
      * @param shardRouting the shard routing that is to be matched in unassigned shards
      * @param unassignedInfo unassigned info to override
+     * @param recoverySource recovery source to override
      */
     protected void initializeUnassignedShard(RoutingAllocation allocation, RoutingNodes routingNodes, RoutingNode routingNode,
-                                             ShardRouting shardRouting, @Nullable UnassignedInfo unassignedInfo) {
+                                             ShardRouting shardRouting, @Nullable UnassignedInfo unassignedInfo,
+                                             @Nullable RecoverySource recoverySource) {
         for (RoutingNodes.UnassignedShards.UnassignedIterator it = routingNodes.unassigned().iterator(); it.hasNext(); ) {
             ShardRouting unassigned = it.next();
             if (!unassigned.equalsIgnoringMetaData(shardRouting)) {
                 continue;
             }
-            if (unassignedInfo != null) {
-                unassigned = it.updateUnassignedInfo(unassignedInfo, allocation.changes());
+            if (unassignedInfo != null || recoverySource != null) {
+                unassigned = it.updateUnassigned(unassignedInfo != null ? unassignedInfo : unassigned.unassignedInfo(),
+                    recoverySource != null ? recoverySource : unassigned.recoverySource(), allocation.changes());
             }
             it.initialize(routingNode.nodeId(), null, allocation.clusterInfo().getShardSize(unassigned, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE), allocation.changes());
             return;

core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java

@@ -20,6 +20,8 @@
 package org.elasticsearch.cluster.routing.allocation.command;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -115,21 +117,20 @@ public class AllocateEmptyPrimaryAllocationCommand extends BasePrimaryAllocation
             return explainOrThrowRejectedCommand(explain, allocation, "primary [" + index + "][" + shardId + "] is already assigned");
         }
 
-        if (shardRouting.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED && acceptDataLoss == false) {
+        if (shardRouting.recoverySource().getType() != RecoverySource.Type.EMPTY_STORE && acceptDataLoss == false) {
             return explainOrThrowRejectedCommand(explain, allocation,
                 "allocating an empty primary for [" + index + "][" + shardId + "] can result in data loss. Please confirm by setting the accept_data_loss parameter to true");
         }
 
         UnassignedInfo unassignedInfoToUpdate = null;
-        if (shardRouting.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED) {
-            // we need to move the unassigned info back to treat it as if it was index creation
-            unassignedInfoToUpdate = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED,
+        if (shardRouting.unassignedInfo().getReason() != UnassignedInfo.Reason.FORCED_EMPTY_PRIMARY) {
+            unassignedInfoToUpdate = new UnassignedInfo(UnassignedInfo.Reason.FORCED_EMPTY_PRIMARY,
                 "force empty allocation from previous reason " + shardRouting.unassignedInfo().getReason() + ", " + shardRouting.unassignedInfo().getMessage(),
                 shardRouting.unassignedInfo().getFailure(), 0, System.nanoTime(), System.currentTimeMillis(), false,
                 shardRouting.unassignedInfo().getLastAllocationStatus());
         }
 
-        initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, unassignedInfoToUpdate);
+        initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, unassignedInfoToUpdate, StoreRecoverySource.EMPTY_STORE_INSTANCE);
 
         return new RerouteExplanation(this, allocation.decision(Decision.YES, name() + " (allocation command)", "ignore deciders"));
     }

core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java

@@ -19,8 +19,8 @@
 
 package org.elasticsearch.cluster.routing.allocation.command;
 
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -120,8 +120,7 @@ public class AllocateStalePrimaryAllocationCommand extends BasePrimaryAllocation
                 "allocating an empty primary for [" + index + "][" + shardId + "] can result in data loss. Please confirm by setting the accept_data_loss parameter to true");
         }
 
-        final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
-        if (shardRouting.allocatedPostIndexCreate(indexMetaData) == false) {
+        if (shardRouting.recoverySource().getType() != RecoverySource.Type.EXISTING_STORE) {
             return explainOrThrowRejectedCommand(explain, allocation,
                 "trying to allocate an existing primary shard [" + index + "][" + shardId + "], while no such shard has ever been active");
         }

core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java

@@ -26,6 +26,7 @@ import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.DiskUsage;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -120,14 +121,14 @@ public class DiskThresholdDecider extends AllocationDecider {
             logger.trace("node [{}] has {}% used disk", node.nodeId(), usedDiskPercentage);
         }
 
-        // a flag for whether the primary shard has been previously allocated
-        IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
-        boolean primaryHasBeenAllocated = shardRouting.primary() && shardRouting.allocatedPostIndexCreate(indexMetaData);
+        // flag that determines whether the low threshold checks below can be skipped. We use this for a primary shard that is freshly
+        // allocated and empty.
+        boolean skipLowTresholdChecks = shardRouting.primary() &&
+            shardRouting.active() == false && shardRouting.recoverySource().getType() == RecoverySource.Type.EMPTY_STORE;
 
         // checks for exact byte comparisons
         if (freeBytes < diskThresholdSettings.getFreeBytesThresholdLow().bytes()) {
-            // If the shard is a replica or has a primary that has already been allocated before, check the low threshold
-            if (!shardRouting.primary() || (shardRouting.primary() && primaryHasBeenAllocated)) {
+            if (skipLowTresholdChecks == false) {
                 if (logger.isDebugEnabled()) {
                     logger.debug("less than the required {} free bytes threshold ({} bytes free) on node {}, preventing allocation",
                             diskThresholdSettings.getFreeBytesThresholdLow(), freeBytes, node.nodeId());
@@ -162,8 +163,8 @@ public class DiskThresholdDecider extends AllocationDecider {
 
         // checks for percentage comparisons
         if (freeDiskPercentage < diskThresholdSettings.getFreeDiskThresholdLow()) {
-            // If the shard is a replica or has a primary that has already been allocated before, check the low threshold
-            if (!shardRouting.primary() || (shardRouting.primary() && primaryHasBeenAllocated)) {
+            // If the shard is a replica or is a non-empty primary, check the low threshold
+            if (skipLowTresholdChecks == false) {
                 if (logger.isDebugEnabled()) {
                     logger.debug("more than the allowed {} used disk threshold ({} used) on node [{}], preventing allocation",
                             Strings.format1Decimals(usedDiskThresholdLow, "%"),
@@ -378,12 +379,13 @@ public class DiskThresholdDecider extends AllocationDecider {
     public static long getExpectedShardSize(ShardRouting shard, RoutingAllocation allocation, long defaultValue) {
         final IndexMetaData metaData = allocation.metaData().getIndexSafe(shard.index());
         final ClusterInfo info = allocation.clusterInfo();
-        if (metaData.getMergeSourceIndex() != null && shard.allocatedPostIndexCreate(metaData) == false) {
+        if (metaData.getMergeSourceIndex() != null && shard.active() == false &&
+            shard.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) {
             // in the shrink index case we sum up the source index shards since we basically make a copy of the shard in
             // the worst case
             long targetShardSize = 0;
             final Index mergeSourceIndex = metaData.getMergeSourceIndex();
-            final IndexMetaData sourceIndexMeta = allocation.metaData().getIndexSafe(metaData.getMergeSourceIndex());
+            final IndexMetaData sourceIndexMeta = allocation.metaData().getIndexSafe(mergeSourceIndex);
             final Set<ShardId> shardIds = IndexMetaData.selectShrinkShards(shard.id(), sourceIndexMeta, metaData.getNumberOfShards());
             for (IndexShardRoutingTable shardRoutingTable : allocation.routingTable().index(mergeSourceIndex.getName())) {
                 if (shardIds.contains(shardRoutingTable.shardId())) {

core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java

@@ -22,6 +22,7 @@ package org.elasticsearch.cluster.routing.allocation.decider;
 import java.util.Locale;
 
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@@ -113,7 +114,8 @@ public class EnableAllocationDecider extends AllocationDecider {
             case NONE:
                 return allocation.decision(Decision.NO, NAME, "no allocations are allowed");
             case NEW_PRIMARIES:
-                if (shardRouting.primary() && shardRouting.allocatedPostIndexCreate(indexMetaData) == false) {
+                if (shardRouting.primary() && shardRouting.active() == false &&
+                    shardRouting.recoverySource().getType() != RecoverySource.Type.EXISTING_STORE) {
                     return allocation.decision(Decision.YES, NAME, "new primary allocations are allowed");
                 } else {
                     return allocation.decision(Decision.NO, NAME, "non-new primary allocations are forbidden");

core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java

@@ -21,6 +21,7 @@ package org.elasticsearch.cluster.routing.allocation.decider;
 
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodeFilters;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@@ -92,7 +93,7 @@ public class FilterAllocationDecider extends AllocationDecider {
             // this is a setting that can only be set within the system!
             IndexMetaData indexMd = allocation.metaData().getIndexSafe(shardRouting.index());
             DiscoveryNodeFilters initialRecoveryFilters = indexMd.getInitialRecoveryFilters();
-            if (shardRouting.allocatedPostIndexCreate(indexMd) == false &&
+            if (shardRouting.recoverySource().getType() != RecoverySource.Type.EXISTING_STORE &&
                 initialRecoveryFilters != null &&
                 initialRecoveryFilters.match(node.node()) == false) {
                 return allocation.decision(Decision.NO, NAME, "node does not match index initial recovery filters [%s]",

core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java

@@ -19,7 +19,8 @@
 
 package org.elasticsearch.cluster.routing.allocation.decider;
 
-import org.elasticsearch.cluster.routing.RestoreSource;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -45,12 +46,12 @@ public class NodeVersionAllocationDecider extends AllocationDecider {
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
         if (shardRouting.primary()) {
             if (shardRouting.currentNodeId() == null) {
-                if (shardRouting.restoreSource() != null) {
+                if (shardRouting.recoverySource() != null && shardRouting.recoverySource().getType() == RecoverySource.Type.SNAPSHOT) {
                     // restoring from a snapshot - check that the node can handle the version
-                    return isVersionCompatible(shardRouting.restoreSource(), node, allocation);
+                    return isVersionCompatible((SnapshotRecoverySource)shardRouting.recoverySource(), node, allocation);
                 } else {
-                    // fresh primary, we can allocate wherever
-                    return allocation.decision(Decision.YES, NAME, "the primary shard is new and can be allocated anywhere");
+                    // existing or fresh primary on the node
+                    return allocation.decision(Decision.YES, NAME, "the primary shard is new or already existed on the node");
                 }
             } else {
                 // relocating primary, only migrate to newer host
@@ -83,14 +84,14 @@ public class NodeVersionAllocationDecider extends AllocationDecider {
         }
     }
 
-    private Decision isVersionCompatible(RestoreSource restoreSource, final RoutingNode target, RoutingAllocation allocation) {
-        if (target.node().getVersion().onOrAfter(restoreSource.version())) {
+    private Decision isVersionCompatible(SnapshotRecoverySource recoverySource, final RoutingNode target, RoutingAllocation allocation) {
+        if (target.node().getVersion().onOrAfter(recoverySource.version())) {
             /* we can allocate if we can restore from a snapshot that is older or on the same version */
             return allocation.decision(Decision.YES, NAME, "target node version [%s] is the same or newer than snapshot version [%s]",
-                target.node().getVersion(), restoreSource.version());
+                target.node().getVersion(), recoverySource.version());
         } else {
             return allocation.decision(Decision.NO, NAME, "target node version [%s] is older than the snapshot version [%s]",
-                target.node().getVersion(), restoreSource.version());
+                target.node().getVersion(), recoverySource.version());
         }
     }
 }

core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.cluster.routing.allocation.decider;
 
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -111,7 +112,7 @@ public class ThrottlingAllocationDecider extends AllocationDecider {
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
         if (shardRouting.primary() && shardRouting.unassigned()) {
-            assert initializingShard(shardRouting, node.nodeId()).isPeerRecovery() == false;
+            assert initializingShard(shardRouting, node.nodeId()).recoverySource().getType() != RecoverySource.Type.PEER;
             // primary is unassigned, means we are going to do recovery from store, snapshot or local shards
             // count *just the primaries* currently doing recovery on the node and check against primariesInitialRecoveries
 
@@ -132,7 +133,7 @@ public class ThrottlingAllocationDecider extends AllocationDecider {
             }
         } else {
             // Peer recovery
-            assert initializingShard(shardRouting, node.nodeId()).isPeerRecovery();
+            assert initializingShard(shardRouting, node.nodeId()).recoverySource().getType() == RecoverySource.Type.PEER;
 
             // Allocating a shard to this node will increase the incoming recoveries
             int currentInRecoveries = allocation.routingNodes().getIncomingRecoveries(node.nodeId());

core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java

@@ -24,15 +24,14 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -105,11 +104,8 @@ public abstract class PrimaryShardAllocator extends AbstractComponent {
                 continue;
             }
 
-            final IndexMetaData indexMetaData = metaData.getIndexSafe(shard.index());
-            // don't go wild here and create a new IndexSetting object for every shard this could cause a lot of garbage
-            // on cluster restart if we allocate a boat load of shards
-            if (shard.allocatedPostIndexCreate(indexMetaData) == false) {
-                // when we create a fresh index
+            if (shard.recoverySource().getType() != RecoverySource.Type.EXISTING_STORE &&
+                shard.recoverySource().getType() != RecoverySource.Type.SNAPSHOT) {
                 continue;
             }
 
@@ -121,14 +117,17 @@ public abstract class PrimaryShardAllocator extends AbstractComponent {
                 continue;
             }
 
-            final Set<String> lastActiveAllocationIds = indexMetaData.inSyncAllocationIds(shard.id());
-            final boolean snapshotRestore = shard.restoreSource() != null;
+            // don't create a new IndexSetting object for every shard as this could cause a lot of garbage
+            // on cluster restart if we allocate a boat load of shards
+            final IndexMetaData indexMetaData = metaData.getIndexSafe(shard.index());
+            final Set<String> inSyncAllocationIds = indexMetaData.inSyncAllocationIds(shard.id());
+            final boolean snapshotRestore = shard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT;
             final boolean recoverOnAnyNode = recoverOnAnyNode(indexMetaData);
 
             final NodeShardsResult nodeShardsResult;
             final boolean enoughAllocationsFound;
 
-            if (lastActiveAllocationIds.isEmpty()) {
+            if (inSyncAllocationIds.isEmpty()) {
                 assert Version.indexCreated(indexMetaData.getSettings()).before(Version.V_5_0_0_alpha1) : "trying to allocated a primary with an empty allocation id set, but index is new";
                 // when we load an old index (after upgrading cluster) or restore a snapshot of an old index
                 // fall back to old version-based allocation mode
@@ -141,18 +140,18 @@ public abstract class PrimaryShardAllocator extends AbstractComponent {
                 }
                 logger.debug("[{}][{}]: version-based allocation for pre-{} index found {} allocations of {}", shard.index(), shard.id(), Version.V_5_0_0_alpha1, nodeShardsResult.allocationsFound, shard);
             } else {
-                assert lastActiveAllocationIds.isEmpty() == false;
+                assert inSyncAllocationIds.isEmpty() == false;
                 // use allocation ids to select nodes
                 nodeShardsResult = buildAllocationIdBasedNodeShardsResult(shard, snapshotRestore || recoverOnAnyNode,
-                        allocation.getIgnoreNodes(shard.shardId()), lastActiveAllocationIds, shardState);
+                        allocation.getIgnoreNodes(shard.shardId()), inSyncAllocationIds, shardState);
                 enoughAllocationsFound = nodeShardsResult.orderedAllocationCandidates.size() > 0;
-                logger.debug("[{}][{}]: found {} allocation candidates of {} based on allocation ids: [{}]", shard.index(), shard.id(), nodeShardsResult.orderedAllocationCandidates.size(), shard, lastActiveAllocationIds);
+                logger.debug("[{}][{}]: found {} allocation candidates of {} based on allocation ids: [{}]", shard.index(), shard.id(), nodeShardsResult.orderedAllocationCandidates.size(), shard, inSyncAllocationIds);
             }
 
             if (enoughAllocationsFound == false){
                 if (snapshotRestore) {
                     // let BalancedShardsAllocator take care of allocating this shard
-                    logger.debug("[{}][{}]: missing local data, will restore from [{}]", shard.index(), shard.id(), shard.restoreSource());
+                    logger.debug("[{}][{}]: missing local data, will restore from [{}]", shard.index(), shard.id(), shard.recoverySource());
                 } else if (recoverOnAnyNode) {
                     // let BalancedShardsAllocator take care of allocating this shard
                     logger.debug("[{}][{}]: missing local data, recover from any node", shard.index(), shard.id());

core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java

@@ -76,8 +76,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                 }
 
                 // if we are allocating a replica because of index creation, no need to go and find a copy, there isn't one...
-                IndexMetaData indexMetaData = metaData.getIndexSafe(shard.index());
-                if (shard.allocatedPostIndexCreate(indexMetaData) == false) {
+                if (shard.unassignedInfo() != null && shard.unassignedInfo().getReason() == UnassignedInfo.Reason.INDEX_CREATED) {
                     continue;
                 }
 
@@ -119,7 +118,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                             "existing allocation of replica to [" + currentNode + "] cancelled, sync id match found on node ["+ nodeWithHighestMatch + "]",
                             null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis(), false, UnassignedInfo.AllocationStatus.NO_ATTEMPT);
                         // don't cancel shard in the loop as it will cause a ConcurrentModificationException
-                        shardCancellationActions.add(() -> routingNodes.failShard(logger, shard, unassignedInfo, indexMetaData, allocation.changes()));
+                        shardCancellationActions.add(() -> routingNodes.failShard(logger, shard, unassignedInfo, metaData.getIndexSafe(shard.index()), allocation.changes()));
                     }
                 }
             }
@@ -132,7 +131,6 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
     public void allocateUnassigned(RoutingAllocation allocation) {
         final RoutingNodes routingNodes = allocation.routingNodes();
         final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = routingNodes.unassigned().iterator();
-        MetaData metaData = allocation.metaData();
         while (unassignedIterator.hasNext()) {
             ShardRouting shard = unassignedIterator.next();
             if (shard.primary()) {
@@ -140,8 +138,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
             }
 
             // if we are allocating a replica because of index creation, no need to go and find a copy, there isn't one...
-            IndexMetaData indexMetaData = metaData.getIndexSafe(shard.index());
-            if (shard.allocatedPostIndexCreate(indexMetaData) == false) {
+            if (shard.unassignedInfo().getReason() == UnassignedInfo.Reason.INDEX_CREATED) {
                 continue;
             }
 

core/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -43,6 +43,8 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.Booleans;
@@ -113,7 +115,7 @@ import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.search.suggest.completion.CompletionFieldStats;
@@ -1179,6 +1181,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     }
 
     public boolean recoverFromLocalShards(BiConsumer<String, MappingMetaData> mappingUpdateConsumer, List<IndexShard> localShards) throws IOException {
+        assert shardRouting.primary() : "recover from local shards only makes sense if the shard is a primary shard";
+        assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS : "invalid recovery type: " + recoveryState.getRecoverySource();
         final List<LocalShardSnapshot> snapshots = new ArrayList<>();
         try {
             for (IndexShard shard : localShards) {
@@ -1199,14 +1203,14 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         // we are the first primary, recover from the gateway
         // if its post api allocation, the index should exists
         assert shardRouting.primary() : "recover from store only makes sense if the shard is a primary shard";
-        boolean shouldExist = shardRouting.allocatedPostIndexCreate(indexSettings.getIndexMetaData());
-
+        assert shardRouting.initializing() : "can only start recovery on initializing shard";
         StoreRecovery storeRecovery = new StoreRecovery(shardId, logger);
-        return storeRecovery.recoverFromStore(this, shouldExist);
+        return storeRecovery.recoverFromStore(this);
     }
 
     public boolean restoreFromRepository(Repository repository) {
         assert shardRouting.primary() : "recover from store only makes sense if the shard is a primary shard";
+        assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.SNAPSHOT : "invalid recovery type: " + recoveryState.getRecoverySource();
         StoreRecovery storeRecovery = new StoreRecovery(shardId, logger);
         return storeRecovery.recoverFromRepository(this, repository);
     }
@@ -1422,26 +1426,57 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         return this.currentEngineReference.get();
     }
 
-    public void startRecovery(RecoveryState recoveryState, RecoveryTargetService recoveryTargetService,
-                              RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
+    public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService,
+                              PeerRecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
                               BiConsumer<String, MappingMetaData> mappingUpdateConsumer,
                               IndicesService indicesService) {
-        switch (recoveryState.getType()) {
-            case PRIMARY_RELOCATION:
-            case REPLICA:
+        // TODO: Create a proper object to encapsulate the recovery context
+        // all of the current methods here follow a pattern of:
+        // resolve context which isn't really dependent on the local shards and then async
+        // call some external method with this pointer.
+        // with a proper recovery context object we can simply change this to:
+        // startRecovery(RecoveryState recoveryState, ShardRecoverySource source ) {
+        //     markAsRecovery("from " + source.getShortDescription(), recoveryState);
+        //     threadPool.generic().execute()  {
+        //           onFailure () { listener.failure() };
+        //           doRun() {
+        //                if (source.recover(this)) {
+        //                  recoveryListener.onRecoveryDone(recoveryState);
+        //                }
+        //           }
+        //     }}
+        // }
+        assert recoveryState.getRecoverySource().equals(shardRouting.recoverySource());
+        switch (recoveryState.getRecoverySource().getType()) {
+            case EMPTY_STORE:
+            case EXISTING_STORE:
+                markAsRecovering("from store", recoveryState); // mark the shard as recovering on the cluster state thread
+                threadPool.generic().execute(() -> {
+                    try {
+                        if (recoverFromStore()) {
+                            recoveryListener.onRecoveryDone(recoveryState);
+                        }
+                    } catch (Exception e) {
+                        recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
+                    }
+                });
+                break;
+            case PEER:
                 try {
                     markAsRecovering("from " + recoveryState.getSourceNode(), recoveryState);
-                    recoveryTargetService.startRecovery(this, recoveryState.getType(), recoveryState.getSourceNode(), recoveryListener);
+                    recoveryTargetService.startRecovery(this, recoveryState.getSourceNode(), recoveryListener);
                 } catch (Exception e) {
                     failShard("corrupted preexisting index", e);
                     recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
                 }
                 break;
-            case STORE:
-                markAsRecovering("from store", recoveryState); // mark the shard as recovering on the cluster state thread
+            case SNAPSHOT:
+                markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread
+                SnapshotRecoverySource recoverySource = (SnapshotRecoverySource) recoveryState.getRecoverySource();
                 threadPool.generic().execute(() -> {
                     try {
-                        if (recoverFromStore()) {
+                        final Repository repository = repositoriesService.repository(recoverySource.snapshot().getRepository());
+                        if (restoreFromRepository(repository)) {
                             recoveryListener.onRecoveryDone(recoveryState);
                         }
                     } catch (Exception e) {
@@ -1467,7 +1502,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                     threadPool.generic().execute(() -> {
                         try {
                             final Set<ShardId> shards = IndexMetaData.selectShrinkShards(shardId().id(), sourceIndexService.getMetaData(),
-                                +                                indexMetaData.getNumberOfShards());
+                                +indexMetaData.getNumberOfShards());
                             if (recoverFromLocalShards(mappingUpdateConsumer, startedShards.stream()
                                 .filter((s) -> shards.contains(s.shardId())).collect(Collectors.toList()))) {
                                 recoveryListener.onRecoveryDone(recoveryState);
@@ -1489,22 +1524,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                     recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
                 }
                 break;
-            case SNAPSHOT:
-                markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread
-                threadPool.generic().execute(() -> {
-                    try {
-                        final Repository repository = repositoriesService.repository(
-                            recoveryState.getRestoreSource().snapshot().getRepository());
-                        if (restoreFromRepository(repository)) {
-                            recoveryListener.onRecoveryDone(recoveryState);
-                        }
-                    } catch (Exception first) {
-                        recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, first), true);
-                    }
-                });
-                break;
             default:
-                throw new IllegalArgumentException("Unknown recovery type " + recoveryState.getType());
+                throw new IllegalArgumentException("Unknown recovery source " + recoveryState.getRecoverySource());
         }
     }
 

core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java

@@ -31,8 +31,8 @@ import org.apache.lucene.store.IndexInput;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.cluster.routing.RestoreSource;
-import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -75,19 +75,18 @@ final class StoreRecovery {
      * exist on disk ie. has been previously allocated or if the shard is a brand new allocation without pre-existing index
      * files / transaction logs. This
      * @param indexShard the index shard instance to recovery the shard into
-     * @param indexShouldExists <code>true</code> iff the index should exist on disk ie. has the shard been allocated previously on the shards store.
      * @return  <code>true</code> if the shard has been recovered successfully, <code>false</code> if the recovery
      * has been ignored due to a concurrent modification of if the clusters state has changed due to async updates.
      * @see Store
      */
-    boolean recoverFromStore(final IndexShard indexShard, final boolean indexShouldExists) {
+    boolean recoverFromStore(final IndexShard indexShard) {
         if (canRecover(indexShard)) {
-            if (indexShard.routingEntry().restoreSource() != null) {
-                throw new IllegalStateException("can't recover - restore source is not null");
-            }
+            RecoverySource.Type recoveryType = indexShard.recoveryState().getRecoverySource().getType();
+            assert recoveryType == RecoverySource.Type.EMPTY_STORE || recoveryType == RecoverySource.Type.EXISTING_STORE :
+                "expected store recovery type but was: " + recoveryType;
             return executeRecovery(indexShard, () -> {
                 logger.debug("starting recovery from store ...");
-                internalRecoverFromStore(indexShard, indexShouldExists);
+                internalRecoverFromStore(indexShard);
             });
         }
         return false;
@@ -95,10 +94,8 @@ final class StoreRecovery {
 
     boolean recoverFromLocalShards(BiConsumer<String, MappingMetaData> mappingUpdateConsumer, final IndexShard indexShard, final List<LocalShardSnapshot> shards) throws IOException {
         if (canRecover(indexShard)) {
-            assert indexShard.recoveryState().getType() == RecoveryState.Type.LOCAL_SHARDS : "invalid recovery type: " + indexShard.recoveryState().getType();
-            if (indexShard.routingEntry().restoreSource() != null) {
-                throw new IllegalStateException("can't recover - restore source is not null");
-            }
+            RecoverySource.Type recoveryType = indexShard.recoveryState().getRecoverySource().getType();
+            assert recoveryType == RecoverySource.Type.LOCAL_SHARDS: "expected local shards recovery type: " + recoveryType;
             if (shards.isEmpty()) {
                 throw new IllegalArgumentException("shards must not be empty");
             }
@@ -118,7 +115,7 @@ final class StoreRecovery {
                     final Directory directory = indexShard.store().directory(); // don't close this directory!!
                     addIndices(indexShard.recoveryState().getIndex(), directory, shards.stream().map(s -> s.getSnapshotDirectory())
                         .collect(Collectors.toList()).toArray(new Directory[shards.size()]));
-                    internalRecoverFromStore(indexShard, true);
+                    internalRecoverFromStore(indexShard);
                     // just trigger a merge to do housekeeping on the
                     // copied segments - we will also see them in stats etc.
                     indexShard.getEngine().forceMerge(false, -1, false, false, false);
@@ -229,13 +226,12 @@ final class StoreRecovery {
      */
     boolean recoverFromRepository(final IndexShard indexShard, Repository repository) {
         if (canRecover(indexShard)) {
-            final ShardRouting shardRouting = indexShard.routingEntry();
-            if (shardRouting.restoreSource() == null) {
-                throw new IllegalStateException("can't restore - restore source is null");
-            }
+            RecoverySource.Type recoveryType = indexShard.recoveryState().getRecoverySource().getType();
+            assert recoveryType == RecoverySource.Type.SNAPSHOT : "expected snapshot recovery type: " + recoveryType;
+            SnapshotRecoverySource recoverySource = (SnapshotRecoverySource) indexShard.recoveryState().getRecoverySource();
             return executeRecovery(indexShard, () -> {
-                logger.debug("restoring from {} ...", shardRouting.restoreSource());
-                restore(indexShard, repository);
+                logger.debug("restoring from {} ...", indexShard.recoveryState().getRecoverySource());
+                restore(indexShard, repository, recoverySource);
             });
         }
         return false;
@@ -308,8 +304,9 @@ final class StoreRecovery {
     /**
      * Recovers the state of the shard from the store.
      */
-    private void internalRecoverFromStore(IndexShard indexShard, boolean indexShouldExists) throws IndexShardRecoveryException {
+    private void internalRecoverFromStore(IndexShard indexShard) throws IndexShardRecoveryException {
         final RecoveryState recoveryState = indexShard.recoveryState();
+        final boolean indexShouldExists = recoveryState.getRecoverySource().getType() != RecoverySource.Type.EMPTY_STORE;
         indexShard.prepareForIndexRecovery();
         long version = -1;
         SegmentInfos si = null;
@@ -340,25 +337,26 @@ final class StoreRecovery {
                         // its a "new index create" API, we have to do something, so better to clean it than use same data
                         logger.trace("cleaning existing shard, shouldn't exists");
                         Lucene.cleanLuceneIndex(store.directory());
+                        si = null;
                     }
                 }
             } catch (Exception e) {
                 throw new IndexShardRecoveryException(shardId, "failed to fetch index version after copying it over", e);
             }
             recoveryState.getIndex().updateVersion(version);
-            // since we recover from local, just fill the files and size
-            try {
-                final RecoveryState.Index index = recoveryState.getIndex();
-                if (si != null && recoveryState.getType() == RecoveryState.Type.STORE) {
-                    addRecoveredFileDetails(si, store, index);
-                }
-            } catch (IOException e) {
-                logger.debug("failed to list file details", e);
-            }
-            if (recoveryState.getType() == RecoveryState.Type.LOCAL_SHARDS) {
+            if (recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) {
                 assert indexShouldExists;
                 indexShard.skipTranslogRecovery();
             } else {
+                // since we recover from local, just fill the files and size
+                try {
+                    final RecoveryState.Index index = recoveryState.getIndex();
+                    if (si != null) {
+                        addRecoveredFileDetails(si, store, index);
+                    }
+                } catch (IOException e) {
+                    logger.debug("failed to list file details", e);
+                }
                 indexShard.performTranslogRecovery(indexShouldExists);
             }
             indexShard.finalizeRecovery();
@@ -379,10 +377,9 @@ final class StoreRecovery {
     }
 
     /**
-     * Restores shard from {@link RestoreSource} associated with this shard in routing table
+     * Restores shard from {@link SnapshotRecoverySource} associated with this shard in routing table
      */
-    private void restore(final IndexShard indexShard, final Repository repository) {
-        RestoreSource restoreSource = indexShard.routingEntry().restoreSource();
+    private void restore(final IndexShard indexShard, final Repository repository, final SnapshotRecoverySource restoreSource) {
         final RecoveryState.Translog translogState = indexShard.recoveryState().getTranslog();
         if (restoreSource == null) {
             throw new IndexShardRestoreFailedException(shardId, "empty restore source");

core/src/main/java/org/elasticsearch/indices/IndicesModule.java

@@ -26,7 +26,6 @@ import org.elasticsearch.action.update.UpdateHelper;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.common.geo.ShapesAvailability;
 import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.AllFieldMapper;
@@ -61,8 +60,8 @@ import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.flush.SyncedFlushService;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.indices.recovery.RecoverySettings;
-import org.elasticsearch.indices.recovery.RecoverySource;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.store.IndicesStore;
 import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData;
 import org.elasticsearch.indices.ttl.IndicesTTLService;
@@ -174,8 +173,8 @@ public class IndicesModule extends AbstractModule {
         bindMapperExtension();
 
         bind(RecoverySettings.class).asEagerSingleton();
-        bind(RecoveryTargetService.class).asEagerSingleton();
-        bind(RecoverySource.class).asEagerSingleton();
+        bind(PeerRecoveryTargetService.class).asEagerSingleton();
+        bind(PeerRecoverySourceService.class).asEagerSingleton();
         bind(IndicesStore.class).asEagerSingleton();
         bind(IndicesClusterStateService.class).asEagerSingleton();
         bind(SyncedFlushService.class).asEagerSingleton();

core/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -36,6 +36,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
@@ -43,7 +44,6 @@ import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -98,7 +98,7 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.search.internal.SearchContext;
@@ -439,16 +439,16 @@ public class IndicesService extends AbstractLifecycleComponent
     }
 
     @Override
-    public IndexShard createShard(ShardRouting shardRouting, RecoveryState recoveryState, RecoveryTargetService recoveryTargetService,
-                RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
-                NodeServicesProvider nodeServicesProvider, Callback<IndexShard.ShardFailure> onShardFailure) throws IOException {
+    public IndexShard createShard(ShardRouting shardRouting, RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService,
+                                  PeerRecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
+                                  NodeServicesProvider nodeServicesProvider, Callback<IndexShard.ShardFailure> onShardFailure) throws IOException {
         ensureChangesAllowed();
         IndexService indexService = indexService(shardRouting.index());
         IndexShard indexShard = indexService.createShard(shardRouting);
         indexShard.addShardFailureCallback(onShardFailure);
         indexShard.startRecovery(recoveryState, recoveryTargetService, recoveryListener, repositoriesService,
             (type, mapping) -> {
-                assert recoveryState.getType() == RecoveryState.Type.LOCAL_SHARDS :
+                assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS:
                     "mapping update consumer only required by local shards recovery";
                 try {
                     nodeServicesProvider.getClient().admin().indices().preparePutMapping()

core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java

@@ -28,6 +28,8 @@ import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.Type;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -60,9 +62,9 @@ import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.flush.SyncedFlushService;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
-import org.elasticsearch.indices.recovery.RecoverySource;
+import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.snapshots.RestoreService;
@@ -85,7 +87,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
     final AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>> indicesService;
     private final ClusterService clusterService;
     private final ThreadPool threadPool;
-    private final RecoveryTargetService recoveryTargetService;
+    private final PeerRecoveryTargetService recoveryTargetService;
     private final ShardStateAction shardStateAction;
     private final NodeMappingRefreshAction nodeMappingRefreshAction;
     private final NodeServicesProvider nodeServicesProvider;
@@ -106,15 +108,15 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
 
     @Inject
     public IndicesClusterStateService(Settings settings, IndicesService indicesService, ClusterService clusterService,
-                                      ThreadPool threadPool, RecoveryTargetService recoveryTargetService,
+                                      ThreadPool threadPool, PeerRecoveryTargetService recoveryTargetService,
                                       ShardStateAction shardStateAction,
                                       NodeMappingRefreshAction nodeMappingRefreshAction,
                                       RepositoriesService repositoriesService, RestoreService restoreService,
                                       SearchService searchService, SyncedFlushService syncedFlushService,
-                                      RecoverySource recoverySource, NodeServicesProvider nodeServicesProvider) {
+                                      PeerRecoverySourceService peerRecoverySourceService, NodeServicesProvider nodeServicesProvider) {
         this(settings, (AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>>) indicesService,
             clusterService, threadPool, recoveryTargetService, shardStateAction,
-            nodeMappingRefreshAction, repositoriesService, restoreService, searchService, syncedFlushService, recoverySource,
+            nodeMappingRefreshAction, repositoriesService, restoreService, searchService, syncedFlushService, peerRecoverySourceService,
             nodeServicesProvider);
     }
 
@@ -122,14 +124,14 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
     IndicesClusterStateService(Settings settings,
                                AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>> indicesService,
                                ClusterService clusterService,
-                               ThreadPool threadPool, RecoveryTargetService recoveryTargetService,
+                               ThreadPool threadPool, PeerRecoveryTargetService recoveryTargetService,
                                ShardStateAction shardStateAction,
                                NodeMappingRefreshAction nodeMappingRefreshAction,
                                RepositoriesService repositoriesService, RestoreService restoreService,
                                SearchService searchService, SyncedFlushService syncedFlushService,
-                               RecoverySource recoverySource, NodeServicesProvider nodeServicesProvider) {
+                               PeerRecoverySourceService peerRecoverySourceService, NodeServicesProvider nodeServicesProvider) {
         super(settings);
-        this.buildInIndexListener = Arrays.asList(recoverySource, recoveryTargetService, searchService, syncedFlushService);
+        this.buildInIndexListener = Arrays.asList(peerRecoverySourceService, recoveryTargetService, searchService, syncedFlushService);
         this.indicesService = indicesService;
         this.clusterService = clusterService;
         this.threadPool = threadPool;
@@ -372,7 +374,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
                     indexService.removeShard(shardId.id(), "removing shard (not allocated)");
                 } else {
                     // remove shards where recovery source has changed. This re-initializes shards later in createOrUpdateShards
-                    if (newShardRouting.isPeerRecovery()) {
+                    if (newShardRouting.recoverySource() != null && newShardRouting.recoverySource().getType() == Type.PEER) {
                         RecoveryState recoveryState = shard.recoveryState();
                         final DiscoveryNode sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, newShardRouting);
                         if (recoveryState.getSourceNode().equals(sourceNode) == false) {
@@ -488,7 +490,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
                 Shard shard = indexService.getShardOrNull(shardId.id());
                 if (shard == null) {
                     assert shardRouting.initializing() : shardRouting + " should have been removed by failMissingShards";
-                    createShard(nodes, routingTable, shardRouting, indexService);
+                    createShard(nodes, routingTable, shardRouting);
                 } else {
                     updateShard(nodes, shardRouting, shard);
                 }
@@ -496,12 +498,11 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
         }
     }
 
-    private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardRouting shardRouting,
-                             AllocatedIndex<? extends Shard> indexService) {
+    private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardRouting shardRouting) {
         assert shardRouting.initializing() : "only allow shard creation for initializing shard but was " + shardRouting;
 
         DiscoveryNode sourceNode = null;
-        if (shardRouting.isPeerRecovery()) {
+        if (shardRouting.recoverySource().getType() == Type.PEER)  {
             sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, shardRouting);
             if (sourceNode == null) {
                 logger.trace("ignoring initializing shard {} - no source node can be found.", shardRouting.shardId());
@@ -511,8 +512,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
 
         try {
             logger.debug("{} creating shard", shardRouting.shardId());
-            RecoveryState recoveryState = recoveryState(nodes.getLocalNode(), sourceNode, shardRouting,
-                indexService.getIndexSettings().getIndexMetaData());
+            RecoveryState recoveryState = new RecoveryState(shardRouting, nodes.getLocalNode(), sourceNode);
             indicesService.createShard(shardRouting, recoveryState, recoveryTargetService, new RecoveryListener(shardRouting),
                 repositoriesService, nodeServicesProvider, failedShardHandler);
         } catch (IndexShardAlreadyExistsException e) {
@@ -556,7 +556,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
 
     /**
      * Finds the routing source node for peer recovery, return null if its not found. Note, this method expects the shard
-     * routing to *require* peer recovery, use {@link ShardRouting#isPeerRecovery()} to
+     * routing to *require* peer recovery, use {@link ShardRouting#recoverySource()} to
      * check if its needed or not.
      */
     private static DiscoveryNode findSourceNodeForPeerRecovery(ESLogger logger, RoutingTable routingTable, DiscoveryNodes nodes,
@@ -586,7 +586,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
         return sourceNode;
     }
 
-    private class RecoveryListener implements RecoveryTargetService.RecoveryListener {
+    private class RecoveryListener implements PeerRecoveryTargetService.RecoveryListener {
 
         private final ShardRouting shardRouting;
 
@@ -596,29 +596,20 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
 
         @Override
         public void onRecoveryDone(RecoveryState state) {
-            if (state.getType() == RecoveryState.Type.SNAPSHOT) {
-                restoreService.indexShardRestoreCompleted(state.getRestoreSource().snapshot(), shardRouting.shardId());
-            }
-            shardStateAction.shardStarted(shardRouting, message(state), SHARD_STATE_ACTION_LISTENER);
-        }
-
-        private String message(RecoveryState state) {
-            switch (state.getType()) {
-                case SNAPSHOT: return "after recovery from repository";
-                case STORE: return "after recovery from store";
-                case PRIMARY_RELOCATION: return "after recovery (primary relocation) from node [" + state.getSourceNode() + "]";
-                case REPLICA: return "after recovery (replica) from node [" + state.getSourceNode() + "]";
-                case LOCAL_SHARDS: return "after recovery from local shards";
-                default: throw new IllegalArgumentException("Unknown recovery type: " + state.getType().name());
+            if (state.getRecoverySource().getType() == Type.SNAPSHOT) {
+                SnapshotRecoverySource snapshotRecoverySource = (SnapshotRecoverySource) state.getRecoverySource();
+                restoreService.indexShardRestoreCompleted(snapshotRecoverySource.snapshot(), shardRouting.shardId());
             }
+            shardStateAction.shardStarted(shardRouting, "after " + state.getRecoverySource(), SHARD_STATE_ACTION_LISTENER);
         }
 
         @Override
         public void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, boolean sendShardFailure) {
-            if (state.getType() == RecoveryState.Type.SNAPSHOT) {
+            if (state.getRecoverySource().getType() == Type.SNAPSHOT) {
                 try {
                     if (Lucene.isCorruptionException(e.getCause())) {
-                        restoreService.failRestore(state.getRestoreSource().snapshot(), shardRouting.shardId());
+                        SnapshotRecoverySource snapshotRecoverySource = (SnapshotRecoverySource) state.getRecoverySource();
+                        restoreService.failRestore(snapshotRecoverySource.snapshot(), shardRouting.shardId());
                     }
                 } catch (Exception inner) {
                     e.addSuppressed(inner);
@@ -631,27 +622,6 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
         }
     }
 
-    private RecoveryState recoveryState(DiscoveryNode localNode, DiscoveryNode sourceNode, ShardRouting shardRouting,
-                                              IndexMetaData indexMetaData) {
-        assert shardRouting.initializing() : "only allow initializing shard routing to be recovered: " + shardRouting;
-        if (shardRouting.isPeerRecovery()) {
-            assert sourceNode != null : "peer recovery started but sourceNode is null for " + shardRouting;
-            RecoveryState.Type type = shardRouting.primary() ? RecoveryState.Type.PRIMARY_RELOCATION : RecoveryState.Type.REPLICA;
-            return new RecoveryState(shardRouting.shardId(), shardRouting.primary(), type, sourceNode, localNode);
-        } else if (shardRouting.restoreSource() == null) {
-            // recover from filesystem store
-            Index mergeSourceIndex = indexMetaData.getMergeSourceIndex();
-            final boolean recoverFromLocalShards = mergeSourceIndex != null && shardRouting.allocatedPostIndexCreate(indexMetaData) == false
-                && shardRouting.primary();
-            return new RecoveryState(shardRouting.shardId(), shardRouting.primary(),
-                recoverFromLocalShards ? RecoveryState.Type.LOCAL_SHARDS : RecoveryState.Type.STORE, localNode, localNode);
-        } else {
-            // recover from a snapshot
-            return new RecoveryState(shardRouting.shardId(), shardRouting.primary(),
-                RecoveryState.Type.SNAPSHOT, shardRouting.restoreSource(), localNode);
-        }
-    }
-
     private synchronized void handleRecoveryFailure(ShardRouting shardRouting, boolean sendShardFailure, Exception failure) {
         failAndRemoveShard(shardRouting, sendShardFailure, "failed recovery", failure);
     }
@@ -821,8 +791,8 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
         /**
          * Creates shard for the specified shard routing and starts recovery,
          */
-        T createShard(ShardRouting shardRouting, RecoveryState recoveryState, RecoveryTargetService recoveryTargetService,
-                      RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
+        T createShard(ShardRouting shardRouting, RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService,
+                      PeerRecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
                       NodeServicesProvider nodeServicesProvider, Callback<IndexShard.ShardFailure> onShardFailure) throws IOException;
 
         /**

core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java

@@ -51,7 +51,7 @@ import java.util.function.Supplier;
  * The source recovery accepts recovery requests from other peer shards and start the recovery process from this
  * source shard to the target shard.
  */
-public class RecoverySource extends AbstractComponent implements IndexEventListener{
+public class PeerRecoverySourceService extends AbstractComponent implements IndexEventListener {
 
     public static class Actions {
         public static final String START_RECOVERY = "internal:index/shard/recovery/start_recovery";
@@ -66,8 +66,8 @@ public class RecoverySource extends AbstractComponent implements IndexEventListe
     private final OngoingRecoveries ongoingRecoveries = new OngoingRecoveries();
 
     @Inject
-    public RecoverySource(Settings settings, TransportService transportService, IndicesService indicesService,
-                          RecoverySettings recoverySettings, ClusterService clusterService) {
+    public PeerRecoverySourceService(Settings settings, TransportService transportService, IndicesService indicesService,
+                                     RecoverySettings recoverySettings, ClusterService clusterService) {
         super(settings);
         this.transportService = transportService;
         this.indicesService = indicesService;
@@ -97,8 +97,7 @@ public class RecoverySource extends AbstractComponent implements IndexEventListe
         }
 
         ShardRouting routingEntry = shard.routingEntry();
-        if (request.recoveryType() == RecoveryState.Type.PRIMARY_RELOCATION &&
-            (routingEntry.relocating() == false || routingEntry.relocatingNodeId().equals(request.targetNode().getId()) == false)) {
+        if (request.isPrimaryRelocation() && (routingEntry.relocating() == false || routingEntry.relocatingNodeId().equals(request.targetNode().getId()) == false)) {
             logger.debug("delaying recovery of {} as source shard is not marked yet as relocating to {}", request.shardId(), request.targetNode());
             throw new DelayRecoveryException("source shard is not marked yet as relocating to [" + request.targetNode() + "]");
         }

core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java

@@ -68,7 +68,7 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
  * Note, it can be safely assumed that there will only be a single recovery per shard (index+id) and
  * not several of them (since we don't allocate several shard replicas to the same node).
  */
-public class RecoveryTargetService extends AbstractComponent implements IndexEventListener {
+public class PeerRecoveryTargetService extends AbstractComponent implements IndexEventListener {
 
     public static class Actions {
         public static final String FILES_INFO = "internal:index/shard/recovery/filesInfo";
@@ -90,7 +90,7 @@ public class RecoveryTargetService extends AbstractComponent implements IndexEve
     private final RecoveriesCollection onGoingRecoveries;
 
     @Inject
-    public RecoveryTargetService(Settings settings, ThreadPool threadPool, TransportService transportService, RecoverySettings
+    public PeerRecoveryTargetService(Settings settings, ThreadPool threadPool, TransportService transportService, RecoverySettings
             recoverySettings, ClusterService clusterService) {
         super(settings);
         this.threadPool = threadPool;
@@ -133,8 +133,7 @@ public class RecoveryTargetService extends AbstractComponent implements IndexEve
         return onGoingRecoveries.cancelRecoveriesForShard(shardId, reason);
     }
 
-    public void startRecovery(final IndexShard indexShard, final RecoveryState.Type recoveryType, final DiscoveryNode sourceNode, final
-    RecoveryListener listener) {
+    public void startRecovery(final IndexShard indexShard, final DiscoveryNode sourceNode, final RecoveryListener listener) {
         // create a new recovery status, and process...
         final long recoveryId = onGoingRecoveries.startRecovery(indexShard, sourceNode, listener, recoverySettings.activityTimeout());
         threadPool.generic().execute(new RecoveryRunner(recoveryId));
@@ -189,8 +188,7 @@ public class RecoveryTargetService extends AbstractComponent implements IndexEve
         }
         logger.trace("{} local file count: [{}]", recoveryTarget, metadataSnapshot.size());
         final StartRecoveryRequest request = new StartRecoveryRequest(recoveryTarget.shardId(), recoveryTarget.sourceNode(),
-                clusterService.localNode(),
-                metadataSnapshot, recoveryTarget.state().getType(), recoveryTarget.recoveryId());
+                clusterService.localNode(), metadataSnapshot, recoveryTarget.state().getPrimary(), recoveryTarget.recoveryId());
 
         final AtomicReference<RecoveryResponse> responseHolder = new AtomicReference<>();
         try {
@@ -198,7 +196,7 @@ public class RecoveryTargetService extends AbstractComponent implements IndexEve
                     .sourceNode());
             recoveryTarget.indexShard().prepareForIndexRecovery();
             recoveryTarget.CancellableThreads().execute(() -> responseHolder.set(
-                    transportService.submitRequest(request.sourceNode(), RecoverySource.Actions.START_RECOVERY, request,
+                    transportService.submitRequest(request.sourceNode(), PeerRecoverySourceService.Actions.START_RECOVERY, request,
                             new FutureTransportResponseHandler<RecoveryResponse>() {
                                 @Override
                                 public RecoveryResponse newInstance() {

core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java

@@ -62,7 +62,7 @@ public class RecoveriesCollection {
      * @return the id of the new recovery.
      */
     public long startRecovery(IndexShard indexShard, DiscoveryNode sourceNode,
-                              RecoveryTargetService.RecoveryListener listener, TimeValue activityTimeout) {
+                              PeerRecoveryTargetService.RecoveryListener listener, TimeValue activityTimeout) {
         RecoveryTarget status = new RecoveryTarget(indexShard, sourceNode, listener, ensureClusterStateVersionCallback);
         RecoveryTarget existingStatus = onGoingRecoveries.putIfAbsent(status.recoveryId(), status);
         assert existingStatus == null : "found two RecoveryStatus instances with the same id";

core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java

@@ -30,6 +30,7 @@ import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.StopWatch;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -386,7 +387,7 @@ public class RecoverySourceHandler {
         logger.trace("[{}][{}] finalizing recovery to {}", indexName, shardId, request.targetNode());
         cancellableThreads.execute(recoveryTarget::finalizeRecovery);
 
-        if (isPrimaryRelocation()) {
+        if (request.isPrimaryRelocation()) {
             // in case of primary relocation we have to ensure that the cluster state on the primary relocation target has all
             // replica shards that have recovered or are still recovering from the current primary, otherwise replication actions
             // will not be send to these replicas. To accomplish this, first block new recoveries, then take version of latest cluster
@@ -410,10 +411,6 @@ public class RecoverySourceHandler {
                 indexName, shardId, request.targetNode(), stopWatch.totalTime());
     }
 
-    protected boolean isPrimaryRelocation() {
-        return request.recoveryType() == RecoveryState.Type.PRIMARY_RELOCATION;
-    }
-
     /**
      * Send the given snapshot's operations to this handler's target node.
      * <p>

core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java

@@ -20,7 +20,8 @@
 package org.elasticsearch.indices.recovery;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RestoreSource;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -96,40 +97,6 @@ public class RecoveryState implements ToXContent, Streamable {
         }
     }
 
-    public enum Type {
-        STORE((byte) 0),
-        SNAPSHOT((byte) 1),
-        REPLICA((byte) 2),
-        PRIMARY_RELOCATION((byte) 3),
-        LOCAL_SHARDS((byte) 4);
-
-        private static final Type[] TYPES = new Type[Type.values().length];
-
-        static {
-            for (Type type : Type.values()) {
-                assert type.id() < TYPES.length && type.id() >= 0;
-                TYPES[type.id] = type;
-            }
-        }
-
-        private final byte id;
-
-        Type(byte id) {
-            this.id = id;
-        }
-
-        public byte id() {
-            return id;
-        }
-
-        public static Type fromId(byte id) {
-            if (id < 0 || id >= TYPES.length) {
-                throw new IllegalArgumentException("No mapping for id [" + id + "]");
-            }
-            return TYPES[id];
-        }
-    }
-
     private Stage stage;
 
     private final Index index = new Index();
@@ -137,9 +104,9 @@ public class RecoveryState implements ToXContent, Streamable {
     private final VerifyIndex verifyIndex = new VerifyIndex();
     private final Timer timer = new Timer();
 
-    private Type type;
+    private RecoverySource recoverySource;
     private ShardId shardId;
-    private RestoreSource restoreSource;
+    @Nullable
     private DiscoveryNode sourceNode;
     private DiscoveryNode targetNode;
     private boolean primary = false;
@@ -147,20 +114,15 @@ public class RecoveryState implements ToXContent, Streamable {
     private RecoveryState() {
     }
 
-    public RecoveryState(ShardId shardId, boolean primary, Type type, DiscoveryNode sourceNode, DiscoveryNode targetNode) {
-        this(shardId, primary, type, sourceNode, null, targetNode);
-    }
-
-    public RecoveryState(ShardId shardId, boolean primary, Type type, RestoreSource restoreSource, DiscoveryNode targetNode) {
-        this(shardId, primary, type, null, restoreSource, targetNode);
-    }
-
-    private RecoveryState(ShardId shardId, boolean primary, Type type, @Nullable DiscoveryNode sourceNode, @Nullable RestoreSource restoreSource, DiscoveryNode targetNode) {
-        this.shardId = shardId;
-        this.primary = primary;
-        this.type = type;
+    public RecoveryState(ShardRouting shardRouting, DiscoveryNode targetNode, @Nullable DiscoveryNode sourceNode) {
+        assert shardRouting.initializing() : "only allow initializing shard routing to be recovered: " + shardRouting;
+        RecoverySource recoverySource = shardRouting.recoverySource();
+        assert (recoverySource.getType() == RecoverySource.Type.PEER) == (sourceNode != null) :
+            "peer recovery requires source node, recovery type: " + recoverySource.getType() + " source node: " + sourceNode;
+        this.shardId = shardRouting.shardId();
+        this.primary = shardRouting.primary();
+        this.recoverySource = recoverySource;
         this.sourceNode = sourceNode;
-        this.restoreSource = restoreSource;
         this.targetNode = targetNode;
         stage = Stage.INIT;
         timer.start();
@@ -237,10 +199,14 @@ public class RecoveryState implements ToXContent, Streamable {
         return timer;
     }
 
-    public Type getType() {
-        return type;
+    public RecoverySource getRecoverySource() {
+        return recoverySource;
     }
 
+    /**
+     * Returns recovery source node (only non-null if peer recovery)
+     */
+    @Nullable
     public DiscoveryNode getSourceNode() {
         return sourceNode;
     }
@@ -249,10 +215,6 @@ public class RecoveryState implements ToXContent, Streamable {
         return targetNode;
     }
 
-    public RestoreSource getRestoreSource() {
-        return restoreSource;
-    }
-
     public boolean getPrimary() {
         return primary;
     }
@@ -266,14 +228,11 @@ public class RecoveryState implements ToXContent, Streamable {
     @Override
     public synchronized void readFrom(StreamInput in) throws IOException {
         timer.readFrom(in);
-        type = Type.fromId(in.readByte());
         stage = Stage.fromId(in.readByte());
         shardId = ShardId.readShardId(in);
-        restoreSource = RestoreSource.readOptionalRestoreSource(in);
+        recoverySource = RecoverySource.readFrom(in);
         targetNode = new DiscoveryNode(in);
-        if (in.readBoolean()) {
-            sourceNode = new DiscoveryNode(in);
-        }
+        sourceNode = in.readOptionalWriteable(DiscoveryNode::new);
         index.readFrom(in);
         translog.readFrom(in);
         verifyIndex.readFrom(in);
@@ -283,15 +242,11 @@ public class RecoveryState implements ToXContent, Streamable {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         timer.writeTo(out);
-        out.writeByte(type.id());
         out.writeByte(stage.id());
         shardId.writeTo(out);
-        out.writeOptionalStreamable(restoreSource);
+        recoverySource.writeTo(out);
         targetNode.writeTo(out);
-        out.writeBoolean(sourceNode != null);
-        if (sourceNode != null) {
-            sourceNode.writeTo(out);
-        }
+        out.writeOptionalWriteable(sourceNode);
         index.writeTo(out);
         translog.writeTo(out);
         verifyIndex.writeTo(out);
@@ -302,7 +257,7 @@ public class RecoveryState implements ToXContent, Streamable {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
 
         builder.field(Fields.ID, shardId.id());
-        builder.field(Fields.TYPE, type.toString());
+        builder.field(Fields.TYPE, recoverySource.getType());
         builder.field(Fields.STAGE, stage.toString());
         builder.field(Fields.PRIMARY, primary);
         builder.dateValueField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, timer.startTime);
@@ -311,10 +266,7 @@ public class RecoveryState implements ToXContent, Streamable {
         }
         builder.timeValueField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, timer.time());
 
-        if (restoreSource != null) {
-            builder.field(Fields.SOURCE);
-            restoreSource.toXContent(builder, params);
-        } else {
+        if (recoverySource.getType() == RecoverySource.Type.PEER) {
             builder.startObject(Fields.SOURCE);
             builder.field(Fields.ID, sourceNode.getId());
             builder.field(Fields.HOST, sourceNode.getHostName());
@@ -322,6 +274,10 @@ public class RecoveryState implements ToXContent, Streamable {
             builder.field(Fields.IP, sourceNode.getHostAddress());
             builder.field(Fields.NAME, sourceNode.getName());
             builder.endObject();
+        } else {
+            builder.startObject(Fields.SOURCE);
+            recoverySource.addAdditionalFields(builder, params);
+            builder.endObject();
         }
 
         builder.startObject(Fields.TARGET);

core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java

@@ -74,7 +74,7 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
     private final DiscoveryNode sourceNode;
     private final String tempFilePrefix;
     private final Store store;
-    private final RecoveryTargetService.RecoveryListener listener;
+    private final PeerRecoveryTargetService.RecoveryListener listener;
     private final Callback<Long> ensureClusterStateVersionCallback;
 
     private final AtomicBoolean finished = new AtomicBoolean();
@@ -92,7 +92,7 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
             copyFrom.ensureClusterStateVersionCallback);
     }
 
-    public RecoveryTarget(IndexShard indexShard, DiscoveryNode sourceNode, RecoveryTargetService.RecoveryListener listener,
+    public RecoveryTarget(IndexShard indexShard, DiscoveryNode sourceNode, PeerRecoveryTargetService.RecoveryListener listener,
                           Callback<Long> ensureClusterStateVersionCallback) {
         this(indexShard, sourceNode, listener, new CancellableThreads(), idGenerator.incrementAndGet(), ensureClusterStateVersionCallback);
     }
@@ -106,7 +106,7 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
      *                                          version. Necessary for primary relocation so that new primary knows about all other ongoing
      *                                          replica recoveries when replicating documents (see {@link RecoverySourceHandler}).
      */
-    private RecoveryTarget(IndexShard indexShard, DiscoveryNode sourceNode, RecoveryTargetService.RecoveryListener listener,
+    private RecoveryTarget(IndexShard indexShard, DiscoveryNode sourceNode, PeerRecoveryTargetService.RecoveryListener listener,
                            CancellableThreads cancellableThreads, long recoveryId, Callback<Long> ensureClusterStateVersionCallback) {
         super("recovery_status");
         this.cancellableThreads = cancellableThreads;

core/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java

@@ -75,7 +75,7 @@ public class RemoteRecoveryTargetHandler implements RecoveryTargetHandler {
 
     @Override
     public void prepareForTranslogOperations(int totalTranslogOps) throws IOException {
-        transportService.submitRequest(targetNode, RecoveryTargetService.Actions.PREPARE_TRANSLOG,
+        transportService.submitRequest(targetNode, PeerRecoveryTargetService.Actions.PREPARE_TRANSLOG,
                 new RecoveryPrepareForTranslogOperationsRequest(recoveryId, shardId, totalTranslogOps),
                 TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionTimeout()).build(),
                 EmptyTransportResponseHandler.INSTANCE_SAME).txGet();
@@ -83,7 +83,7 @@ public class RemoteRecoveryTargetHandler implements RecoveryTargetHandler {
 
     @Override
     public void finalizeRecovery() {
-        transportService.submitRequest(targetNode, RecoveryTargetService.Actions.FINALIZE,
+        transportService.submitRequest(targetNode, PeerRecoveryTargetService.Actions.FINALIZE,
                 new RecoveryFinalizeRecoveryRequest(recoveryId, shardId),
                 TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionLongTimeout()).build(),
                 EmptyTransportResponseHandler.INSTANCE_SAME).txGet();
@@ -91,7 +91,7 @@ public class RemoteRecoveryTargetHandler implements RecoveryTargetHandler {
 
     @Override
     public void ensureClusterStateVersion(long clusterStateVersion) {
-        transportService.submitRequest(targetNode, RecoveryTargetService.Actions.WAIT_CLUSTERSTATE,
+        transportService.submitRequest(targetNode, PeerRecoveryTargetService.Actions.WAIT_CLUSTERSTATE,
             new RecoveryWaitForClusterStateRequest(recoveryId, shardId, clusterStateVersion),
             TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionLongTimeout()).build(),
             EmptyTransportResponseHandler.INSTANCE_SAME).txGet();
@@ -101,7 +101,7 @@ public class RemoteRecoveryTargetHandler implements RecoveryTargetHandler {
     public void indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps) {
         final RecoveryTranslogOperationsRequest translogOperationsRequest = new RecoveryTranslogOperationsRequest(
                 recoveryId, shardId, operations, totalTranslogOps);
-        transportService.submitRequest(targetNode, RecoveryTargetService.Actions.TRANSLOG_OPS, translogOperationsRequest,
+        transportService.submitRequest(targetNode, PeerRecoveryTargetService.Actions.TRANSLOG_OPS, translogOperationsRequest,
                 translogOpsRequestOptions, EmptyTransportResponseHandler.INSTANCE_SAME).txGet();
     }
 
@@ -111,7 +111,7 @@ public class RemoteRecoveryTargetHandler implements RecoveryTargetHandler {
 
         RecoveryFilesInfoRequest recoveryInfoFilesRequest = new RecoveryFilesInfoRequest(recoveryId, shardId,
                 phase1FileNames, phase1FileSizes, phase1ExistingFileNames, phase1ExistingFileSizes, totalTranslogOps);
-        transportService.submitRequest(targetNode, RecoveryTargetService.Actions.FILES_INFO, recoveryInfoFilesRequest,
+        transportService.submitRequest(targetNode, PeerRecoveryTargetService.Actions.FILES_INFO, recoveryInfoFilesRequest,
                 TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionTimeout()).build(),
                 EmptyTransportResponseHandler.INSTANCE_SAME).txGet();
 
@@ -119,7 +119,7 @@ public class RemoteRecoveryTargetHandler implements RecoveryTargetHandler {
 
     @Override
     public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaData) throws IOException {
-        transportService.submitRequest(targetNode, RecoveryTargetService.Actions.CLEAN_FILES,
+        transportService.submitRequest(targetNode, PeerRecoveryTargetService.Actions.CLEAN_FILES,
                 new RecoveryCleanFilesRequest(recoveryId, shardId, sourceMetaData, totalTranslogOps),
                 TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionTimeout()).build(),
                 EmptyTransportResponseHandler.INSTANCE_SAME).txGet();
@@ -150,7 +150,7 @@ public class RemoteRecoveryTargetHandler implements RecoveryTargetHandler {
             throttleTimeInNanos = 0;
         }
 
-        transportService.submitRequest(targetNode, RecoveryTargetService.Actions.FILE_CHUNK,
+        transportService.submitRequest(targetNode, PeerRecoveryTargetService.Actions.FILE_CHUNK,
                 new RecoveryFileChunkRequest(recoveryId, shardId, fileMetaData, position, content, lastChunk,
                         totalTranslogOps,
                                 /* we send totalOperations with every request since we collect stats on the target and that way we can

core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java

@@ -50,7 +50,7 @@ public class SharedFSRecoverySourceHandler extends RecoverySourceHandler {
         boolean engineClosed = false;
         try {
             logger.trace("{} recovery [phase1] to {}: skipping phase 1 for shared filesystem", request.shardId(), request.targetNode());
-            if (isPrimaryRelocation()) {
+            if (request.isPrimaryRelocation()) {
                 logger.debug("[phase1] closing engine on primary for shared filesystem recovery");
                 try {
                     // if we relocate we need to close the engine in order to open a new

core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.indices.recovery;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.shard.ShardId;
@@ -43,7 +44,7 @@ public class StartRecoveryRequest extends TransportRequest {
 
     private Store.MetadataSnapshot metadataSnapshot;
 
-    private RecoveryState.Type recoveryType;
+    private boolean primaryRelocation;
 
     public StartRecoveryRequest() {
     }
@@ -54,13 +55,13 @@ public class StartRecoveryRequest extends TransportRequest {
      * @param sourceNode       The node to recover from
      * @param targetNode       The node to recover to
      */
-    public StartRecoveryRequest(ShardId shardId, DiscoveryNode sourceNode, DiscoveryNode targetNode, Store.MetadataSnapshot metadataSnapshot, RecoveryState.Type recoveryType, long recoveryId) {
+    public StartRecoveryRequest(ShardId shardId, DiscoveryNode sourceNode, DiscoveryNode targetNode, Store.MetadataSnapshot metadataSnapshot, boolean primaryRelocation, long recoveryId) {
         this.recoveryId = recoveryId;
         this.shardId = shardId;
         this.sourceNode = sourceNode;
         this.targetNode = targetNode;
-        this.recoveryType = recoveryType;
         this.metadataSnapshot = metadataSnapshot;
+        this.primaryRelocation = primaryRelocation;
     }
 
     public long recoveryId() {
@@ -79,8 +80,8 @@ public class StartRecoveryRequest extends TransportRequest {
         return targetNode;
     }
 
-    public RecoveryState.Type recoveryType() {
-        return recoveryType;
+    public boolean isPrimaryRelocation() {
+        return primaryRelocation;
     }
 
     public Store.MetadataSnapshot metadataSnapshot() {
@@ -95,8 +96,7 @@ public class StartRecoveryRequest extends TransportRequest {
         sourceNode = new DiscoveryNode(in);
         targetNode = new DiscoveryNode(in);
         metadataSnapshot = new Store.MetadataSnapshot(in);
-        recoveryType = RecoveryState.Type.fromId(in.readByte());
-
+        primaryRelocation = in.readBoolean();
     }
 
     @Override
@@ -107,7 +107,7 @@ public class StartRecoveryRequest extends TransportRequest {
         sourceNode.writeTo(out);
         targetNode.writeTo(out);
         metadataSnapshot.writeTo(out);
-        out.writeByte(recoveryType.id());
+        out.writeBoolean(primaryRelocation);
     }
 
 }

core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java

@@ -24,6 +24,8 @@ import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.inject.Inject;
@@ -147,14 +149,16 @@ public class RestRecoveryAction extends AbstractCatAction {
                 t.addCell(index);
                 t.addCell(state.getShardId().id());
                 t.addCell(new TimeValue(state.getTimer().time()));
-                t.addCell(state.getType().toString().toLowerCase(Locale.ROOT));
+                t.addCell(state.getRecoverySource().getType().toString().toLowerCase(Locale.ROOT));
                 t.addCell(state.getStage().toString().toLowerCase(Locale.ROOT));
                 t.addCell(state.getSourceNode() == null ? "n/a" : state.getSourceNode().getHostName());
                 t.addCell(state.getSourceNode() == null ? "n/a" : state.getSourceNode().getName());
                 t.addCell(state.getTargetNode().getHostName());
                 t.addCell(state.getTargetNode().getName());
-                t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshot().getRepository());
-                t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshot().getSnapshotId().getName());
+                t.addCell(state.getRecoverySource() == null || state.getRecoverySource().getType() != RecoverySource.Type.SNAPSHOT ? "n/a" :
+                    ((SnapshotRecoverySource) state.getRecoverySource()).snapshot().getRepository());
+                t.addCell(state.getRecoverySource() == null || state.getRecoverySource().getType() != RecoverySource.Type.SNAPSHOT ? "n/a" :
+                    ((SnapshotRecoverySource) state.getRecoverySource()).snapshot().getSnapshotId().getName());
                 t.addCell(state.getIndex().totalRecoverFiles());
                 t.addCell(state.getIndex().recoveredFileCount());
                 t.addCell(String.format(Locale.ROOT, "%1.1f%%", state.getIndex().recoveredFilesPercent()));

core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java

@@ -43,6 +43,8 @@ import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestActionListener;
 import org.elasticsearch.rest.action.RestResponseListener;
 
+import java.util.Locale;
+
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestShardsAction extends AbstractCatAction {
@@ -104,6 +106,8 @@ public class RestShardsAction extends AbstractCatAction {
         table.addCell("unassigned.for", "alias:uf;default:false;text-align:right;desc:time has been unassigned");
         table.addCell("unassigned.details", "alias:ud;default:false;desc:additional details as to why the shard became unassigned");
 
+        table.addCell("recoverysource.type", "alias:rs;default:false;desc:recovery source type");
+
         table.addCell("completion.size", "alias:cs,completionSize;default:false;text-align:right;desc:size of completion");
 
         table.addCell("fielddata.memory_size", "alias:fm,fielddataMemory;default:false;text-align:right;desc:used fielddata cache");
@@ -240,6 +244,12 @@ public class RestShardsAction extends AbstractCatAction {
                 table.addCell(null);
             }
 
+            if (shard.recoverySource() != null) {
+                table.addCell(shard.recoverySource().getType().toString().toLowerCase(Locale.ROOT));
+            } else {
+                table.addCell(null);
+            }
+
             table.addCell(commonStats == null ? null : commonStats.getCompletion().getSize());
 
             table.addCell(commonStats == null ? null : commonStats.getFieldData().getMemorySize());

core/src/main/java/org/elasticsearch/snapshots/RestoreService.java

@@ -41,7 +41,7 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
-import org.elasticsearch.cluster.routing.RestoreSource;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -109,13 +109,13 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet;
  * First {@link #restoreSnapshot(RestoreRequest, org.elasticsearch.action.ActionListener)}
  * method reads information about snapshot and metadata from repository. In update cluster state task it checks restore
  * preconditions, restores global state if needed, creates {@link RestoreInProgress} record with list of shards that needs
- * to be restored and adds this shard to the routing table using {@link org.elasticsearch.cluster.routing.RoutingTable.Builder#addAsRestore(IndexMetaData, RestoreSource)}
+ * to be restored and adds this shard to the routing table using {@link RoutingTable.Builder#addAsRestore(IndexMetaData, SnapshotRecoverySource)}
  * method.
  * <p>
  * Individual shards are getting restored as part of normal recovery process in
  * {@link IndexShard#restoreFromRepository(Repository)} )}
  * method, which detects that shard should be restored from snapshot rather than recovered from gateway by looking
- * at the {@link org.elasticsearch.cluster.routing.ShardRouting#restoreSource()} property.
+ * at the {@link ShardRouting#recoverySource()} property.
  * <p>
  * At the end of the successful restore process {@code IndexShardSnapshotAndRestoreService} calls {@link #indexShardRestoreCompleted(Snapshot, ShardId)},
  * which updates {@link RestoreInProgress} in cluster state or removes it when all shards are completed. In case of
@@ -241,7 +241,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                         for (Map.Entry<String, String> indexEntry : renamedIndices.entrySet()) {
                             String index = indexEntry.getValue();
                             boolean partial = checkPartial(index);
-                            RestoreSource restoreSource = new RestoreSource(snapshot, snapshotInfo.version(), index);
+                            SnapshotRecoverySource recoverySource = new SnapshotRecoverySource(snapshot, snapshotInfo.version(), index);
                             String renamedIndexName = indexEntry.getKey();
                             IndexMetaData snapshotIndexMetaData = metaData.index(index);
                             snapshotIndexMetaData = updateIndexSettings(snapshotIndexMetaData, request.indexSettings, request.ignoreIndexSettings);
@@ -273,7 +273,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                                 if (partial) {
                                     populateIgnoredShards(index, ignoreShards);
                                 }
-                                rtBuilder.addAsNewRestore(updatedIndexMetaData, restoreSource, ignoreShards);
+                                rtBuilder.addAsNewRestore(updatedIndexMetaData, recoverySource, ignoreShards);
                                 blocks.addBlocks(updatedIndexMetaData);
                                 mdBuilder.put(updatedIndexMetaData, true);
                                 renamedIndex = updatedIndexMetaData.getIndex();
@@ -298,7 +298,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                                 }
                                 indexMdBuilder.settings(Settings.builder().put(snapshotIndexMetaData.getSettings()).put(IndexMetaData.SETTING_INDEX_UUID, currentIndexMetaData.getIndexUUID()));
                                 IndexMetaData updatedIndexMetaData = indexMdBuilder.index(renamedIndexName).build();
-                                rtBuilder.addAsRestore(updatedIndexMetaData, restoreSource);
+                                rtBuilder.addAsRestore(updatedIndexMetaData, recoverySource);
                                 blocks.updateBlocks(updatedIndexMetaData);
                                 mdBuilder.put(updatedIndexMetaData, true);
                                 renamedIndex = updatedIndexMetaData.getIndex();

core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java

@@ -25,6 +25,8 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
@@ -55,8 +57,8 @@ import static java.util.Collections.emptySet;
 public final class ClusterAllocationExplanationTests extends ESTestCase {
 
     private Index i = new Index("foo", "uuid");
-    private ShardRouting primaryShard = ShardRouting.newUnassigned(new ShardId(i, 0), null, true,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+    private ShardRouting primaryShard = ShardRouting.newUnassigned(new ShardId(i, 0), true, StoreRecoverySource.EMPTY_STORE_INSTANCE,
+        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
     private IndexMetaData indexMetaData = IndexMetaData.builder("foo")
             .settings(Settings.builder()
                     .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
@@ -87,16 +89,14 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
         Float nodeWeight = randomFloat();
         Set<String> activeAllocationIds = new HashSet<>();
         activeAllocationIds.add("eggplant");
-        ShardRouting primaryStartedShard = ShardRouting.newUnassigned(new ShardId(i, 0), null, true,
+        ShardRouting primaryStartedShard = ShardRouting.newUnassigned(new ShardId(i, 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE,
                 new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "foo"));
-        assertTrue(primaryStartedShard.allocatedPostIndexCreate(indexMetaData));
-        ShardRouting replicaStartedShard = ShardRouting.newUnassigned(new ShardId(i, 0), null, false,
+        ShardRouting replicaStartedShard = ShardRouting.newUnassigned(new ShardId(i, 0), false, PeerRecoverySource.INSTANCE,
                 new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "foo"));
-        assertTrue(replicaStartedShard.allocatedPostIndexCreate(indexMetaData));
 
         IndicesShardStoresResponse.StoreStatus storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
                 IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, e);
-        NodeExplanation ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node,
+        NodeExplanation ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node,
                 yesDecision, nodeWeight, storeStatus, "", activeAllocationIds, false);
         assertExplanations(ne, "the copy of the shard cannot be read",
                 ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.IO_ERROR);
@@ -125,8 +125,8 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
 
         storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
                 IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, corruptE);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                storeStatus, "", activeAllocationIds, false);
+        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node, yesDecision,
+                nodeWeight, storeStatus, "", activeAllocationIds, false);
         assertExplanations(ne, "the copy of the shard is corrupt",
                 ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.CORRUPT);
 
@@ -169,8 +169,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
                 IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null);
         ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(replicaStartedShard, indexMetaData, node, noDecision,
                 nodeWeight, storeStatus, "", activeAllocationIds, true);
-        assertExplanations(ne, "the shard cannot be assigned because allocation deciders return a NO " +
-                        "decision and the shard's state is still being fetched",
+        assertExplanations(ne, "the shard cannot be assigned because allocation deciders return a NO decision",
                 ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
     }
 

core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java

@@ -115,7 +115,7 @@ public class ClusterStateCreationUtils {
         } else {
             unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null);
         }
-        indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, 0, primaryNode, relocatingNode, null, true,
+        indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, 0, primaryNode, relocatingNode, true,
                 primaryState, unassignedInfo));
 
         for (ShardRoutingState replicaState : replicaStates) {
@@ -132,7 +132,7 @@ public class ClusterStateCreationUtils {
                 unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null);
             }
             indexShardRoutingBuilder.addShard(
-                    TestShardRouting.newShardRouting(index, shardId.id(), replicaNode, relocatingNode, null, false, replicaState,
+                    TestShardRouting.newShardRouting(index, shardId.id(), replicaNode, relocatingNode, false, replicaState,
                             unassignedInfo));
         }
 
@@ -170,10 +170,10 @@ public class ClusterStateCreationUtils {
             routing.addAsNew(indexMetaData);
             final ShardId shardId = new ShardId(index, "_na_", i);
             IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
-            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(0).getId(), null, null, true,
-                    ShardRoutingState.STARTED, null));
-            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(1).getId(), null, null, false,
-                    ShardRoutingState.STARTED, null));
+            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(0).getId(), null, true,
+                    ShardRoutingState.STARTED));
+            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(1).getId(), null, false,
+                    ShardRoutingState.STARTED));
             indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder.build());
         }
         state.routingTable(RoutingTable.builder().add(indexRoutingTableBuilder.build()).build());

core/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java

@@ -18,15 +18,12 @@
  */
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.routing.RestoreSource;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.snapshots.Snapshot;
-import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.ESTestCase;
 
 public class ClusterInfoTests extends ESTestCase {
@@ -74,12 +71,8 @@ public class ClusterInfoTests extends ESTestCase {
         int numEntries = randomIntBetween(0, 128);
         ImmutableOpenMap.Builder<ShardRouting, String> builder = ImmutableOpenMap.builder(numEntries);
         for (int i = 0; i < numEntries; i++) {
-            RestoreSource restoreSource = new RestoreSource(new Snapshot(randomAsciiOfLength(4),
-                    new SnapshotId(randomAsciiOfLength(4), randomAsciiOfLength(4))), Version.CURRENT, randomAsciiOfLength(4));
-            UnassignedInfo.Reason reason = randomFrom(UnassignedInfo.Reason.values());
-            UnassignedInfo unassignedInfo = new UnassignedInfo(reason, randomAsciiOfLength(4));
             ShardId shardId = new ShardId(randomAsciiOfLength(32), randomAsciiOfLength(32), randomIntBetween(0, Integer.MAX_VALUE));
-            ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, restoreSource, randomBoolean(), unassignedInfo);
+            ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, null, randomBoolean(), ShardRoutingState.UNASSIGNED);
             builder.put(shardRouting, randomAsciiOfLength(32));
         }
         return builder.build();

core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java

@@ -245,7 +245,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
                     unassignedInfo = new UnassignedInfo(randomReason(), randomAsciiOfLength(10));
                 }
                 indexShard.addShard(
-                        TestShardRouting.newShardRouting(index, i, randomFrom(nodeIds), null, null, j == 0,
+                        TestShardRouting.newShardRouting(index, i, randomFrom(nodeIds), null, j == 0,
                                 ShardRoutingState.fromValue((byte) randomIntBetween(2, 3)), unassignedInfo));
             }
             builder.addIndexShard(indexShard.build());

core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java

@@ -26,6 +26,8 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingHelper;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -101,13 +103,13 @@ public class DiskUsageTests extends ESTestCase {
 
     public void testFillShardLevelInfo() {
         final Index index = new Index("test", "0xdeadbeef");
-        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_0 = ShardRoutingHelper.initialize(test_0, "node1");
         test_0 = ShardRoutingHelper.moveToStarted(test_0);
         Path test0Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
         CommonStats commonStats0 = new CommonStats();
         commonStats0.store = new StoreStats(100, 1);
-        ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_1 = ShardRoutingHelper.initialize(test_1, "node2");
         test_1 = ShardRoutingHelper.moveToStarted(test_1);
         Path test1Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("1");
@@ -136,14 +138,14 @@ public class DiskUsageTests extends ESTestCase {
 
     public void testFillShardsWithShadowIndices() {
         final Index index = new Index("non-shadow", "0xcafe0000");
-        ShardRouting s0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting s0 = ShardRouting.newUnassigned(new ShardId(index, 0), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         s0 = ShardRoutingHelper.initialize(s0, "node1");
         s0 = ShardRoutingHelper.moveToStarted(s0);
         Path i0Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
         CommonStats commonStats0 = new CommonStats();
         commonStats0.store = new StoreStats(100, 1);
         final Index index2 = new Index("shadow", "0xcafe0001");
-        ShardRouting s1 = ShardRouting.newUnassigned(new ShardId(index2, 0), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting s1 = ShardRouting.newUnassigned(new ShardId(index2, 0), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         s1 = ShardRoutingHelper.initialize(s1, "node2");
         s1 = ShardRoutingHelper.moveToStarted(s1);
         Path i1Path = createTempDir().resolve("indices").resolve(index2.getUUID()).resolve("0");

core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java

@@ -25,6 +25,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -43,7 +44,6 @@ import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.junit.Before;
 
 import java.util.ArrayList;
@@ -56,7 +56,6 @@ import java.util.stream.IntStream;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.not;
 
 public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCase {
 

core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java

@@ -35,6 +35,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTableGenerator;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -518,7 +519,7 @@ public class ClusterStateHealthTests extends ESTestCase {
     }
 
     // returns true if the inactive primaries in the index are only due to cluster recovery
-    // (not because of allocation failure or previously having allocation ids assigned)
+    // (not because of allocation of existing shard or previously having allocation ids assigned)
     private boolean primaryInactiveDueToRecovery(final String indexName, final ClusterState clusterState) {
         for (final IntObjectCursor<IndexShardRoutingTable> shardRouting : clusterState.routingTable().index(indexName).shards()) {
             final ShardRouting primaryShard = shardRouting.value.primaryShard();
@@ -526,8 +527,8 @@ public class ClusterStateHealthTests extends ESTestCase {
                 if (clusterState.metaData().index(indexName).inSyncAllocationIds(shardRouting.key).isEmpty() == false) {
                     return false;
                 }
-                if (primaryShard.unassignedInfo() != null &&
-                        primaryShard.unassignedInfo().getReason() == UnassignedInfo.Reason.ALLOCATION_FAILED) {
+                if (primaryShard.recoverySource() != null &&
+                    primaryShard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE) {
                     return false;
                 }
             }

core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.cluster.routing;
 
+import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -38,7 +39,7 @@ import static org.hamcrest.Matchers.nullValue;
 public class AllocationIdTests extends ESTestCase {
     public void testShardToStarted() {
         logger.info("-- create unassigned shard");
-        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
         assertThat(shard.allocationId(), nullValue());
 
         logger.info("-- initialize the shard");
@@ -58,7 +59,7 @@ public class AllocationIdTests extends ESTestCase {
 
     public void testSuccessfulRelocation() {
         logger.info("-- build started shard");
-        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
         shard = shard.initialize("node1", null, -1);
         shard = shard.moveToStarted();
 
@@ -81,7 +82,7 @@ public class AllocationIdTests extends ESTestCase {
 
     public void testCancelRelocation() {
         logger.info("-- build started shard");
-        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
         shard = shard.initialize("node1", null, -1);
         shard = shard.moveToStarted();
 
@@ -101,7 +102,7 @@ public class AllocationIdTests extends ESTestCase {
 
     public void testMoveToUnassigned() {
         logger.info("-- build started shard");
-        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
         shard = shard.initialize("node1", null, -1);
         shard = shard.moveToStarted();
 
@@ -112,13 +113,13 @@ public class AllocationIdTests extends ESTestCase {
 
     public void testReinitializing() {
         logger.info("-- build started shard");
-        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
         shard = shard.initialize("node1", null, -1);
         shard = shard.moveToStarted();
         AllocationId allocationId = shard.allocationId();
 
         logger.info("-- reinitializing shard");
-        shard = shard.reinitializeShard();
+        shard = shard.reinitializePrimaryShard();
         assertThat(shard.allocationId().getId(), notNullValue());
         assertThat(shard.allocationId().getRelocationId(), nullValue());
         assertThat(shard.allocationId().getId(), not(equalTo(allocationId.getId())));

core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java

@@ -37,7 +37,8 @@ public final class RandomShardRoutingMutator {
                 if (shardRouting.unassigned() == false && shardRouting.primary() == false) {
                     shardRouting = shardRouting.moveToUnassigned(new UnassignedInfo(randomReason(), randomAsciiOfLength(10)));
                 } else if (shardRouting.unassignedInfo() != null) {
-                    shardRouting = shardRouting.updateUnassignedInfo(new UnassignedInfo(randomReason(), randomAsciiOfLength(10)));
+                    shardRouting = shardRouting.updateUnassigned(new UnassignedInfo(randomReason(), randomAsciiOfLength(10)),
+                        shardRouting.recoverySource());
                 }
                 break;
             case 1:

core/src/test/java/org/elasticsearch/cluster/routing/RecoverySourceTests.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class RecoverySourceTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        RecoverySource recoverySource = TestShardRouting.randomRecoverySource();
+        BytesStreamOutput out = new BytesStreamOutput();
+        recoverySource.writeTo(out);
+        RecoverySource serializedRecoverySource = RecoverySource.readFrom(out.bytes().streamInput());
+        assertEquals(recoverySource.getType(), serializedRecoverySource.getType());
+        assertEquals(recoverySource, serializedRecoverySource);
+    }
+
+    public void testRecoverySourceTypeOrder() {
+        assertEquals(RecoverySource.Type.EMPTY_STORE.ordinal(), 0);
+        assertEquals(RecoverySource.Type.EXISTING_STORE.ordinal(), 1);
+        assertEquals(RecoverySource.Type.PEER.ordinal(), 2);
+        assertEquals(RecoverySource.Type.SNAPSHOT.ordinal(), 3);
+        assertEquals(RecoverySource.Type.LOCAL_SHARDS.ordinal(), 4);
+        // check exhaustiveness
+        for (RecoverySource.Type type : RecoverySource.Type.values()) {
+            assertThat(type.ordinal(), greaterThanOrEqualTo(0));
+            assertThat(type.ordinal(), lessThanOrEqualTo(4));
+        }
+    }
+}

core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableGenerator.java

@@ -45,13 +45,13 @@ public class RoutingTableGenerator {
         switch (state) {
             case STARTED:
                 return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++),
-                                                        null, null, primary, ShardRoutingState.STARTED);
+                                                        null, primary, ShardRoutingState.STARTED);
             case INITIALIZING:
                 return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++),
-                                                        null, null, primary, ShardRoutingState.INITIALIZING);
+                                                        null, primary, ShardRoutingState.INITIALIZING);
             case RELOCATING:
                 return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++),
-                                                        "node_" + Integer.toString(node_id++), null, primary, ShardRoutingState.RELOCATING);
+                                                        "node_" + Integer.toString(node_id++), primary, ShardRoutingState.RELOCATING);
             default:
                 throw new ElasticsearchException("Unknown state: " + state.name());
         }
@@ -62,11 +62,11 @@ public class RoutingTableGenerator {
         final String index = indexMetaData.getIndex().getName();
         IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(new ShardId(index, "_na_", shardId));
         ShardRouting shardRouting = genShardRouting(index, shardId, true);
-        counter.update(shardRouting, indexMetaData);
+        counter.update(shardRouting);
         builder.addShard(shardRouting);
         for (int replicas = indexMetaData.getNumberOfReplicas(); replicas > 0; replicas--) {
             shardRouting = genShardRouting(index, shardId, false);
-            counter.update(shardRouting, indexMetaData);
+            counter.update(shardRouting);
             builder.addShard(shardRouting);
         }
 
@@ -104,7 +104,7 @@ public class RoutingTableGenerator {
             return ClusterHealthStatus.GREEN;
         }
 
-        public void update(ShardRouting shardRouting, IndexMetaData indexMetaData) {
+        public void update(ShardRouting shardRouting) {
             if (shardRouting.active()) {
                 active++;
                 if (shardRouting.primary()) {
@@ -119,7 +119,7 @@ public class RoutingTableGenerator {
             if (shardRouting.primary()) {
                 primaryInactive++;
                 if (inactivePrimaryCausesRed == false) {
-                    inactivePrimaryCausesRed = getInactivePrimaryHealth(shardRouting, indexMetaData) == ClusterHealthStatus.RED;
+                    inactivePrimaryCausesRed = getInactivePrimaryHealth(shardRouting) == ClusterHealthStatus.RED;
                 }
             }
             if (shardRouting.initializing()) {

core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java

@@ -19,6 +19,8 @@
 
 package org.elasticsearch.cluster.routing;
 
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
+
 /**
  * A helper class that allows access to package private APIs for testing.
  */
@@ -44,17 +46,17 @@ public class ShardRoutingHelper {
         return routing.initialize(nodeId, null, expectedSize);
     }
 
-    public static ShardRouting reinit(ShardRouting routing) {
-        return routing.reinitializeShard();
+    public static ShardRouting reinitPrimary(ShardRouting routing) {
+        return routing.reinitializePrimaryShard();
     }
 
-    public static ShardRouting reinit(ShardRouting routing, UnassignedInfo.Reason reason) {
-        return routing.reinitializeShard().updateUnassignedInfo(new UnassignedInfo(reason, "test_reinit"));
+    public static ShardRouting reinitPrimary(ShardRouting routing, UnassignedInfo.Reason reason, RecoverySource recoverySource) {
+        return routing.reinitializePrimaryShard().updateUnassigned(new UnassignedInfo(reason, "test_reinit"), recoverySource);
     }
 
-    public static ShardRouting initWithSameId(ShardRouting copy) {
-        return new ShardRouting(copy.shardId(), copy.currentNodeId(), copy.relocatingNodeId(), copy.restoreSource(),
-            copy.primary(), ShardRoutingState.INITIALIZING, new UnassignedInfo(UnassignedInfo.Reason.REINITIALIZED, null),
+    public static ShardRouting initWithSameId(ShardRouting copy, RecoverySource recoverySource) {
+        return new ShardRouting(copy.shardId(), copy.currentNodeId(), copy.relocatingNodeId(),
+            copy.primary(), ShardRoutingState.INITIALIZING, recoverySource, new UnassignedInfo(UnassignedInfo.Reason.REINITIALIZED, null),
             copy.allocationId(), copy.getExpectedShardSize());
     }
 
@@ -62,8 +64,8 @@ public class ShardRoutingHelper {
         return routing.moveToUnassigned(info);
     }
 
-    public static ShardRouting newWithRestoreSource(ShardRouting routing, RestoreSource restoreSource) {
-        return new ShardRouting(routing.shardId(), routing.currentNodeId(), routing.relocatingNodeId(), restoreSource,
-            routing.primary(), routing.state(), routing.unassignedInfo(), routing.allocationId(), routing.getExpectedShardSize());
+    public static ShardRouting newWithRestoreSource(ShardRouting routing, SnapshotRecoverySource recoverySource) {
+        return new ShardRouting(routing.shardId(), routing.currentNodeId(), routing.relocatingNodeId(), routing.primary(), routing.state(),
+            recoverySource, routing.unassignedInfo(), routing.allocationId(), routing.getExpectedShardSize());
     }
 }

core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java

@@ -21,6 +21,8 @@ package org.elasticsearch.cluster.routing;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.snapshots.Snapshot;
@@ -54,20 +56,6 @@ public class ShardRoutingTests extends ESTestCase {
         assertFalse(unassignedShard0.isSameAllocation(startedShard1));
     }
 
-    public void testIsSameShard() {
-        ShardRouting index1Shard0a = randomShardRouting("index1", 0);
-        ShardRouting index1Shard0b = randomShardRouting("index1", 0);
-        ShardRouting index1Shard1 = randomShardRouting("index1", 1);
-        ShardRouting index2Shard0 = randomShardRouting("index2", 0);
-        ShardRouting index2Shard1 = randomShardRouting("index2", 1);
-
-        assertTrue(index1Shard0a.isSameShard(index1Shard0a));
-        assertTrue(index1Shard0a.isSameShard(index1Shard0b));
-        assertFalse(index1Shard0a.isSameShard(index1Shard1));
-        assertFalse(index1Shard0a.isSameShard(index2Shard0));
-        assertFalse(index1Shard0a.isSameShard(index2Shard1));
-    }
-
     private ShardRouting randomShardRouting(String index, int shard) {
         ShardRoutingState state = randomFrom(ShardRoutingState.values());
         return TestShardRouting.newShardRouting(index, shard, state == ShardRoutingState.UNASSIGNED ? null : "1", state != ShardRoutingState.UNASSIGNED && randomBoolean(), state);
@@ -126,39 +114,49 @@ public class ShardRoutingTests extends ESTestCase {
 
         Integer[] changeIds = new Integer[]{0, 1, 2, 3, 4, 5, 6};
         for (int changeId : randomSubsetOf(randomIntBetween(1, changeIds.length), changeIds)) {
+            boolean unchanged = false;
             switch (changeId) {
                 case 0:
                     // change index
-                    otherRouting = TestShardRouting.newShardRouting(otherRouting.getIndexName() + "a", otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(),
-                            otherRouting.restoreSource(), otherRouting.primary(), otherRouting.state(), otherRouting.unassignedInfo());
+                    ShardId shardId = new ShardId(new Index("blubb", randomAsciiOfLength(10)), otherRouting.id());
+                    otherRouting = new ShardRouting(shardId, otherRouting.currentNodeId(), otherRouting.relocatingNodeId(),
+                            otherRouting.primary(), otherRouting.state(), otherRouting.recoverySource(), otherRouting.unassignedInfo(),
+                            otherRouting.allocationId(), otherRouting.getExpectedShardSize());
                     break;
                 case 1:
                     // change shard id
-                    otherRouting = TestShardRouting.newShardRouting(otherRouting.getIndexName(), otherRouting.id() + 1, otherRouting.currentNodeId(), otherRouting.relocatingNodeId(),
-                            otherRouting.restoreSource(), otherRouting.primary(), otherRouting.state(), otherRouting.unassignedInfo());
+                    otherRouting = new ShardRouting(new ShardId(otherRouting.index(), otherRouting.id() + 1), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(),
+                            otherRouting.primary(), otherRouting.state(), otherRouting.recoverySource(), otherRouting.unassignedInfo(),
+                            otherRouting.allocationId(), otherRouting.getExpectedShardSize());
                     break;
                 case 2:
                     // change current node
-                    otherRouting = TestShardRouting.newShardRouting(otherRouting.getIndexName(), otherRouting.id(), otherRouting.currentNodeId() == null ? "1" : otherRouting.currentNodeId() + "_1", otherRouting.relocatingNodeId(),
-                            otherRouting.restoreSource(), otherRouting.primary(), otherRouting.state(), otherRouting.unassignedInfo());
+                    otherRouting = new ShardRouting(otherRouting.shardId(), otherRouting.currentNodeId() == null ? "1" : otherRouting.currentNodeId() + "_1", otherRouting.relocatingNodeId(),
+                            otherRouting.primary(), otherRouting.state(), otherRouting.recoverySource(), otherRouting.unassignedInfo(),
+                            otherRouting.allocationId(), otherRouting.getExpectedShardSize());
                     break;
                 case 3:
                     // change relocating node
-                    otherRouting = TestShardRouting.newShardRouting(otherRouting.getIndexName(), otherRouting.id(), otherRouting.currentNodeId(),
+                    otherRouting = new ShardRouting(otherRouting.shardId(), otherRouting.currentNodeId(),
                             otherRouting.relocatingNodeId() == null ? "1" : otherRouting.relocatingNodeId() + "_1",
-                            otherRouting.restoreSource(), otherRouting.primary(), otherRouting.state(), otherRouting.unassignedInfo());
+                            otherRouting.primary(), otherRouting.state(), otherRouting.recoverySource(), otherRouting.unassignedInfo(),
+                            otherRouting.allocationId(), otherRouting.getExpectedShardSize());
                     break;
                 case 4:
-                    // change restore source
-                    otherRouting = TestShardRouting.newShardRouting(otherRouting.getIndexName(), otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(),
-                            otherRouting.restoreSource() == null ? new RestoreSource(new Snapshot("test", new SnapshotId("s1", UUIDs.randomBase64UUID())), Version.CURRENT, "test") :
-                                    new RestoreSource(otherRouting.restoreSource().snapshot(), Version.CURRENT, otherRouting.index() + "_1"),
-                            otherRouting.primary(), otherRouting.state(), otherRouting.unassignedInfo());
+                    // change recovery source (only works for inactive primaries)
+                    if (otherRouting.active() || otherRouting.primary() == false) {
+                        unchanged = true;
+                    } else {
+                        otherRouting = new ShardRouting(otherRouting.shardId(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(),
+                            otherRouting.primary(), otherRouting.state(),
+                            new RecoverySource.SnapshotRecoverySource(new Snapshot("test", new SnapshotId("s1", UUIDs.randomBase64UUID())), Version.CURRENT, "test"),
+                            otherRouting.unassignedInfo(), otherRouting.allocationId(), otherRouting.getExpectedShardSize());
+                    }
                     break;
                 case 5:
                     // change primary flag
                     otherRouting = TestShardRouting.newShardRouting(otherRouting.getIndexName(), otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(),
-                            otherRouting.restoreSource(), otherRouting.primary() == false, otherRouting.state(), otherRouting.unassignedInfo());
+                            otherRouting.primary() == false, otherRouting.state(), otherRouting.unassignedInfo());
                     break;
                 case 6:
                     // change state
@@ -173,20 +171,22 @@ public class ShardRoutingTests extends ESTestCase {
                     }
 
                     otherRouting = TestShardRouting.newShardRouting(otherRouting.getIndexName(), otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(),
-                            otherRouting.restoreSource(), otherRouting.primary(), newState, unassignedInfo);
+                            otherRouting.primary(), newState, unassignedInfo);
                     break;
             }
 
             if (randomBoolean()) {
                 // change unassigned info
                 otherRouting = TestShardRouting.newShardRouting(otherRouting.getIndexName(), otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(),
-                        otherRouting.restoreSource(), otherRouting.primary(), otherRouting.state(),
+                        otherRouting.primary(), otherRouting.state(),
                         otherRouting.unassignedInfo() == null ? new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "test") :
                                 new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, otherRouting.unassignedInfo().getMessage() + "_1"));
             }
 
-            logger.debug("comparing\nthis  {} to\nother {}", routing, otherRouting);
-            assertFalse("expected non-equality\nthis  " + routing + ",\nother " + otherRouting, routing.equalsIgnoringMetaData(otherRouting));
+            if (unchanged == false) {
+                logger.debug("comparing\nthis  {} to\nother {}", routing, otherRouting);
+                assertFalse("expected non-equality\nthis  " + routing + ",\nother " + otherRouting, routing.equalsIgnoringMetaData(otherRouting));
+            }
         }
     }
 

core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java

@@ -24,9 +24,11 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
@@ -38,7 +40,6 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -69,7 +70,8 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
                 UnassignedInfo.Reason.REROUTE_CANCELLED,
                 UnassignedInfo.Reason.REINITIALIZED,
                 UnassignedInfo.Reason.REALLOCATED_REPLICA,
-                UnassignedInfo.Reason.PRIMARY_FAILED};
+                UnassignedInfo.Reason.PRIMARY_FAILED,
+                UnassignedInfo.Reason.FORCED_EMPTY_PRIMARY};
         for (int i = 0; i < order.length; i++) {
             assertThat(order[i].ordinal(), equalTo(i));
         }
@@ -136,7 +138,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
                 .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
-                .routingTable(RoutingTable.builder().addAsNewRestore(metaData.index("test"), new RestoreSource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())), Version.CURRENT, "test"), new IntHashSet()).build()).build();
+                .routingTable(RoutingTable.builder().addAsNewRestore(metaData.index("test"), new SnapshotRecoverySource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())), Version.CURRENT, "test"), new IntHashSet()).build()).build();
         for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
             assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.NEW_INDEX_RESTORED));
         }
@@ -148,7 +150,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
                 .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
-                .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"), new RestoreSource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())), Version.CURRENT, "test")).build()).build();
+                .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"), new SnapshotRecoverySource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())), Version.CURRENT, "test")).build()).build();
         for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
             assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED));
         }
@@ -194,7 +196,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
      * The unassigned meta is kept when a shard goes to INITIALIZING, but cleared when it moves to STARTED.
      */
     public void testStateTransitionMetaHandling() {
-        ShardRouting shard = TestShardRouting.newShardRouting("test", 1, null, null, null, true, ShardRoutingState.UNASSIGNED, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+        ShardRouting shard = TestShardRouting.newShardRouting("test", 1, null, null, true, ShardRoutingState.UNASSIGNED, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
         assertThat(shard.unassignedInfo(), notNullValue());
         shard = shard.initialize("test_node", null, -1);
         assertThat(shard.state(), equalTo(ShardRoutingState.INITIALIZING));

core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java

@@ -53,6 +53,8 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 
+import java.util.Collections;
+
 import static java.util.Collections.singleton;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
@@ -126,7 +128,8 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
         logger.info("--> building initial routing table");
         MetaData metaData = MetaData.builder()
-                .put(IndexMetaData.builder(index).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
+                .put(IndexMetaData.builder(index).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)
+                    .putInSyncAllocationIds(0, Collections.singleton("asdf")).putInSyncAllocationIds(1, Collections.singleton("qwertz")))
                 .build();
         // shard routing is added as "from recovery" instead of "new index creation" so that we can test below that allocating an empty
         // primary with accept_data_loss flag set to false fails

core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java

@@ -82,7 +82,7 @@ public abstract class CatAllocationTestCase extends ESAllocationTestCase {
                     ShardRoutingState state = ShardRoutingState.valueOf(matcher.group(4));
                     String ip = matcher.group(5);
                     nodes.add(ip);
-                    ShardRouting routing = TestShardRouting.newShardRouting(index, shard, ip, null, null, primary, state);
+                    ShardRouting routing = TestShardRouting.newShardRouting(index, shard, ip, null, primary, state);
                     idx.add(routing);
                     logger.debug("Add routing {}", routing);
                 } else {

core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java

@@ -82,14 +82,12 @@ public class FilterAllocationDeciderTests extends ESAllocationTestCase {
         // ok now we are started and can be allocated anywhere!! lets see...
         assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), STARTED);
         assertEquals(routingTable.index("idx").shard(0).shards().get(0).currentNodeId(), "node2");
-        assertTrue(routingTable.index("idx").shard(0).shards().get(0).allocatedPostIndexCreate(state.getMetaData().index("idx")));
 
         // we fail it again to check if we are initializing immediately on the other node
         state = stateFromResult(state, service.applyFailedShard(state, routingTable.index("idx").shard(0).shards().get(0)));
         routingTable = state.routingTable();
         assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
         assertEquals(routingTable.index("idx").shard(0).shards().get(0).currentNodeId(), "node1");
-        assertTrue(routingTable.index("idx").shard(0).shards().get(0).allocatedPostIndexCreate(state.getMetaData().index("idx")));
 
         allocation = new RoutingAllocation(allocationDeciders, state.getRoutingNodes(), state,
             null, 0, false);

core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java

@@ -30,7 +30,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
-import org.elasticsearch.cluster.routing.RestoreSource;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -362,7 +362,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
         ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData)
             .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"),
-                new RestoreSource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())),
+                new SnapshotRecoverySource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())),
                 Version.CURRENT, "test")).build())
             .nodes(DiscoveryNodes.builder().add(newNode).add(oldNode1).add(oldNode2)).build();
         AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY, Arrays.asList(

core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java

@@ -25,7 +25,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RestoreSource;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
@@ -330,10 +330,10 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
             case 1: routingTableBuilder.addAsFromCloseToOpen(indexMetaData); break;
             case 2: routingTableBuilder.addAsFromDangling(indexMetaData); break;
             case 3: routingTableBuilder.addAsNewRestore(indexMetaData,
-                new RestoreSource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT,
+                new SnapshotRecoverySource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT,
                 indexMetaData.getIndex().getName()), new IntHashSet()); break;
             case 4: routingTableBuilder.addAsRestore(indexMetaData,
-                new RestoreSource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT,
+                new SnapshotRecoverySource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT,
                 indexMetaData.getIndex().getName())); break;
             case 5: routingTableBuilder.addAsNew(indexMetaData); break;
             default: throw new IndexOutOfBoundsException();

core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java

@@ -858,8 +858,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .build();
 
         // Two shards consuming each 80% of disk space while 70% is allowed, so shard 0 isn't allowed here
-        ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, null, true, ShardRoutingState.STARTED);
-        ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", null, null, true, ShardRoutingState.STARTED);
+        ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, true, ShardRoutingState.STARTED);
+        ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", null, true, ShardRoutingState.STARTED);
         RoutingNode firstRoutingNode = new RoutingNode("node1", discoveryNode1, firstRouting, secondRouting);
         RoutingTable.Builder builder = RoutingTable.builder().add(
                 IndexRoutingTable.builder(firstRouting.index())
@@ -879,8 +879,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         assertThat(decision.type(), equalTo(Decision.Type.NO));
 
         // Two shards consuming each 80% of disk space while 70% is allowed, but one is relocating, so shard 0 can stay
-        firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, null, true, ShardRoutingState.STARTED);
-        secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", "node2", null, true, ShardRoutingState.RELOCATING);
+        firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, true, ShardRoutingState.STARTED);
+        secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", "node2", true, ShardRoutingState.RELOCATING);
         firstRoutingNode = new RoutingNode("node1", discoveryNode1, firstRouting, secondRouting);
         builder = RoutingTable.builder().add(
                 IndexRoutingTable.builder(firstRouting.index())
@@ -974,8 +974,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .build();
 
         // Two shards consumes 80% of disk space in data node, but we have only one data node, shards should remain.
-        ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node2", null, null, true, ShardRoutingState.STARTED);
-        ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node2", null, null, true, ShardRoutingState.STARTED);
+        ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node2", null, true, ShardRoutingState.STARTED);
+        ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node2", null, true, ShardRoutingState.STARTED);
         RoutingNode firstRoutingNode = new RoutingNode("node2", discoveryNode2, firstRouting, secondRouting);
 
         RoutingTable.Builder builder = RoutingTable.builder().add(
@@ -1034,8 +1034,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         ClusterState updateClusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(discoveryNode3)).build();
 
-        firstRouting = TestShardRouting.newShardRouting("test", 0, "node2", null, null, true, ShardRoutingState.STARTED);
-        secondRouting = TestShardRouting.newShardRouting("test", 1, "node2", "node3", null, true, ShardRoutingState.RELOCATING);
+        firstRouting = TestShardRouting.newShardRouting("test", 0, "node2", null, true, ShardRoutingState.STARTED);
+        secondRouting = TestShardRouting.newShardRouting("test", 1, "node2", "node3", true, ShardRoutingState.RELOCATING);
         firstRoutingNode = new RoutingNode("node2", discoveryNode2, firstRouting, secondRouting);
         builder = RoutingTable.builder().add(
                 IndexRoutingTable.builder(firstRouting.index())

core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java

@@ -28,6 +28,9 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -66,7 +69,7 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
 
         final Index index = metaData.index("test").getIndex();
 
-        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         DiscoveryNode node_0 = new DiscoveryNode("node_0", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                 new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
         DiscoveryNode node_1 = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
@@ -115,22 +118,22 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
                 .build();
         final IndexMetaData indexMetaData = metaData.index("test");
 
-        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 0), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_0 = ShardRoutingHelper.initialize(test_0, node_0.getId());
         test_0 = ShardRoutingHelper.moveToStarted(test_0);
         shardRoutingMap.put(test_0, "/node0/least");
 
-        ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 1), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 1), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_1 = ShardRoutingHelper.initialize(test_1, node_1.getId());
         test_1 = ShardRoutingHelper.moveToStarted(test_1);
         shardRoutingMap.put(test_1, "/node1/least");
 
-        ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 2), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 2), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_2 = ShardRoutingHelper.initialize(test_2, node_1.getId());
         test_2 = ShardRoutingHelper.moveToStarted(test_2);
         shardRoutingMap.put(test_2, "/node1/most");
 
-        ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 3), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 3), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_3 = ShardRoutingHelper.initialize(test_3, node_1.getId());
         test_3 = ShardRoutingHelper.moveToStarted(test_3);
         // Intentionally not in the shardRoutingMap. We want to test what happens when we don't know where it is.
@@ -203,17 +206,17 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
         RoutingAllocation allocation = new RoutingAllocation(null, null, clusterState, info, 0, false);
 
         final Index index = new Index("test", "1234");
-        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_0 = ShardRoutingHelper.initialize(test_0, "node1");
         test_0 = ShardRoutingHelper.moveToStarted(test_0);
         test_0 = ShardRoutingHelper.relocate(test_0, "node2");
 
-        ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_1 = ShardRoutingHelper.initialize(test_1, "node2");
         test_1 = ShardRoutingHelper.moveToStarted(test_1);
         test_1 = ShardRoutingHelper.relocate(test_1, "node1");
 
-        ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(index, 2), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(index, 2), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_2 = ShardRoutingHelper.initialize(test_2, "node1");
         test_2 = ShardRoutingHelper.moveToStarted(test_2);
 
@@ -228,13 +231,13 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
         assertEquals(0L, DiskThresholdDecider.sizeOfRelocatingShards(node, allocation, true, "/dev/some/other/dev"));
         assertEquals(0L, DiskThresholdDecider.sizeOfRelocatingShards(node, allocation, true, "/dev/some/other/dev"));
 
-        ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(index, 3), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(index, 3), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_3 = ShardRoutingHelper.initialize(test_3, "node1");
         test_3 = ShardRoutingHelper.moveToStarted(test_3);
         assertEquals(0L, DiskThresholdDecider.getExpectedShardSize(test_3, allocation, 0));
 
 
-        ShardRouting other_0 = ShardRouting.newUnassigned(new ShardId("other", "5678", 0), null, randomBoolean(), new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting other_0 = ShardRouting.newUnassigned(new ShardId("other", "5678", 0), randomBoolean(), PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         other_0 = ShardRoutingHelper.initialize(other_0, "node2");
         other_0 = ShardRoutingHelper.moveToStarted(other_0);
         other_0 = ShardRoutingHelper.relocate(other_0, "node1");
@@ -286,22 +289,22 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
         RoutingAllocation allocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, info, 0, false);
 
         final Index index = new Index("test", "1234");
-        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, true,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true,
+            LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_0 = ShardRoutingHelper.initialize(test_0, "node1");
         test_0 = ShardRoutingHelper.moveToStarted(test_0);
 
-        ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), null, true,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), true,
+            LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_1 = ShardRoutingHelper.initialize(test_1, "node2");
         test_1 = ShardRoutingHelper.moveToStarted(test_1);
 
-        ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(index, 2), null, true,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(index, 2), true,
+            LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_2 = ShardRoutingHelper.initialize(test_2, "node1");
 
-        ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(index, 3), null, true,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(index, 3), true,
+            LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_3 = ShardRoutingHelper.initialize(test_3, "node1");
         assertEquals(500L, DiskThresholdDecider.getExpectedShardSize(test_3, allocation, 0));
         assertEquals(500L, DiskThresholdDecider.getExpectedShardSize(test_2, allocation, 0));
@@ -310,15 +313,15 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
 
 
         ShardRouting target = ShardRouting.newUnassigned(new ShardId(new Index("target", "5678"), 0),
-            null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+            true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         assertEquals(1110L, DiskThresholdDecider.getExpectedShardSize(target, allocation, 0));
 
         ShardRouting target2 = ShardRouting.newUnassigned(new ShardId(new Index("target2", "9101112"), 0),
-            null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+            true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         assertEquals(110L, DiskThresholdDecider.getExpectedShardSize(target2, allocation, 0));
 
         target2 = ShardRouting.newUnassigned(new ShardId(new Index("target2", "9101112"), 1),
-            null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+            true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         assertEquals(1000L, DiskThresholdDecider.getExpectedShardSize(target2, allocation, 0));
     }
 

core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java

@@ -606,16 +606,16 @@ public class NodeJoinControllerTests extends ESTestCase {
             final DiscoveryNode primaryNode = randomBoolean() ? masterNode : otherNode;
             final DiscoveryNode replicaNode = primaryNode.equals(masterNode) ? otherNode : masterNode;
             final boolean primaryStarted = randomBoolean();
-            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting("test", 0, primaryNode.getId(), null, null, true,
+            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting("test", 0, primaryNode.getId(), null, true,
                 primaryStarted ? ShardRoutingState.STARTED : ShardRoutingState.INITIALIZING,
                 primaryStarted ? null : new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "getting there")));
             if (primaryStarted) {
                 boolean replicaStared = randomBoolean();
-                indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting("test", 0, replicaNode.getId(), null, null, false,
+                indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting("test", 0, replicaNode.getId(), null, false,
                     replicaStared ? ShardRoutingState.STARTED : ShardRoutingState.INITIALIZING,
                     replicaStared ? null : new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "getting there")));
             } else {
-                indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting("test", 0, null, null, null, false,
+                indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting("test", 0, null, null, false,
                     ShardRoutingState.UNASSIGNED, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "life sucks")));
             }
             indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder.build());

core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java

@@ -29,8 +29,8 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RestoreSource;
 import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -436,7 +436,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
 
         final Snapshot snapshot = new Snapshot("test", new SnapshotId("test", UUIDs.randomBase64UUID()));
         RoutingTable routingTable = RoutingTable.builder()
-            .addAsRestore(metaData.index(shardId.getIndex()), new RestoreSource(snapshot, version, shardId.getIndexName()))
+            .addAsRestore(metaData.index(shardId.getIndex()), new SnapshotRecoverySource(snapshot, version, shardId.getIndexName()))
             .build();
         ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData)
@@ -519,7 +519,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
             .build();
 
         RoutingTable routingTable = RoutingTable.builder()
-            .addAsRestore(metaData.index(shardId.getIndex()), new RestoreSource(new Snapshot("test", new SnapshotId("test", UUIDs.randomBase64UUID())), Version.CURRENT, shardId.getIndexName()))
+            .addAsRestore(metaData.index(shardId.getIndex()), new SnapshotRecoverySource(new Snapshot("test", new SnapshotId("test", UUIDs.randomBase64UUID())), Version.CURRENT, shardId.getIndexName()))
             .build();
         ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData)

core/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java

@@ -41,8 +41,10 @@ public class PriorityComparatorTests extends ESTestCase {
 
     public void testPreferNewIndices() {
         RoutingNodes.UnassignedShards shards = new RoutingNodes.UnassignedShards(mock(RoutingNodes.class));
-        List<ShardRouting> shardRoutings = Arrays.asList(TestShardRouting.newShardRouting("oldest", 0, null, null, null,
-                randomBoolean(), ShardRoutingState.UNASSIGNED, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")), TestShardRouting.newShardRouting("newest", 0, null, null, null,
+        List<ShardRouting> shardRoutings = Arrays.asList(
+            TestShardRouting.newShardRouting("oldest", 0, null, null,
+                randomBoolean(), ShardRoutingState.UNASSIGNED, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")),
+            TestShardRouting.newShardRouting("newest", 0, null, null,
                 randomBoolean(), ShardRoutingState.UNASSIGNED, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")));
         Collections.shuffle(shardRoutings, random());
         for (ShardRouting routing : shardRoutings) {
@@ -71,8 +73,10 @@ public class PriorityComparatorTests extends ESTestCase {
 
     public void testPreferPriorityIndices() {
         RoutingNodes.UnassignedShards shards = new RoutingNodes.UnassignedShards(mock(RoutingNodes.class));
-        List<ShardRouting> shardRoutings = Arrays.asList(TestShardRouting.newShardRouting("oldest", 0, null, null, null,
-                randomBoolean(), ShardRoutingState.UNASSIGNED, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")), TestShardRouting.newShardRouting("newest", 0, null, null, null,
+        List<ShardRouting> shardRoutings = Arrays.asList(
+            TestShardRouting.newShardRouting("oldest", 0, null, null,
+                randomBoolean(), ShardRoutingState.UNASSIGNED, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")),
+            TestShardRouting.newShardRouting("newest", 0, null, null,
                 randomBoolean(), ShardRoutingState.UNASSIGNED, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")));
         Collections.shuffle(shardRoutings, random());
         for (ShardRouting routing : shardRoutings) {
@@ -116,7 +120,7 @@ public class PriorityComparatorTests extends ESTestCase {
         int numShards = randomIntBetween(10, 100);
         for (int i = 0; i < numShards; i++) {
             IndexMeta indexMeta = randomFrom(indices);
-            shards.add(TestShardRouting.newShardRouting(indexMeta.name, randomIntBetween(1, 5), null, null, null,
+            shards.add(TestShardRouting.newShardRouting(indexMeta.name, randomIntBetween(1, 5), null, null,
                     randomBoolean(), ShardRoutingState.UNASSIGNED, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")));
         }
         shards.sort(new PriorityComparator() {

core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java

@@ -29,6 +29,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
@@ -298,9 +299,11 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                 .add(IndexRoutingTable.builder(shardId.getIndex())
                                 .addIndexShard(new IndexShardRoutingTable.Builder(shardId)
                                         .addShard(primaryShard)
-                                        .addShard(ShardRouting.newUnassigned(shardId, null, false,
+                                        .addShard(ShardRouting.newUnassigned(shardId, false,
+                                            RecoverySource.PeerRecoverySource.INSTANCE,
                                             new UnassignedInfo(reason, null, null, failedAllocations, System.nanoTime(),
-                                                System.currentTimeMillis(), delayed, UnassignedInfo.AllocationStatus.NO_ATTEMPT)))
+                                                System.currentTimeMillis(), delayed, UnassignedInfo.AllocationStatus.NO_ATTEMPT)
+                                            ))
                                         .build())
                 )
                 .build();
@@ -322,7 +325,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                 .add(IndexRoutingTable.builder(shardId.getIndex())
                                 .addIndexShard(new IndexShardRoutingTable.Builder(shardId)
                                         .addShard(primaryShard)
-                                        .addShard(TestShardRouting.newShardRouting(shardId, node2.getId(), null, null, false, ShardRoutingState.INITIALIZING, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)))
+                                        .addShard(TestShardRouting.newShardRouting(shardId, node2.getId(), null, false, ShardRoutingState.INITIALIZING, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)))
                                         .build())
                 )
                 .build();

core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java

@@ -48,7 +48,7 @@ import org.elasticsearch.index.shard.ShadowIndexShard;
 import org.elasticsearch.index.store.FsDirectoryService;
 import org.elasticsearch.index.translog.TranslogStats;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.sort.SortOrder;
@@ -495,7 +495,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
                     public void sendRequest(DiscoveryNode node, long requestId, String action,
                                             TransportRequest request, TransportRequestOptions options)
                             throws IOException, TransportException {
-                        if (keepFailing.get() && action.equals(RecoveryTargetService.Actions.TRANSLOG_OPS)) {
+                        if (keepFailing.get() && action.equals(PeerRecoveryTargetService.Actions.TRANSLOG_OPS)) {
                             logger.info("--> failing translog ops");
                             throw new ElasticsearchException("failing on purpose");
                         }

core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java

@@ -41,6 +41,8 @@ import org.elasticsearch.action.support.replication.TransportWriteAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingHelper;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -73,7 +75,7 @@ import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoverySourceHandler;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.recovery.RecoveryTarget;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.recovery.StartRecoveryRequest;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
@@ -111,7 +113,7 @@ public abstract class ESIndexLevelReplicationTestCase extends ESTestCase {
     private final Index index = new Index("test", "uuid");
     private final ShardId shardId = new ShardId(index, 0);
     private final Map<String, String> indexMapping = Collections.singletonMap("type", "{ \"type\": {} }");
-    protected static final RecoveryTargetService.RecoveryListener recoveryListener = new RecoveryTargetService.RecoveryListener() {
+    protected static final PeerRecoveryTargetService.RecoveryListener recoveryListener = new PeerRecoveryTargetService.RecoveryListener() {
         @Override
         public void onRecoveryDone(RecoveryState state) {
 
@@ -208,7 +210,8 @@ public abstract class ESIndexLevelReplicationTestCase extends ESTestCase {
         // add node name to settings for propper logging
         final Settings nodeSettings = Settings.builder().put("node.name", node.getName()).build();
         final IndexSettings indexSettings = new IndexSettings(indexMetaData, nodeSettings);
-        ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, node.getId(), primary, ShardRoutingState.INITIALIZING);
+        ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, node.getId(), primary, ShardRoutingState.INITIALIZING,
+            primary ? StoreRecoverySource.EMPTY_STORE_INSTANCE : PeerRecoverySource.INSTANCE);
         final Path path = Files.createDirectories(homePath.resolve(node.getId()));
         final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(path);
         ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
@@ -268,7 +271,7 @@ public abstract class ESIndexLevelReplicationTestCase extends ESTestCase {
 
         public synchronized void startAll() throws IOException {
             final DiscoveryNode pNode = getDiscoveryNode(primary.routingEntry().currentNodeId());
-            primary.markAsRecovering("store", new RecoveryState(primary.shardId(), true, RecoveryState.Type.STORE, pNode, pNode));
+            primary.markAsRecovering("store", new RecoveryState(primary.routingEntry(), pNode, null));
             primary.recoverFromStore();
             primary.updateRoutingEntry(ShardRoutingHelper.moveToStarted(primary.routingEntry()));
             for (IndexShard replicaShard : replicas) {
@@ -293,14 +296,15 @@ public abstract class ESIndexLevelReplicationTestCase extends ESTestCase {
             final DiscoveryNode pNode = getPrimaryNode();
             final DiscoveryNode rNode = getDiscoveryNode(replica.routingEntry().currentNodeId());
             if (markAsRecovering) {
-                replica.markAsRecovering("remote", new RecoveryState(replica.shardId(), false, RecoveryState.Type.REPLICA, pNode, rNode));
+                replica.markAsRecovering("remote",
+                    new RecoveryState(replica.routingEntry(), pNode, rNode));
             } else {
                 assertEquals(replica.state(), IndexShardState.RECOVERING);
             }
             replica.prepareForIndexRecovery();
             RecoveryTarget recoveryTarget = targetSupplier.apply(replica, pNode);
             StartRecoveryRequest request = new StartRecoveryRequest(replica.shardId(), pNode, rNode,
-                getMetadataSnapshotOrEmpty(replica), RecoveryState.Type.REPLICA, 0);
+                getMetadataSnapshotOrEmpty(replica), false, 0);
             RecoverySourceHandler recovery = new RecoverySourceHandler(primary, recoveryTarget, request, () -> 0L, e -> () -> {},
                 (int) ByteSizeUnit.MB.toKB(1), logger);
             recovery.recoverToTarget();

core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java

@@ -25,7 +25,7 @@ import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.recovery.RecoveryTarget;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 
 import java.io.IOException;
 import java.util.EnumSet;
@@ -65,7 +65,7 @@ public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestC
         private final ESLogger logger;
 
         BlockingTarget(RecoveryState.Stage stageToBlock, CountDownLatch recoveryBlocked, CountDownLatch releaseRecovery, IndexShard shard,
-                       DiscoveryNode sourceNode, RecoveryTargetService.RecoveryListener listener, ESLogger logger) {
+                       DiscoveryNode sourceNode, PeerRecoveryTargetService.RecoveryListener listener, ESLogger logger) {
             super(shard, sourceNode, listener, version -> {});
             this.recoveryBlocked = recoveryBlocked;
             this.releaseRecovery = releaseRecovery;

core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java

@@ -55,7 +55,9 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.AllocationId;
-import org.elasticsearch.cluster.routing.RestoreSource;
+import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingHelper;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -518,16 +520,14 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         client().admin().indices().prepareFlush().get();
         ShardRouting routing = shard.routingEntry();
         test.removeShard(0, "b/c simon says so");
-        routing = ShardRoutingHelper.reinit(routing);
+        routing = routing.reinitializePrimaryShard();
         IndexShard newShard = test.createShard(routing);
-        newShard.updateRoutingEntry(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
 
         Store.MetadataSnapshot snapshot = newShard.snapshotStoreMetadata();
         assertThat(snapshot.getSegmentsFile().name(), equalTo("segments_2"));
 
-        newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode,
-            localNode));
+        newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null));
 
         snapshot = newShard.snapshotStoreMetadata();
         assertThat(snapshot.getSegmentsFile().name(), equalTo("segments_2"));
@@ -1115,11 +1115,11 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         }
         ShardRouting routing = shard.routingEntry();
         test.removeShard(0, "b/c simon says so");
-        routing = ShardRoutingHelper.reinit(routing);
+        routing = ShardRoutingHelper.reinitPrimary(routing);
         IndexShard newShard = test.createShard(routing);
         newShard.updateRoutingEntry(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
+        newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
         assertTrue(newShard.recoverFromStore());
         assertEquals(translogOps, newShard.recoveryState().getTranslog().recoveredOperations());
         assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperations());
@@ -1142,12 +1142,11 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         }
         ShardRouting routing = shard.routingEntry();
         test.removeShard(0, "b/c simon says so");
-        routing = ShardRoutingHelper.reinit(routing, UnassignedInfo.Reason.INDEX_CREATED);
+        routing = ShardRoutingHelper.reinitPrimary(routing, UnassignedInfo.Reason.INDEX_CREATED, StoreRecoverySource.EMPTY_STORE_INSTANCE);
         IndexShard newShard = test.createShard(routing);
         newShard.updateRoutingEntry(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode,
-            localNode));
+        newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
         assertTrue(newShard.recoverFromStore());
         assertEquals(0, newShard.recoveryState().getTranslog().recoveredOperations());
         assertEquals(0, newShard.recoveryState().getTranslog().totalOperations());
@@ -1177,10 +1176,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         test.removeShard(0, "b/c simon says so");
         cleanLuceneIndex(store.directory());
         store.decRef();
-        routing = ShardRoutingHelper.reinit(routing);
+        routing = ShardRoutingHelper.reinitPrimary(routing);
         IndexShard newShard = test.createShard(routing);
-        newShard.updateRoutingEntry(routing);
-        newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
+        newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
         try {
             newShard.recoverFromStore();
             fail("index not there!");
@@ -1192,15 +1190,15 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         routing = ShardRoutingHelper.initialize(routing, origRouting.currentNodeId());
         assertTrue("it's already recovering, we should ignore new ones", newShard.ignoreRecoveryAttempt());
         try {
-            newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
+            newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
             fail("we are already recovering, can't mark again");
         } catch (IllegalIndexShardStateException e) {
             // OK!
         }
         test.removeShard(0, "I broken it");
+        routing = routing.updateUnassigned(routing.unassignedInfo(), StoreRecoverySource.EMPTY_STORE_INSTANCE);
         newShard = test.createShard(routing);
-        newShard.updateRoutingEntry(routing);
-        newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
+        newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
         assertTrue("recover even if there is nothing to recover", newShard.recoverFromStore());
 
         newShard.updateRoutingEntry(getInitializingShardRouting(routing).moveToStarted());
@@ -1249,9 +1247,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertSearchHits(client().prepareSearch("test_target").get(), "1");
         client().admin().indices().prepareFlush("test").get(); // only flush test
         final ShardRouting origRouting = test_target.getShardOrNull(0).routingEntry();
-        ShardRouting routing = ShardRoutingHelper.reinit(origRouting);
+        ShardRouting routing = ShardRoutingHelper.reinitPrimary(origRouting);
         final Snapshot snapshot = new Snapshot("foo", new SnapshotId("bar", UUIDs.randomBase64UUID()));
-        routing = ShardRoutingHelper.newWithRestoreSource(routing, new RestoreSource(snapshot, Version.CURRENT, "test"));
+        routing = ShardRoutingHelper.newWithRestoreSource(routing, new SnapshotRecoverySource(snapshot, Version.CURRENT, "test"));
         test_target.removeShard(0, "just do it man!");
         final IndexShard test_target_shard = test_target.createShard(routing);
         Store sourceStore = test_shard.store();
@@ -1259,7 +1257,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
 
         test_target_shard.updateRoutingEntry(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        test_target_shard.markAsRecovering("store", new RecoveryState(routing.shardId(), routing.primary(), RecoveryState.Type.SNAPSHOT, routing.restoreSource(), localNode));
+        test_target_shard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
         assertTrue(test_target_shard.restoreFromRepository(new RestoreOnlyRepository("test") {
             @Override
             public void restoreShard(IndexShard shard, SnapshotId snapshotId, Version version, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState) {
@@ -1530,7 +1528,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
 
     public  static final IndexShard recoverShard(IndexShard newShard) throws IOException {
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), newShard.routingEntry().primary(), RecoveryState.Type.STORE, localNode, localNode));
+        newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null));
         assertTrue(newShard.recoverFromStore());
         newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted());
         return newShard;
@@ -1549,7 +1547,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         ShardRouting shardRouting = TestShardRouting.newShardRouting(existingShardRouting.shardId(),
             existingShardRouting.currentNodeId(), null, existingShardRouting.primary(), ShardRoutingState.INITIALIZING,
             existingShardRouting.allocationId());
-        shardRouting = shardRouting.updateUnassignedInfo(new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "fake recovery"));
+        shardRouting = shardRouting.updateUnassigned(new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "fake recovery"),
+            StoreRecoverySource.EXISTING_STORE_INSTANCE);
         return shardRouting;
     }
 
@@ -1570,7 +1569,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         test.removeShard(0, "b/c britta says so");
         IndexShard newShard = test.createShard(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("for testing", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.REPLICA, localNode, localNode));
+        newShard.markAsRecovering("for testing", new RecoveryState(routing, localNode, null));
         List<Translog.Operation> operations = new ArrayList<>();
         operations.add(new Translog.Index("testtype", "1", BytesReference.toBytes(jsonBuilder().startObject().field("foo", "bar").endObject().bytes())));
         newShard.prepareForIndexRecovery();
@@ -1598,7 +1597,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndexShard newShard = test.createShard(routing);
         newShard.shardRouting = routing;
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("for testing", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.REPLICA, localNode, localNode));
+        newShard.markAsRecovering("for testing", new RecoveryState(routing, localNode, null));
         // Shard is still inactive since we haven't started recovering yet
         assertFalse(newShard.isActive());
         newShard.prepareForIndexRecovery();
@@ -1626,7 +1625,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         test.removeShard(0, "b/c britta says so");
         IndexShard newShard = test.createShard(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("for testing", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.REPLICA, localNode, localNode));
+        newShard.markAsRecovering("for testing", new RecoveryState(routing, localNode, null));
         // Shard is still inactive since we haven't started recovering yet
         assertFalse(newShard.isActive());
         List<Translog.Operation> operations = new ArrayList<>();
@@ -1658,13 +1657,12 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("index_1"));
         IndexShard shard = test.getShardOrNull(0);
-        ShardRouting routing = ShardRoutingHelper.initWithSameId(shard.routingEntry());
+        ShardRouting routing = ShardRoutingHelper.initWithSameId(shard.routingEntry(), LocalShardsRecoverySource.INSTANCE);
         test.removeShard(0, "b/c simon says so");
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         {
             final IndexShard newShard = test.createShard(routing);
-            newShard.updateRoutingEntry(routing);
-            newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.LOCAL_SHARDS, localNode, localNode));
+            newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
 
             BiConsumer<String, MappingMetaData> mappingConsumer = (type, mapping) -> {
                 try {
@@ -1702,9 +1700,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         {
             routing = shard.routingEntry();
             test.removeShard(0, "b/c simon says so");
-            routing = ShardRoutingHelper.reinit(routing);
+            routing = ShardRoutingHelper.reinitPrimary(routing);
             final IndexShard newShard = test.createShard(routing);
-            newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.LOCAL_SHARDS, localNode, localNode));
+            newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
             assertTrue(newShard.recoverFromStore());
             routing = ShardRoutingHelper.moveToStarted(routing);
             newShard.updateRoutingEntry(routing);

core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java

@@ -61,7 +61,7 @@ import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
@@ -347,7 +347,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
 
                 @Override
                 public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
-                    if (corrupt.get() && action.equals(RecoveryTargetService.Actions.FILE_CHUNK)) {
+                    if (corrupt.get() && action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                         RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
                         byte[] array = BytesRef.deepCopyOf(req.content().toBytesRef()).bytes;
                         int i = randomIntBetween(0, req.content().length() - 1);
@@ -419,7 +419,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
 
                 @Override
                 public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
-                    if (action.equals(RecoveryTargetService.Actions.FILE_CHUNK)) {
+                    if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                         RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
                         if (truncate && req.length() > 1) {
                             BytesRef bytesRef = req.content().toBytesRef();

core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java

@@ -21,6 +21,7 @@ package org.elasticsearch.indices;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
@@ -448,7 +449,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
             assertEquals(0, imc.availableShards().size());
             ShardRouting routing = newShard.routingEntry();
             DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
-            newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
+            newShard.markAsRecovering("store", new RecoveryState(routing, localNode, null));
 
             assertEquals(1, imc.availableShards().size());
             assertTrue(newShard.recoverFromStore());

core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java

@@ -21,6 +21,7 @@ package org.elasticsearch.indices;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingHelper;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -99,13 +100,15 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas
             idx = index.index();
             ShardRouting newRouting = shardRouting;
             String nodeId = newRouting.currentNodeId();
-            newRouting = ShardRoutingHelper.moveToUnassigned(newRouting, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom"));
+            UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom");
+            newRouting = newRouting.moveToUnassigned(unassignedInfo)
+                .updateUnassigned(unassignedInfo, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE);
             newRouting = ShardRoutingHelper.initialize(newRouting, nodeId);
             IndexShard shard = index.createShard(newRouting);
             shard.updateRoutingEntry(newRouting);
             final DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(),
                     emptyMap(), emptySet(), Version.CURRENT);
-            shard.markAsRecovering("store", new RecoveryState(shard.shardId(), newRouting.primary(), RecoveryState.Type.SNAPSHOT, newRouting.restoreSource(), localNode));
+            shard.markAsRecovering("store", new RecoveryState(newRouting, localNode, null));
             shard.recoverFromStore();
             newRouting = ShardRoutingHelper.moveToStarted(newRouting);
             shard.updateRoutingEntry(newRouting);

core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java

@@ -40,7 +40,7 @@ import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedInd
 import org.elasticsearch.indices.cluster.IndicesClusterStateService.Shard;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -207,8 +207,9 @@ public abstract class AbstractIndicesClusterStateServiceTestCase extends ESTestC
 
         @Override
         public MockIndexShard createShard(ShardRouting shardRouting, RecoveryState recoveryState,
-                                          RecoveryTargetService recoveryTargetService,
-                                          RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
+                                          PeerRecoveryTargetService recoveryTargetService,
+                                          PeerRecoveryTargetService.RecoveryListener recoveryListener,
+                                          RepositoriesService repositoriesService,
                                           NodeServicesProvider nodeServicesProvider, Callback<IndexShard.ShardFailure> onShardFailure)
             throws IOException {
             failRandomly();

core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java

@@ -45,7 +45,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -340,7 +340,7 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice
         final ClusterService clusterService = mock(ClusterService.class);
         final RepositoriesService repositoriesService = new RepositoriesService(Settings.EMPTY, clusterService,
             transportService, null);
-        final RecoveryTargetService recoveryTargetService = new RecoveryTargetService(Settings.EMPTY, threadPool,
+        final PeerRecoveryTargetService recoveryTargetService = new PeerRecoveryTargetService(Settings.EMPTY, threadPool,
             transportService, null, clusterService);
         final ShardStateAction shardStateAction = mock(ShardStateAction.class);
         return new IndicesClusterStateService(Settings.EMPTY, indicesService, clusterService,

core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.indices.recovery;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
@@ -32,6 +33,10 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
@@ -42,8 +47,8 @@ import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.recovery.RecoveryState.Stage;
-import org.elasticsearch.indices.recovery.RecoveryState.Type;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -99,10 +104,11 @@ public class IndexRecoveryIT extends ESIntegTestCase {
         return Arrays.asList(MockTransportService.TestPlugin.class, MockFSIndexStore.TestPlugin.class);
     }
 
-    private void assertRecoveryStateWithoutStage(RecoveryState state, int shardId, Type type,
-                                                 String sourceNode, String targetNode, boolean hasRestoreSource) {
+    private void assertRecoveryStateWithoutStage(RecoveryState state, int shardId, RecoverySource recoverySource, boolean primary,
+                                                 String sourceNode, String targetNode) {
         assertThat(state.getShardId().getId(), equalTo(shardId));
-        assertThat(state.getType(), equalTo(type));
+        assertThat(state.getRecoverySource(), equalTo(recoverySource));
+        assertThat(state.getPrimary(), equalTo(primary));
         if (sourceNode == null) {
             assertNull(state.getSourceNode());
         } else {
@@ -115,23 +121,17 @@ public class IndexRecoveryIT extends ESIntegTestCase {
             assertNotNull(state.getTargetNode());
             assertThat(state.getTargetNode().getName(), equalTo(targetNode));
         }
-        if (hasRestoreSource) {
-            assertNotNull(state.getRestoreSource());
-        } else {
-            assertNull(state.getRestoreSource());
-        }
-
     }
 
-    private void assertRecoveryState(RecoveryState state, int shardId, Type type, Stage stage,
-                                     String sourceNode, String targetNode, boolean hasRestoreSource) {
-        assertRecoveryStateWithoutStage(state, shardId, type, sourceNode, targetNode, hasRestoreSource);
+    private void assertRecoveryState(RecoveryState state, int shardId, RecoverySource type, boolean primary, Stage stage,
+                                     String sourceNode, String targetNode) {
+        assertRecoveryStateWithoutStage(state, shardId, type, primary, sourceNode, targetNode);
         assertThat(state.getStage(), equalTo(stage));
     }
 
-    private void assertOnGoingRecoveryState(RecoveryState state, int shardId, Type type,
-                                            String sourceNode, String targetNode, boolean hasRestoreSource) {
-        assertRecoveryStateWithoutStage(state, shardId, type, sourceNode, targetNode, hasRestoreSource);
+    private void assertOnGoingRecoveryState(RecoveryState state, int shardId, RecoverySource type, boolean primary,
+                                            String sourceNode, String targetNode) {
+        assertRecoveryStateWithoutStage(state, shardId, type, primary, sourceNode, targetNode);
         assertThat(state.getStage(), not(equalTo(Stage.DONE)));
     }
 
@@ -179,7 +179,7 @@ public class IndexRecoveryIT extends ESIntegTestCase {
 
         RecoveryState recoveryState = recoveryStates.get(0);
 
-        assertRecoveryState(recoveryState, 0, Type.STORE, Stage.DONE, node, node, false);
+        assertRecoveryState(recoveryState, 0, StoreRecoverySource.EXISTING_STORE_INSTANCE, true, Stage.DONE, null, node);
 
         validateIndexRecoveryState(recoveryState.getIndex());
     }
@@ -232,12 +232,12 @@ public class IndexRecoveryIT extends ESIntegTestCase {
 
         // validate node A recovery
         RecoveryState nodeARecoveryState = nodeAResponses.get(0);
-        assertRecoveryState(nodeARecoveryState, 0, Type.STORE, Stage.DONE, nodeA, nodeA, false);
+        assertRecoveryState(nodeARecoveryState, 0, StoreRecoverySource.EMPTY_STORE_INSTANCE, true, Stage.DONE, null, nodeA);
         validateIndexRecoveryState(nodeARecoveryState.getIndex());
 
         // validate node B recovery
         RecoveryState nodeBRecoveryState = nodeBResponses.get(0);
-        assertRecoveryState(nodeBRecoveryState, 0, Type.REPLICA, Stage.DONE, nodeA, nodeB, false);
+        assertRecoveryState(nodeBRecoveryState, 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeA, nodeB);
         validateIndexRecoveryState(nodeBRecoveryState.getIndex());
     }
 
@@ -285,10 +285,10 @@ public class IndexRecoveryIT extends ESIntegTestCase {
         List<RecoveryState> nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
         assertThat(nodeBRecoveryStates.size(), equalTo(1));
 
-        assertRecoveryState(nodeARecoveryStates.get(0), 0, Type.STORE, Stage.DONE, nodeA, nodeA, false);
+        assertRecoveryState(nodeARecoveryStates.get(0), 0, StoreRecoverySource.EMPTY_STORE_INSTANCE, true, Stage.DONE, null, nodeA);
         validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
 
-        assertOnGoingRecoveryState(nodeBRecoveryStates.get(0), 0, Type.PRIMARY_RELOCATION, nodeA, nodeB, false);
+        assertOnGoingRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, nodeA, nodeB);
         validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
 
         logger.info("--> request node recovery stats");
@@ -341,7 +341,7 @@ public class IndexRecoveryIT extends ESIntegTestCase {
         recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
         assertThat(recoveryStates.size(), equalTo(1));
 
-        assertRecoveryState(recoveryStates.get(0), 0, Type.PRIMARY_RELOCATION, Stage.DONE, nodeA, nodeB, false);
+        assertRecoveryState(recoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
         validateIndexRecoveryState(recoveryStates.get(0).getIndex());
 
         statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
@@ -399,14 +399,14 @@ public class IndexRecoveryIT extends ESIntegTestCase {
         List<RecoveryState> nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
         assertThat(nodeCRecoveryStates.size(), equalTo(1));
 
-        assertRecoveryState(nodeARecoveryStates.get(0), 0, Type.REPLICA, Stage.DONE, nodeB, nodeA, false);
+        assertRecoveryState(nodeARecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeB, nodeA);
         validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
 
-        assertRecoveryState(nodeBRecoveryStates.get(0), 0, Type.PRIMARY_RELOCATION, Stage.DONE, nodeA, nodeB, false);
+        assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
         validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
 
         // relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
-        assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, Type.REPLICA, nodeB, nodeC, false);
+        assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
         validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
 
         if (randomBoolean()) {
@@ -424,10 +424,10 @@ public class IndexRecoveryIT extends ESIntegTestCase {
             nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
             assertThat(nodeCRecoveryStates.size(), equalTo(1));
 
-            assertRecoveryState(nodeBRecoveryStates.get(0), 0, Type.PRIMARY_RELOCATION, Stage.DONE, nodeA, nodeB, false);
+            assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
             validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
 
-            assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, Type.REPLICA, nodeB, nodeC, false);
+            assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
             validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
         }
 
@@ -445,11 +445,11 @@ public class IndexRecoveryIT extends ESIntegTestCase {
         nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
         assertThat(nodeCRecoveryStates.size(), equalTo(1));
 
-        assertRecoveryState(nodeBRecoveryStates.get(0), 0, Type.PRIMARY_RELOCATION, Stage.DONE, nodeA, nodeB, false);
+        assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
         validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
 
         // relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
-        assertRecoveryState(nodeCRecoveryStates.get(0), 0, Type.REPLICA, Stage.DONE, nodeB, nodeC, false);
+        assertRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeB, nodeC);
         validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
     }
 
@@ -498,7 +498,10 @@ public class IndexRecoveryIT extends ESIntegTestCase {
             assertThat(recoveryStates.size(), equalTo(totalShards));
 
             for (RecoveryState recoveryState : recoveryStates) {
-                assertRecoveryState(recoveryState, 0, Type.SNAPSHOT, Stage.DONE, null, nodeA, true);
+                SnapshotRecoverySource recoverySource = new SnapshotRecoverySource(
+                    new Snapshot(REPO_NAME, createSnapshotResponse.getSnapshotInfo().snapshotId()),
+                    Version.CURRENT, INDEX_NAME);
+                assertRecoveryState(recoveryState, 0, recoverySource, true, Stage.DONE, null, nodeA);
                 validateIndexRecoveryState(recoveryState.getIndex());
             }
         }
@@ -591,13 +594,13 @@ public class IndexRecoveryIT extends ESIntegTestCase {
         assertHitCount(searchResponse, numDocs);
 
         String[] recoveryActions = new String[]{
-                RecoverySource.Actions.START_RECOVERY,
-                RecoveryTargetService.Actions.FILES_INFO,
-                RecoveryTargetService.Actions.FILE_CHUNK,
-                RecoveryTargetService.Actions.CLEAN_FILES,
+                PeerRecoverySourceService.Actions.START_RECOVERY,
+                PeerRecoveryTargetService.Actions.FILES_INFO,
+                PeerRecoveryTargetService.Actions.FILE_CHUNK,
+                PeerRecoveryTargetService.Actions.CLEAN_FILES,
                 //RecoveryTarget.Actions.TRANSLOG_OPS, <-- may not be sent if already flushed
-                RecoveryTargetService.Actions.PREPARE_TRANSLOG,
-                RecoveryTargetService.Actions.FINALIZE
+                PeerRecoveryTargetService.Actions.PREPARE_TRANSLOG,
+                PeerRecoveryTargetService.Actions.FINALIZE
         };
         final String recoveryActionToBlock = randomFrom(recoveryActions);
         final boolean dropRequests = randomBoolean();

core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java

@@ -35,6 +35,7 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
@@ -83,7 +84,7 @@ public class RecoverySourceHandlerTests extends ESTestCase {
         StartRecoveryRequest request = new StartRecoveryRequest(shardId,
                 new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
                 new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
-            null, RecoveryState.Type.STORE, randomLong());
+            null, randomBoolean(), randomLong());
         Store store = newStore(createTempDir());
         RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, () -> 0L, e -> () -> {},
             recoverySettings.getChunkSize().bytesAsInt(), logger);
@@ -135,7 +136,7 @@ public class RecoverySourceHandlerTests extends ESTestCase {
         StartRecoveryRequest request = new StartRecoveryRequest(shardId,
                 new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
                 new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
-            null, RecoveryState.Type.STORE, randomLong());
+            null, randomBoolean(), randomLong());
         Path tempDir = createTempDir();
         Store store = newStore(tempDir, false);
         AtomicBoolean failedEngine = new AtomicBoolean(false);
@@ -199,7 +200,7 @@ public class RecoverySourceHandlerTests extends ESTestCase {
         StartRecoveryRequest request = new StartRecoveryRequest(shardId,
                 new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
                 new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
-            null, RecoveryState.Type.STORE, randomLong());
+            null, randomBoolean(), randomLong());
         Path tempDir = createTempDir();
         Store store = newStore(tempDir, false);
         AtomicBoolean failedEngine = new AtomicBoolean(false);
@@ -258,7 +259,7 @@ public class RecoverySourceHandlerTests extends ESTestCase {
         StartRecoveryRequest request = new StartRecoveryRequest(shardId,
             new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
-            null, RecoveryState.Type.REPLICA, randomLong());
+            null, false, randomLong());
         IndexShard shard = mock(IndexShard.class);
         Translog.View translogView = mock(Translog.View.class);
         when(shard.acquireTranslogView()).thenReturn(translogView);
@@ -288,7 +289,7 @@ public class RecoverySourceHandlerTests extends ESTestCase {
         StartRecoveryRequest request = new StartRecoveryRequest(shardId,
             new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
-            null, RecoveryState.Type.PRIMARY_RELOCATION, randomLong());
+            null, true, randomLong());
         AtomicBoolean phase1Called = new AtomicBoolean();
         AtomicBoolean phase2Called = new AtomicBoolean();
         AtomicBoolean ensureClusterStateVersionCalled = new AtomicBoolean();

core/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java

@@ -43,7 +43,7 @@ public class RecoveryStatusTests extends ESSingleNodeTestCase {
 
         IndexShard indexShard = service.getShardOrNull(0);
         DiscoveryNode node = new DiscoveryNode("foo", new LocalTransportAddress("bar"), emptyMap(), emptySet(), Version.CURRENT);
-        RecoveryTarget status = new RecoveryTarget(indexShard, node, new RecoveryTargetService.RecoveryListener() {
+        RecoveryTarget status = new RecoveryTarget(indexShard, node, new PeerRecoveryTargetService.RecoveryListener() {
             @Override
             public void onRecoveryDone(RecoveryState state) {
             }

core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java

@@ -20,6 +20,10 @@ package org.elasticsearch.indices.recovery;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -31,7 +35,6 @@ import org.elasticsearch.indices.recovery.RecoveryState.Index;
 import org.elasticsearch.indices.recovery.RecoveryState.Stage;
 import org.elasticsearch.indices.recovery.RecoveryState.Timer;
 import org.elasticsearch.indices.recovery.RecoveryState.Translog;
-import org.elasticsearch.indices.recovery.RecoveryState.Type;
 import org.elasticsearch.indices.recovery.RecoveryState.VerifyIndex;
 import org.elasticsearch.test.ESTestCase;
 
@@ -351,8 +354,10 @@ public class RecoveryTargetTests extends ESTestCase {
         stages[i] = stages[j];
         stages[j] = t;
         try {
-            RecoveryState state = new RecoveryState(
-                    new ShardId("bla", "_na_", 0), randomBoolean(), randomFrom(Type.values()), discoveryNode, discoveryNode);
+            ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId("bla", "_na_", 0), discoveryNode.getId(),
+                randomBoolean(), ShardRoutingState.INITIALIZING);
+            RecoveryState state = new RecoveryState(shardRouting, discoveryNode,
+                shardRouting.recoverySource().getType() == RecoverySource.Type.PEER ? discoveryNode : null);
             for (Stage stage : stages) {
                 state.setStage(stage);
             }
@@ -366,8 +371,10 @@ public class RecoveryTargetTests extends ESTestCase {
         i = randomIntBetween(1, stages.length - 1);
         ArrayList<Stage> list = new ArrayList<>(Arrays.asList(Arrays.copyOfRange(stages, 0, i)));
         list.addAll(Arrays.asList(stages));
-        RecoveryState state = new RecoveryState(new ShardId("bla", "_na_", 0), randomBoolean(), randomFrom(Type.values()), discoveryNode,
-                discoveryNode);
+        ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId("bla", "_na_", 0), discoveryNode.getId(),
+            randomBoolean(), ShardRoutingState.INITIALIZING);
+        RecoveryState state = new RecoveryState(shardRouting, discoveryNode,
+            shardRouting.recoverySource().getType() == RecoverySource.Type.PEER ? discoveryNode : null);
         for (Stage stage : list) {
             state.setStage(stage);
         }

core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java

@@ -46,7 +46,7 @@ public class StartRecoveryRequestTests extends ESTestCase {
                 new DiscoveryNode("a", new LocalTransportAddress("1"), emptyMap(), emptySet(), targetNodeVersion),
                 new DiscoveryNode("b", new LocalTransportAddress("1"), emptyMap(), emptySet(), targetNodeVersion),
                 Store.MetadataSnapshot.EMPTY,
-                RecoveryState.Type.PRIMARY_RELOCATION,
+                randomBoolean(),
                 1L
         );
         ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
@@ -65,7 +65,7 @@ public class StartRecoveryRequestTests extends ESTestCase {
         assertThat(outRequest.targetNode(), equalTo(inRequest.targetNode()));
         assertThat(outRequest.metadataSnapshot().asMap(), equalTo(inRequest.metadataSnapshot().asMap()));
         assertThat(outRequest.recoveryId(), equalTo(inRequest.recoveryId()));
-        assertThat(outRequest.recoveryType(), equalTo(inRequest.recoveryType()));
+        assertThat(outRequest.isPrimaryRelocation(), equalTo(inRequest.isPrimaryRelocation()));
     }
 
 }

core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java

@@ -45,7 +45,7 @@ import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.indices.recovery.RecoverySource;
+import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -494,7 +494,7 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
 
         @Override
         public void requestSent(DiscoveryNode node, long requestId, String action, TransportRequestOptions options) {
-            if (action.equals(RecoverySource.Actions.START_RECOVERY)) {
+            if (action.equals(PeerRecoverySourceService.Actions.START_RECOVERY)) {
                 logger.info("sent: {}, relocation starts", action);
                 beginRelocationLatch.countDown();
             }

core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java

@@ -129,7 +129,7 @@ public class IndicesStoreTests extends ESTestCase {
                 if (state == ShardRoutingState.UNASSIGNED) {
                     unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null);
                 }
-                routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", null, null, j == 0, state, unassignedInfo));
+                routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", null, j == 0, state, unassignedInfo));
             }
         }
         assertFalse(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build()));

core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java

@@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.MapBuilder;
@@ -151,7 +152,7 @@ public class FullRollingRestartIT extends ESIntegTestCase {
         ClusterState state = client().admin().cluster().prepareState().get().getState();
         RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").get();
         for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
-            assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode() + "\n" + state.prettyPrint(), recoveryState.getType() != RecoveryState.Type.PRIMARY_RELOCATION);
+            assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode() + "\n" + state.prettyPrint(), recoveryState.getRecoverySource().getType() != RecoverySource.Type.PEER || recoveryState.getPrimary() == false);
         }
         internalCluster().restartRandomDataNode();
         ensureGreen();
@@ -159,7 +160,7 @@ public class FullRollingRestartIT extends ESIntegTestCase {
 
         recoveryResponse = client().admin().indices().prepareRecoveries("test").get();
         for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
-           assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode()+ "-- \nbefore: \n" + state.prettyPrint() + "\nafter: \n" + afterState.prettyPrint(), recoveryState.getType() != RecoveryState.Type.PRIMARY_RELOCATION);
+           assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode()+ "-- \nbefore: \n" + state.prettyPrint() + "\nafter: \n" + afterState.prettyPrint(), recoveryState.getRecoverySource().getType() != RecoverySource.Type.PEER || recoveryState.getPrimary() == false);
         }
     }
 }

core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java

@@ -27,7 +27,7 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.RecoveriesCollection;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -37,7 +37,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThan;
 
 public class RecoveriesCollectionTests extends ESIndexLevelReplicationTestCase {
-    static final RecoveryTargetService.RecoveryListener listener = new RecoveryTargetService.RecoveryListener() {
+    static final PeerRecoveryTargetService.RecoveryListener listener = new PeerRecoveryTargetService.RecoveryListener() {
         @Override
         public void onRecoveryDone(RecoveryState state) {
 
@@ -72,7 +72,7 @@ public class RecoveriesCollectionTests extends ESIndexLevelReplicationTestCase {
             final AtomicBoolean failed = new AtomicBoolean();
             final CountDownLatch latch = new CountDownLatch(1);
             final long recoveryId = startRecovery(collection, shards.getPrimaryNode(), shards.addReplica(),
-                new RecoveryTargetService.RecoveryListener() {
+                new PeerRecoveryTargetService.RecoveryListener() {
                     @Override
                     public void onRecoveryDone(RecoveryState state) {
                         latch.countDown();
@@ -154,10 +154,9 @@ public class RecoveriesCollectionTests extends ESIndexLevelReplicationTestCase {
     }
 
     long startRecovery(RecoveriesCollection collection, DiscoveryNode sourceNode, IndexShard indexShard,
-                       RecoveryTargetService.RecoveryListener listener, TimeValue timeValue) {
+                       PeerRecoveryTargetService.RecoveryListener listener, TimeValue timeValue) {
         final DiscoveryNode rNode = getDiscoveryNode(indexShard.routingEntry().currentNodeId());
-        indexShard.markAsRecovering("remote", new RecoveryState(indexShard.shardId(), false, RecoveryState.Type.REPLICA, sourceNode,
-            rNode));
+        indexShard.markAsRecovering("remote", new RecoveryState(indexShard.routingEntry(), sourceNode, rNode));
         indexShard.prepareForIndexRecovery();
         return collection.startRecovery(indexShard, sourceNode, listener, timeValue);
     }

core/src/test/java/org/elasticsearch/recovery/RelocationIT.java

@@ -45,7 +45,7 @@ import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
@@ -498,7 +498,7 @@ public class RelocationIT extends ESIntegTestCase {
 
         @Override
         public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
-            if (action.equals(RecoveryTargetService.Actions.FILE_CHUNK)) {
+            if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                 RecoveryFileChunkRequest chunkRequest = (RecoveryFileChunkRequest) request;
                 if (chunkRequest.name().startsWith(IndexFileNames.SEGMENTS)) {
                     // corrupting the segments_N files in order to make sure future recovery re-send files

core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java

@@ -33,7 +33,7 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.recovery.IndexRecoveryIT;
 import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest;
 import org.elasticsearch.indices.recovery.RecoverySettings;
-import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -122,7 +122,7 @@ public class TruncatedRecoveryIT extends ESIntegTestCase {
 
                 @Override
                 public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
-                    if (action.equals(RecoveryTargetService.Actions.FILE_CHUNK)) {
+                    if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                         RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
                         logger.debug("file chunk [{}] lastChunk: {}", req, req.lastChunk());
                         if ((req.name().endsWith("cfs") || req.name().endsWith("fdt")) && req.lastChunk() && truncate.get()) {

core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java

@@ -30,6 +30,8 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.Table;
@@ -134,8 +136,10 @@ public class RestIndicesActionTests extends ESTestCase {
             for (int i = 0; i < 2; i++) {
                 ShardId shardId = new ShardId(index, i);
                 Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(i));
-                ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, null, i == 0,
-                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+                ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, i == 0,
+                    i == 0 ? StoreRecoverySource.EMPTY_STORE_INSTANCE : PeerRecoverySource.INSTANCE,
+                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+                    );
                 shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
                 shardRouting = shardRouting.moveToStarted();
                 CommonStats stats = new CommonStats();

core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java

@@ -21,10 +21,10 @@ package org.elasticsearch.rest.action.cat;
 
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RestoreSource;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
+import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.settings.Settings;
@@ -33,7 +33,6 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.rest.RestController;
-import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -66,7 +65,7 @@ public class RestRecoveryActionTests extends ESTestCase {
             final RecoveryState.Timer timer = mock(RecoveryState.Timer.class);
             when(timer.time()).thenReturn((long)randomIntBetween(1000000, 10 * 1000000));
             when(state.getTimer()).thenReturn(timer);
-            when(state.getType()).thenReturn(randomFrom(RecoveryState.Type.values()));
+            when(state.getRecoverySource()).thenReturn(TestShardRouting.randomRecoverySource());
             when(state.getStage()).thenReturn(randomFrom(RecoveryState.Stage.values()));
             final DiscoveryNode sourceNode = randomBoolean() ? mock(DiscoveryNode.class) : null;
             if (sourceNode != null) {
@@ -77,13 +76,6 @@ public class RestRecoveryActionTests extends ESTestCase {
             when(targetNode.getHostName()).thenReturn(randomAsciiOfLength(8));
             when(state.getTargetNode()).thenReturn(targetNode);
 
-            final RestoreSource restoreSource = randomBoolean() ? mock(RestoreSource.class) : null;
-            if (restoreSource != null) {
-                final Snapshot snapshot = new Snapshot(randomAsciiOfLength(8),
-                                                       new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
-                when(restoreSource.snapshot()).thenReturn(snapshot);
-            }
-
             RecoveryState.Index index = mock(RecoveryState.Index.class);
 
             final int totalRecoveredFiles = randomIntBetween(1, 64);
@@ -160,7 +152,7 @@ public class RestRecoveryActionTests extends ESTestCase {
             assertThat(cells.get(0).value, equalTo("index"));
             assertThat(cells.get(1).value, equalTo(i));
             assertThat(cells.get(2).value, equalTo(new TimeValue(state.getTimer().time())));
-            assertThat(cells.get(3).value, equalTo(state.getType().name().toLowerCase(Locale.ROOT)));
+            assertThat(cells.get(3).value, equalTo(state.getRecoverySource().getType().name().toLowerCase(Locale.ROOT)));
             assertThat(cells.get(4).value, equalTo(state.getStage().name().toLowerCase(Locale.ROOT)));
             assertThat(cells.get(5).value, equalTo(state.getSourceNode() == null ? "n/a" : state.getSourceNode().getHostName()));
             assertThat(cells.get(6).value, equalTo(state.getSourceNode() == null ? "n/a" : state.getSourceNode().getName()));
@@ -168,10 +160,14 @@ public class RestRecoveryActionTests extends ESTestCase {
             assertThat(cells.get(8).value, equalTo(state.getTargetNode().getName()));
             assertThat(
                     cells.get(9).value,
-                    equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshot().getRepository()));
+                    equalTo(state.getRecoverySource() == null || state.getRecoverySource().getType() != RecoverySource.Type.SNAPSHOT ?
+                        "n/a" :
+                        ((SnapshotRecoverySource) state.getRecoverySource()).snapshot().getRepository()));
             assertThat(
                     cells.get(10).value,
-                    equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshot().getSnapshotId().getName()));
+                    equalTo(state.getRecoverySource() == null || state.getRecoverySource().getType() != RecoverySource.Type.SNAPSHOT ?
+                        "n/a" :
+                        ((SnapshotRecoverySource) state.getRecoverySource()).snapshot().getSnapshotId().getName()));
             assertThat(cells.get(11).value, equalTo(state.getIndex().totalRecoverFiles()));
             assertThat(cells.get(12).value, equalTo(state.getIndex().recoveredFileCount()));
             assertThat(cells.get(13).value, equalTo(percent(state.getIndex().recoveredFilesPercent())));

core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java

@@ -20,8 +20,9 @@
 package org.elasticsearch.search.internal;
 
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.cluster.routing.RestoreSource;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -33,10 +34,7 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchRequestTests;
-import org.elasticsearch.snapshots.Snapshot;
-import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -101,11 +99,8 @@ public class ShardSearchTransportRequestTests extends ESTestCase {
     private static ShardSearchTransportRequest createShardSearchTransportRequest() throws IOException {
         SearchRequest searchRequest = SearchRequestTests.createSearchRequest();
         ShardId shardId = new ShardId(randomAsciiOfLengthBetween(2, 10), randomAsciiOfLengthBetween(2, 10), randomInt());
-        Snapshot snapshot = new Snapshot(randomAsciiOfLengthBetween(3, 10),
-                new SnapshotId(randomAsciiOfLengthBetween(3, 10), randomAsciiOfLengthBetween(3, 10)));
-        RestoreSource restoreSource = new RestoreSource(snapshot, VersionUtils.randomVersion(random()), randomAsciiOfLengthBetween(3, 10));
-        ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, restoreSource, randomBoolean(),
-                new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "reason"));
+        ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, null, null, randomBoolean(), ShardRoutingState.UNASSIGNED,
+            new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "reason"));
         String[] filteringAliases;
         if (randomBoolean()) {
             filteringAliases = generateRandomStringArray(10, 10, false, false);

rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml

@@ -26,7 +26,7 @@
                 index1      \s+
                 \d          \s+                                 # shard
                 (?:\d+ms|\d+(?:\.\d+)?s)    \s+                 # time in ms or seconds
-                (store|replica|snapshot|relocating)     \s+     # type
+                (empty_store|existing_store|peer|snapshot|local_shards) \s+ # source type
                 (init|index|verify_index|translog|finalize|done) \s+ # stage
                 [-\w./]+    \s+                                 # source_host
                 [-\w./]+    \s+                                 # target_host

rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml

@@ -20,6 +20,7 @@
                     unassigned.at                    .+   \n
                     unassigned.for                   .+   \n
                     unassigned.details               .+   \n
+                    recoverysource.type              .+   \n
                     completion.size                  .+   \n
                     fielddata.memory_size            .+   \n
                     fielddata.evictions              .+   \n

rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yaml

@@ -18,7 +18,7 @@
         index: [test_1]
         human: true
 
-  - match: { test_1.shards.0.type:                                 "STORE"                 }
+  - match: { test_1.shards.0.type:                                 "EMPTY_STORE"           }
   - match: { test_1.shards.0.stage:                                "DONE"                  }
   - match: { test_1.shards.0.primary:                              true                    }
   - match: { test_1.shards.0.start_time:                           /^2\d\d\d-.+/           }

test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java

@@ -20,16 +20,11 @@
 package org.elasticsearch.cluster;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterInfoService;
-import org.elasticsearch.cluster.ClusterModule;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.EmptyClusterInfoService;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
-import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@@ -40,7 +35,6 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
-import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
@@ -51,10 +45,8 @@ import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.gateway.NoopGatewayAllocator;
 
-import java.lang.reflect.Constructor;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -65,8 +57,6 @@ import java.util.Set;
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
 
 /**
  */
@@ -243,8 +233,7 @@ public abstract class ESAllocationTestCase extends ESTestCase {
             final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = allocation.routingNodes().unassigned().iterator();
             while (unassignedIterator.hasNext()) {
                 ShardRouting shard = unassignedIterator.next();
-                IndexMetaData indexMetaData = allocation.metaData().index(shard.getIndexName());
-                if (shard.primary() || shard.allocatedPostIndexCreate(indexMetaData) == false) {
+                if (shard.primary() || shard.unassignedInfo().getReason() == UnassignedInfo.Reason.INDEX_CREATED) {
                     continue;
                 }
                 replicaShardAllocator.ignoreUnassignedIfDelayed(unassignedIterator, shard, allocation.changes());

test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java

@@ -19,11 +19,16 @@
 
 package org.elasticsearch.cluster.routing;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.index.Index;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.ESTestCase;
 
+import static org.elasticsearch.test.ESTestCase.randomAsciiOfLength;
+
 /**
  * A helper that allows to create shard routing instances within tests, while not requiring to expose
  * different simplified constructors on the ShardRouting itself.
@@ -35,7 +40,11 @@ public class TestShardRouting {
     }
 
     public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, boolean primary, ShardRoutingState state) {
-        return new ShardRouting(shardId, currentNodeId, null, null, primary, state, buildUnassignedInfo(state), buildAllocationId(state), -1);
+        return new ShardRouting(shardId, currentNodeId, null, primary, state, buildRecoveryTarget(primary, state), buildUnassignedInfo(state), buildAllocationId(state), -1);
+    }
+
+    public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, boolean primary, ShardRoutingState state, RecoverySource recoverySource) {
+        return new ShardRouting(shardId, currentNodeId, null, primary, state, recoverySource, buildUnassignedInfo(state), buildAllocationId(state), -1);
     }
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state) {
@@ -43,7 +52,7 @@ public class TestShardRouting {
     }
 
     public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state) {
-        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, null, primary, state, buildUnassignedInfo(state), buildAllocationId(state), -1);
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, primary, state, buildRecoveryTarget(primary, state), buildUnassignedInfo(state), buildAllocationId(state), -1);
     }
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state, AllocationId allocationId) {
@@ -51,33 +60,43 @@ public class TestShardRouting {
     }
 
     public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state, AllocationId allocationId) {
-        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, null, primary, state, buildUnassignedInfo(state), allocationId, -1);
-    }
-
-    public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state) {
-        return newShardRouting(new ShardId(index, IndexMetaData.INDEX_UUID_NA_VALUE, shardId), currentNodeId, relocatingNodeId, restoreSource, primary, state);
-    }
-
-    public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state) {
-        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, buildUnassignedInfo(state), buildAllocationId(state), -1);
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, primary, state, buildRecoveryTarget(primary, state), buildUnassignedInfo(state), allocationId, -1);
     }
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId,
-                                               String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state,
+                                               String relocatingNodeId, boolean primary, ShardRoutingState state,
                                                UnassignedInfo unassignedInfo) {
-        return newShardRouting(new ShardId(index, IndexMetaData.INDEX_UUID_NA_VALUE, shardId), currentNodeId, relocatingNodeId, restoreSource, primary, state, unassignedInfo);
+        return newShardRouting(new ShardId(index, IndexMetaData.INDEX_UUID_NA_VALUE, shardId), currentNodeId, relocatingNodeId, primary, state, unassignedInfo);
     }
 
     public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId,
-                                               String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state,
+                                               String relocatingNodeId, boolean primary, ShardRoutingState state,
                                                UnassignedInfo unassignedInfo) {
-        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, unassignedInfo, buildAllocationId(state), -1);
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, primary, state, buildRecoveryTarget(primary, state), unassignedInfo, buildAllocationId(state), -1);
     }
 
     public static ShardRouting relocate(ShardRouting shardRouting, String relocatingNodeId, long expectedShardSize) {
         return shardRouting.relocate(relocatingNodeId, expectedShardSize);
     }
 
+    private static RecoverySource buildRecoveryTarget(boolean primary, ShardRoutingState state) {
+        switch (state) {
+            case UNASSIGNED:
+            case INITIALIZING:
+                if (primary) {
+                    return ESTestCase.randomFrom(RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE,
+                        RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE);
+                } else {
+                    return RecoverySource.PeerRecoverySource.INSTANCE;
+                }
+            case STARTED:
+            case RELOCATING:
+                return null;
+            default:
+                throw new IllegalStateException("illegal state");
+        }
+    }
+
     private static AllocationId buildAllocationId(ShardRoutingState state) {
         switch (state) {
             case UNASSIGNED:
@@ -105,4 +124,15 @@ public class TestShardRouting {
                 throw new IllegalStateException("illegal state");
         }
     }
+
+    public static RecoverySource randomRecoverySource() {
+        return ESTestCase.randomFrom(RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE,
+            RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE,
+            RecoverySource.PeerRecoverySource.INSTANCE,
+            RecoverySource.LocalShardsRecoverySource.INSTANCE,
+            new RecoverySource.SnapshotRecoverySource(
+                new Snapshot("repo", new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID())),
+                Version.CURRENT,
+                "some_index"));
+    }
 }