Ensure test replicas have valid recovery state

When starting a new replication group in an index level replication test
case, a started replica would not have a valid recovery state. This
violates simple assumptions as replicas always have to have recovered
before being started. This commit causes this to be the case that this
assumption is not violated too.

core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java

@@ -209,6 +209,10 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
             primary.markAsRecovering("store", new RecoveryState(primary.routingEntry(), pNode, null));
             primary.recoverFromStore();
             primary.updateRoutingEntry(ShardRoutingHelper.moveToStarted(primary.routingEntry()));
+            for (final IndexShard replica : replicas) {
+                //replica.markAsRecovering("peer recovery", new RecoveryState(replica.routingEntry(), getDiscoveryNode(replica.routingEntry().currentNodeId()), pNode));
+                recoverReplica(replica);
+            }
             updateAllocationIDsOnPrimary();
         }
 

core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java

@@ -77,7 +77,6 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase
     public void testAppendWhileRecovering() throws Exception {
         try (ReplicationGroup shards = createGroup(0)) {
             shards.startAll();
-            IndexShard replica = shards.addReplica();
             CountDownLatch latch = new CountDownLatch(2);
             int numDocs = randomIntBetween(100, 200);
             shards.appendDocs(1);// just append one to the translog so we can assert below
@@ -94,6 +93,7 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase
                 }
             };
             thread.start();
+            IndexShard replica = shards.addReplica();
             Future<Void> future = shards.asyncRecoverReplica(replica, (indexShard, node)
                     -> new RecoveryTarget(indexShard, node, recoveryListener, version -> {
             }) {