testRelocationWithConcurrentIndexing: wait for green (on relevan index) and shard initialization to settle down before starting relocation

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

@@ -242,12 +242,14 @@ public class RecoveryIT extends ESRestTestCase {
             case MIXED:
                 final String newNode = getNodeId(v -> v.equals(Version.CURRENT));
                 final String oldNode = getNodeId(v -> v.before(Version.CURRENT));
-                // remove the replica now that we know that the primary is an old node
+                // remove the replica and guaranteed the primary is placed on the old node
                 updateIndexSetting(index, Settings.builder()
                     .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
                     .put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), (String)null)
                     .put("index.routing.allocation.include._id", oldNode)
                 );
+                ensureGreen(index); // wait for the primary to be assigned
+                ensureNoInitializingShards(); // wait for all other shard activity to finish
                 updateIndexSetting(index, Settings.builder().put("index.routing.allocation.include._id", newNode));
                 asyncIndexDocs(index, 10, 50).get();
                 ensureGreen(index);

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -406,6 +406,18 @@ public abstract class ESRestTestCase extends ESTestCase {
         assertOK(client().performRequest("GET", "_cluster/health/" + index, params));
     }
 
+    /**
+     * waits until all shard initialization is completed. This is a handy alternative to ensureGreen as it relates to all shards
+     * in the cluster and doesn't require to know how many nodes/replica there are.
+     */
+    protected void ensureNoInitializingShards() throws IOException {
+        Map<String, String> params = new HashMap<>();
+        params.put("wait_for_no_initializing_shards", "true");
+        params.put("timeout", "70s");
+        params.put("level", "shards");
+        assertOK(client().performRequest("GET", "_cluster/health/", params));
+    }
+
     protected void createIndex(String name, Settings settings) throws IOException {
         createIndex(name, settings, "");
     }