diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml
index 4def6fa5da5..14282e6a00f 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml
@@ -11,6 +11,12 @@
 
  - match: { hits.total: 5 } # no new indexed data, so expect the original 5 documents from the old cluster
 
+ - do:
+     search:
+       index: index_with_replicas
+
+ - match: { hits.total: 5 } # just check we recovered fine
+
  - do:
      bulk:
         refresh: true
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml
index a5e9a26a9a1..14c25aa101d 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml
@@ -8,24 +8,43 @@
             index:
               number_of_replicas: 0
 
+  - do:
+      indices.create:
+        index: index_with_replicas # dummy index to ensure we can recover indices with replicas just fine
+
   - do:
       bulk:
         refresh: true
         body:
-          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
           - '{"f1": "v1_old", "f2": 0}'
-          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
           - '{"f1": "v2_old", "f2": 1}'
-          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
           - '{"f1": "v3_old", "f2": 2}'
-          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
           - '{"f1": "v4_old", "f2": 3}'
-          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
           - '{"f1": "v5_old", "f2": 4}'
 
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+
   - do:
       indices.flush:
-        index: test_index
+        index: test_index,index_with_replicas
 
   - do:
       search:
@@ -33,6 +52,12 @@
 
   - match: { hits.total: 5 }
 
+  - do:
+      search:
+        index: index_with_replicas
+
+  - match: { hits.total: 5 }
+
   - do:
       snapshot.create_repository:
         repository: my_repo
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
index 93c85582c90..725883e7ed5 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
@@ -11,6 +11,12 @@
 
  - match: { hits.total: 10 } # no new indexed data, so expect the original 10 documents from the old and mixed clusters
 
+ - do:
+     search:
+       index: index_with_replicas
+
+ - match: { hits.total: 5 } # just check we recovered fine
+
  - do:
      bulk:
         refresh: true