Test search templates during rolling upgrade test (#24258)

In #24251 we fix an issue with stored search templates that
this test would have discovered: stored search templates cause
the node to refuse to start. Technically a "restart" test would
have caught this as well and would have caught it more quickly.
But we already *have* an upgrade test and we don't have restart tests.
And testing this on upgrade is a good thing too.

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml

@@ -59,6 +59,16 @@
 
  - match: { hits.total: 10 }
 
+---
+"Verify that we can still find things with the template":
+  - do:
+      search_template:
+        body:
+          id: test_search_template
+          params:
+            f1: v5_old
+  - match: { hits.total: 1 }
+
 ---
 "Verify custom cluster metadata still exists during upgrade":
   - do:

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml

@@ -1,15 +1,15 @@
 ---
-"Index data and search on the old cluster":
- - do:
-     indices.create:
+"Index data, search, and create things in the cluster state that we'll validate are there after the ugprade":
+  - do:
+      indices.create:
         index: test_index
         body:
           settings:
             index:
               number_of_replicas: 0
 
- - do:
-     bulk:
+  - do:
+      bulk:
         refresh: true
         body:
           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
@@ -23,18 +23,16 @@
           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
           - '{"f1": "v5_old", "f2": 4}'
 
- - do:
-     indices.flush:
+  - do:
+      indices.flush:
         index: test_index
 
- - do:
-     search:
+  - do:
+      search:
         index: test_index
 
- - match: { hits.total: 5 }
+  - match: { hits.total: 5 }
 
----
-"Add stuff to cluster state so that we can verify that it remains to exist during and after the rolling upgrade":
   - do:
       snapshot.create_repository:
         repository: my_repo
@@ -54,3 +52,20 @@
             ]
           }
   - match: { "acknowledged": true }
+
+  - do:
+      put_template:
+        id: test_search_template
+        body:
+          query:
+            match:
+              f1: "{{f1}}"
+  - match: { acknowledged: true }
+
+  - do:
+      search_template:
+        body:
+          id: test_search_template
+          params:
+            f1: v5_old
+  - match: { hits.total: 1 }

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml

@@ -36,6 +36,16 @@
 
  - match: { hits.total: 15 } # 10 docs from previous clusters plus 5 new docs
 
+---
+"Verify that we can still find things with the template":
+  - do:
+      search_template:
+        body:
+          id: test_search_template
+          params:
+            f1: v5_old
+  - match: { hits.total: 1 }
+
 ---
 "Verify custom cluster metadata still exists after rolling upgrade":
   - do: