[ML] Fixing datafeed bwc tests (#52959)

Datafeed bwc tests have been muted for some time in the 7.x. This is because of date_histogram interval deprecation warnings.

This commit fixes the tests as must as possible while still handling deprecation warnings.

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml

@@ -1,8 +1,3 @@
-setup:
-  - skip:
-      version: "all"
-      reason:  "AwaitsFix https://github.com/elastic/elasticsearch/issues/42258"
-
 ---
 "Test old cluster datafeed without aggs":
   - do:
@@ -23,9 +18,9 @@ setup:
 ---
 "Test old cluster datafeed with aggs":
   - skip:
-      features: "warnings"
+      features: allowed_warnings
   - do:
-      warnings:
+      allowed_warnings:
         - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.get_datafeeds:
         datafeed_id: old-cluster-datafeed-with-aggs
@@ -37,6 +32,8 @@ setup:
   - match: { datafeeds.0.aggregations.buckets.aggregations.time.max.field: time }
 
   - do:
+      allowed_warnings:
+        - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.get_datafeed_stats:
         datafeed_id: old-cluster-datafeed-with-aggs
   - match: { datafeeds.0.state: "stopped"}
@@ -91,7 +88,7 @@ setup:
 ---
 "Put job and datafeed with aggs in mixed cluster":
   - skip:
-      features: "warnings"
+      features: allowed_warnings
 
   - do:
       ml.put_job:
@@ -114,7 +111,7 @@ setup:
           }
 
   - do:
-      warnings:
+      allowed_warnings:
         - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.put_datafeed:
         datafeed_id: mixed-cluster-datafeed-with-aggs
@@ -151,8 +148,9 @@ setup:
               }
             }
           }
-
   - do:
+      allowed_warnings:
+        - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.get_datafeed_stats:
         datafeed_id: mixed-cluster-datafeed-with-aggs
   - match: { datafeeds.0.state: stopped}

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml

@@ -1,7 +1,3 @@
-setup:
-  - skip:
-      version: "all"
-      reason:  "AwaitsFix https://github.com/elastic/elasticsearch/issues/42258"
 ---
 "Put job and datafeed without aggs in old cluster":
 
@@ -52,8 +48,8 @@ setup:
 ---
 "Put job and datafeed with aggs in old cluster - pre-deprecated interval":
   - skip:
-      version: "8.0.0 - "  #TODO change this after backport
-      reason:  calendar_interval introduced in 7.1.0
+      version: "7.2.0 - "
+      reason:  calendar_interval introduced in 7.2.0
 
   - do:
       ml.put_job:
@@ -120,10 +116,84 @@ setup:
   - is_false: datafeeds.0.node
 
 ---
-"Put job and datafeed with aggs in old cluster - deprecated interval with warning":
+"Put job and datafeed with aggs in old cluster - deprecated interval with warning and warning on stats":
   - skip:
-      version: " - 7.1.99"
-      reason:  calendar_interval introduced in 7.2.0
+      version: " - 7.3.99"
+      reason:  calendar_interval introduced in 7.2.0 and datafeed config expansion not introduced on stats until 7.4.0
+      features: warnings
+
+  - do:
+      ml.put_job:
+        job_id: old-cluster-datafeed-job-with-aggs
+        body:  >
+          {
+            "description":"Cluster upgrade",
+            "analysis_config" : {
+                "bucket_span": "60s",
+                "summary_count_field_name": "doc_count",
+                "detectors" :[{"function":"count"}]
+            },
+            "analysis_limits" : {
+                "model_memory_limit": "50mb"
+            },
+            "data_description" : {
+                "format":"xcontent",
+                "time_field":"time"
+            }
+          }
+  - match: { job_id: old-cluster-datafeed-job-with-aggs }
+
+  - do:
+      warnings:
+        - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
+      ml.put_datafeed:
+        datafeed_id: old-cluster-datafeed-with-aggs
+        body:  >
+          {
+            "job_id":"old-cluster-datafeed-job-with-aggs",
+            "indices":["airline-data"],
+            "scroll_size": 2000,
+            "aggregations": {
+              "buckets": {
+                "date_histogram": {
+                  "field": "time",
+                  "interval": "30s",
+                  "time_zone": "UTC"
+                },
+                "aggregations": {
+                  "time": {
+                    "max": {"field": "time"}
+                  },
+                  "airline": {
+                    "terms": {
+                      "field": "airline",
+                      "size": 100
+                    },
+                    "aggregations": {
+                      "responsetime": {
+                        "avg": {
+                          "field": "responsetime"
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+
+  - do:
+      warnings:
+        - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
+      ml.get_datafeed_stats:
+        datafeed_id: old-cluster-datafeed-with-aggs
+  - match: { datafeeds.0.state: stopped}
+  - is_false: datafeeds.0.node
+---
+"Put job and datafeed with aggs in old cluster - deprecated interval with warning but not on stats":
+  - skip:
+      version: " - 7.1.99, 7.4.0 - "
+      reason:  calendar_interval introduced in 7.2.0, and datafeeds expanded in stats after and on 7.4.0
       features: warnings
 
   - do:

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml

@@ -1,7 +1,4 @@
 setup:
-  - skip:
-      version: "all"
-      reason:  "AwaitsFix https://github.com/elastic/elasticsearch/issues/42258"
   - do:
      cluster.health:
         wait_for_status: green
@@ -109,7 +106,7 @@ setup:
 ---
 "Test old and mixed cluster datafeeds with aggs":
   - skip:
-      features: "warnings"
+      features: allowed_warnings
   - do:
       indices.create:
         index: airline-data
@@ -120,7 +117,7 @@ setup:
                 type: date
 
   - do:
-      warnings:
+      allowed_warnings:
         - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.get_datafeeds:
         datafeed_id: old-cluster-datafeed-with-aggs
@@ -132,13 +129,15 @@ setup:
   - match: { datafeeds.0.aggregations.buckets.aggregations.time.max.field: time }
 
   - do:
+      allowed_warnings:
+        - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.get_datafeed_stats:
         datafeed_id: old-cluster-datafeed-with-aggs
   - match: { datafeeds.0.state: "stopped"}
   - is_false: datafeeds.0.node
 
   - do:
-      warnings:
+      allowed_warnings:
         - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.get_datafeeds:
         datafeed_id: mixed-cluster-datafeed-with-aggs
@@ -150,6 +149,8 @@ setup:
   - match: { datafeeds.0.aggregations.buckets.aggregations.time.max.field: time }
 
   - do:
+      allowed_warnings:
+        - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.get_datafeed_stats:
         datafeed_id: mixed-cluster-datafeed-with-aggs
   - match: { datafeeds.0.state: "stopped"}
@@ -160,13 +161,15 @@ setup:
         job_id: old-cluster-datafeed-job-with-aggs
 
   - do:
-      warnings:
+      allowed_warnings:
         - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.start_datafeed:
         datafeed_id: old-cluster-datafeed-with-aggs
         start: 0
 
   - do:
+      allowed_warnings:
+        - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.stop_datafeed:
         datafeed_id: old-cluster-datafeed-with-aggs
 
@@ -175,6 +178,8 @@ setup:
         job_id: old-cluster-datafeed-job-with-aggs
 
   - do:
+      allowed_warnings:
+        - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.delete_datafeed:
         datafeed_id: old-cluster-datafeed-with-aggs
 
@@ -188,13 +193,15 @@ setup:
         job_id: mixed-cluster-datafeed-job-with-aggs
 
   - do:
-      warnings:
+      allowed_warnings:
         - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.start_datafeed:
         datafeed_id: mixed-cluster-datafeed-with-aggs
         start: 0
 
   - do:
+      allowed_warnings:
+        - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.stop_datafeed:
         datafeed_id: mixed-cluster-datafeed-with-aggs
 
@@ -203,6 +210,8 @@ setup:
         job_id: mixed-cluster-datafeed-job-with-aggs
 
   - do:
+      allowed_warnings:
+        - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
       ml.delete_datafeed:
         datafeed_id: mixed-cluster-datafeed-with-aggs