[ML] Add data frame analytics bwc testing (#63012)

This commit adds bwc testing for data frame analytics.

The bwc tests only go back to the 7.9.0.
Meaning, initially only rolling upgrades from 7.9.x -> 7.10.0 are tested.

Since the feature was experimental in < 7.9.0, this is acceptable.

x-pack/qa/rolling-upgrade/build.gradle

@@ -106,6 +106,13 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
     if (Version.fromString(oldVersion).before('7.3.0')) {
       toBlackList << 'old_cluster/80_transform_jobs_crud/Test put continuous transform on old cluster'
     }
+    if (Version.fromString(oldVersion).before('7.9.0')) {
+      toBlackList.addAll([
+        'old_cluster/90_ml_data_frame_analytics_crud/Put outlier_detection job on the old cluster',
+        'old_cluster/90_ml_data_frame_analytics_crud/Put regression job on the old cluster',
+        'old_cluster/90_ml_data_frame_analytics_crud/Put classification job on the old cluster'
+      ])
+    }
     if (!toBlackList.empty) {
       systemProperty 'tests.rest.blacklist', toBlackList.join(',')
     }
@@ -130,6 +137,8 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
       'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed with aggs in mixed cluster',
       'mixed_cluster/80_transform_jobs_crud/Test put batch transform on mixed cluster',
       'mixed_cluster/80_transform_jobs_crud/Test put continuous transform on mixed cluster',
+      'mixed_cluster/90_ml_data_frame_analytics_crud/Put an outlier_detection job on the mixed cluster',
+      'mixed_cluster/90_ml_data_frame_analytics_crud/Put and delete jobs',
       'mixed_cluster/110_enrich/Enrich stats query smoke test for mixed cluster',
     ]
     // transform in mixed cluster is effectively disabled till 7.4, see gh#48019
@@ -139,6 +148,18 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
         'mixed_cluster/80_transform_jobs_crud/Test GET, stop, start, old continuous transforms'
       ])
     }
+    if (Version.fromString(oldVersion).before('7.9.0')) {
+      toBlackList.addAll([
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Get old outlier_detection job',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Get old outlier_detection job stats',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Start and stop old outlier_detection job',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Get old regression job',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Get old regression job stats',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Start and stop old regression job',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Get old classification job',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Get old classification job stats',
+      ])
+    }
     systemProperty 'tests.rest.blacklist', toBlackList.join(',')
   }
 
@@ -162,6 +183,20 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
         'mixed_cluster/80_transform_jobs_crud/Test GET, stop, start, old continuous transforms'
       ])
     }
+    if (Version.fromString(oldVersion).before('7.9.0')) {
+      toBlackList.addAll([
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Get old outlier_detection job',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Get old outlier_detection job stats',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Start and stop old outlier_detection job',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Get old regression job',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Get old regression job stats',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Start and stop old regression job',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Get old classification job',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Get old classification job stats',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Put an outlier_detection job on the mixed cluster',
+        'mixed_cluster/90_ml_data_frame_analytics_crud/Put and delete jobs'
+      ])
+    }
     if (!toBlackList.empty) {
       systemProperty 'tests.rest.blacklist', toBlackList.join(',')
     }
@@ -192,6 +227,19 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
       toBlackList << 'upgraded_cluster/80_transform_jobs_crud/Get start, stop mixed cluster batch transform'
       toBlackList << 'upgraded_cluster/80_transform_jobs_crud/Test GET, mixed continuous transforms'
     }
+    if (Version.fromString(oldVersion).before('7.9.0')) {
+      toBlackList.addAll([
+        'upgraded_cluster/90_ml_data_frame_analytics_crud/Get old cluster outlier_detection job',
+        'upgraded_cluster/90_ml_data_frame_analytics_crud/Get old cluster outlier_detection job stats',
+        'upgraded_cluster/90_ml_data_frame_analytics_crud/Get old cluster regression job',
+        'upgraded_cluster/90_ml_data_frame_analytics_crud/Get old cluster regression job stats',
+        'upgraded_cluster/90_ml_data_frame_analytics_crud/Get old classification job',
+        'upgraded_cluster/90_ml_data_frame_analytics_crud/Get old classification job stats',
+        'upgraded_cluster/90_ml_data_frame_analytics_crud/Get mixed cluster outlier_detection job',
+        'upgraded_cluster/90_ml_data_frame_analytics_crud/Get mixed cluster outlier_detection job stats',
+        'upgraded_cluster/90_ml_data_frame_analytics_crud/Get old classification job stats'
+      ])
+    }
 
     if (!toBlackList.empty) {
       systemProperty 'tests.rest.blacklist', toBlackList.join(',')

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml

@@ -0,0 +1,243 @@
+---
+"Get old outlier_detection job":
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "old_cluster_outlier_detection_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_outlier_detection_job" }
+  - match: { data_frame_analytics.0.source.index: ["bwc_ml_outlier_detection_job_source"] }
+  - match: { data_frame_analytics.0.source.query: {"term" : { "user.keyword" : "Kimchy" }} }
+  - match: { data_frame_analytics.0.dest.index: "old_cluster_outlier_detection_job_results" }
+  - match: { data_frame_analytics.0.analysis: {
+    "outlier_detection":{
+      "compute_feature_influence": true,
+      "outlier_fraction": 0.05,
+      "standardization_enabled": true
+    }
+  }}
+
+---
+"Get old outlier_detection job stats":
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "old_cluster_outlier_detection_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_outlier_detection_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
+---
+"Start and stop old outlier_detection job":
+
+  - do:
+      ml.start_data_frame_analytics:
+        id: "old_cluster_outlier_detection_job"
+  - match: { acknowledged: true }
+
+  - do:
+      ml.stop_data_frame_analytics:
+        id: "old_cluster_outlier_detection_job"
+  - match: { stopped: true }
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "old_cluster_outlier_detection_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_outlier_detection_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
+---
+"Get old regression job":
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "old_cluster_regression_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_regression_job" }
+  - match: { data_frame_analytics.0.source.index: ["bwc_ml_regression_job_source"] }
+  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
+  - match: { data_frame_analytics.0.dest.index: "old_cluster_regression_job_results" }
+  - match: { data_frame_analytics.0.analysis.regression.dependent_variable: "foo" }
+  - match: { data_frame_analytics.0.analysis.regression.training_percent: 100.0 }
+  - is_true: data_frame_analytics.0.analysis.regression.randomize_seed
+
+---
+"Get old regression job stats":
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "old_cluster_regression_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_regression_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
+---
+"Start and stop old regression job":
+
+  - do:
+      ml.start_data_frame_analytics:
+        id: "old_cluster_regression_job"
+  - match: { acknowledged: true }
+
+  - do:
+      ml.stop_data_frame_analytics:
+        id: "old_cluster_regression_job"
+  - match: { stopped: true }
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "old_cluster_regression_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_regression_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
+---
+"Get old classification job":
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "old_cluster_classification_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
+  - match: { data_frame_analytics.0.source.index: ["bwc_ml_classification_job_source"] }
+  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
+  - match: { data_frame_analytics.0.dest.index: "old_cluster_classification_job_results" }
+  - match: { data_frame_analytics.0.analysis.classification.dependent_variable: "foo" }
+  - match: { data_frame_analytics.0.analysis.classification.training_percent: 100.0 }
+  - is_true: data_frame_analytics.0.analysis.classification.randomize_seed
+
+---
+"Get old classification job stats":
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "old_cluster_classification_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
+---
+"Put an outlier_detection job on the mixed cluster":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "mixed_cluster_outlier_detection_job"
+        body: >
+          {
+            "source": {
+              "index": "bwc_ml_outlier_detection_job_source",
+              "query": {"term" : { "user.keyword" : "Kimchy" }}
+            },
+            "dest": {
+              "index": "mixed_cluster_outlier_detection_job_results"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+  - match: { id: "mixed_cluster_outlier_detection_job" }
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "mixed_cluster_outlier_detection_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "mixed_cluster_outlier_detection_job" }
+  - match: { data_frame_analytics.0.source.index: ["bwc_ml_outlier_detection_job_source"] }
+  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
+  - match: { data_frame_analytics.0.dest.index: "mixed_cluster_outlier_detection_job_results" }
+  - match: { data_frame_analytics.0.analysis: {
+    "outlier_detection":{
+      "compute_feature_influence": true,
+      "outlier_fraction": 0.05,
+      "standardization_enabled": true
+    }
+  }}
+
+  - do:
+      ml.start_data_frame_analytics:
+        id: "mixed_cluster_outlier_detection_job"
+  - match: { acknowledged: true }
+
+  - do:
+      ml.stop_data_frame_analytics:
+        id: "mixed_cluster_outlier_detection_job"
+  - match: { stopped: true }
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "mixed_cluster_outlier_detection_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "mixed_cluster_outlier_detection_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
+---
+"Put and delete jobs":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "mixed_cluster_job_to_delete"
+        body: >
+          {
+            "source": {
+              "index": "bwc_ml_outlier_detection_job_source"
+            },
+            "dest": {
+              "index": "mixed_cluster_job_to_delete_results"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+  - match: { id: "mixed_cluster_job_to_delete" }
+
+  - do:
+      ml.delete_data_frame_analytics:
+        id: mixed_cluster_job_to_delete
+  - match: { acknowledged: true }
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "mixed_cluster_job_to_delete"
+        body: >
+          {
+            "source": {
+              "index": "bwc_ml_classification_job_source",
+              "query": {"term" : { "user.keyword" : "Kimchy" }}
+            },
+            "dest": {
+              "index": "old_cluster_classification_job_results"
+            },
+            "analysis": {
+              "classification":{
+                "dependent_variable": "foo"
+              }
+            }
+          }
+  - match: { id: "mixed_cluster_job_to_delete" }
+
+  - do:
+      ml.delete_data_frame_analytics:
+        id: mixed_cluster_job_to_delete
+  - match: { acknowledged: true }
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "mixed_cluster_job_to_delete"
+        body: >
+          {
+            "source": {
+              "index": "bwc_ml_regression_job_source",
+              "query": {"term" : { "user.keyword" : "Kimchy" }}
+            },
+            "dest": {
+              "index": "old_cluster_regression_job_results"
+            },
+            "analysis": {
+              "regression":{
+                "dependent_variable": "foo"
+              }
+            }
+          }
+  - match: { id: "mixed_cluster_job_to_delete" }
+
+  - do:
+      ml.delete_data_frame_analytics:
+        id: mixed_cluster_job_to_delete
+  - match: { acknowledged: true }

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml

@@ -0,0 +1,99 @@
+setup:
+
+  - do:
+      index:
+        index: bwc_ml_outlier_detection_job_source
+        body: >
+          {
+            "numeric_field_1": 42.0,
+            "user": "Kimchy"
+          }
+
+  - do:
+      index:
+        index: bwc_ml_regression_job_source
+        body: >
+          {
+            "numeric_field_1": 1.0,
+            "foo": 10.0,
+            "user": "Kimchy"
+          }
+
+  - do:
+      index:
+        index: bwc_ml_classification_job_source
+        body: >
+          {
+            "numeric_field_1": 1.0,
+            "foo": "a",
+            "user": "Kimchy"
+          }
+
+  - do:
+      indices.refresh:
+        index: bwc_ml_*
+
+---
+"Put outlier_detection job on the old cluster":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "old_cluster_outlier_detection_job"
+        body: >
+          {
+            "source": {
+              "index": "bwc_ml_outlier_detection_job_source",
+              "query": {"term" : { "user.keyword" : "Kimchy" }}
+            },
+            "dest": {
+              "index": "old_cluster_outlier_detection_job_results"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+  - match: { id: "old_cluster_outlier_detection_job" }
+
+---
+"Put regression job on the old cluster":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "old_cluster_regression_job"
+        body: >
+          {
+            "source": {
+              "index": "bwc_ml_regression_job_source",
+              "query": {"term" : { "user.keyword" : "Kimchy" }}
+            },
+            "dest": {
+              "index": "old_cluster_regression_job_results"
+            },
+            "analysis": {
+              "regression":{
+                "dependent_variable": "foo"
+              }
+            }
+          }
+  - match: { id: "old_cluster_regression_job" }
+
+---
+"Put classification job on the old cluster":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "old_cluster_classification_job"
+        body: >
+          {
+            "source": {
+              "index": "bwc_ml_classification_job_source",
+              "query": {"term" : { "user.keyword" : "Kimchy" }}
+            },
+            "dest": {
+              "index": "old_cluster_classification_job_results"
+            },
+            "analysis": {
+              "classification":{
+                "dependent_variable": "foo"
+              }
+            }
+          }
+  - match: { id: "old_cluster_classification_job" }

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml

@@ -0,0 +1,107 @@
+---
+"Get old cluster outlier_detection job":
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "old_cluster_outlier_detection_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_outlier_detection_job" }
+  - match: { data_frame_analytics.0.source.index: ["bwc_ml_outlier_detection_job_source"] }
+  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
+  - match: { data_frame_analytics.0.dest.index: "old_cluster_outlier_detection_job_results" }
+  - match: { data_frame_analytics.0.analysis: {
+    "outlier_detection":{
+      "compute_feature_influence": true,
+      "outlier_fraction": 0.05,
+      "standardization_enabled": true
+    }
+  }}
+
+---
+"Get old cluster outlier_detection job stats":
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "old_cluster_outlier_detection_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_outlier_detection_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
+---
+"Get old cluster regression job":
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "old_cluster_regression_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_regression_job" }
+  - match: { data_frame_analytics.0.source.index: ["bwc_ml_regression_job_source"] }
+  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
+  - match: { data_frame_analytics.0.dest.index: "old_cluster_regression_job_results" }
+  - match: { data_frame_analytics.0.analysis.regression.dependent_variable: "foo" }
+  - match: { data_frame_analytics.0.analysis.regression.training_percent: 100.0 }
+  - match: { data_frame_analytics.0.analysis.regression.loss_function: "mse" }
+  - is_true: data_frame_analytics.0.analysis.regression.randomize_seed
+
+---
+"Get old cluster regression job stats":
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "old_cluster_regression_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_regression_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
+---
+"Get old classification job":
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "old_cluster_classification_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
+  - match: { data_frame_analytics.0.source.index: ["bwc_ml_classification_job_source"] }
+  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
+  - match: { data_frame_analytics.0.dest.index: "old_cluster_classification_job_results" }
+  - match: { data_frame_analytics.0.analysis.classification.dependent_variable: "foo" }
+  - match: { data_frame_analytics.0.analysis.classification.training_percent: 100.0 }
+  - match: { data_frame_analytics.0.analysis.classification.class_assignment_objective: "maximize_minimum_recall" }
+  - is_true: data_frame_analytics.0.analysis.classification.randomize_seed
+
+---
+"Get old classification job stats":
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "old_cluster_classification_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
+---
+"Get mixed cluster outlier_detection job":
+  - do:
+      ml.get_data_frame_analytics:
+        id: "mixed_cluster_outlier_detection_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "mixed_cluster_outlier_detection_job" }
+  - match: { data_frame_analytics.0.source.index: ["bwc_ml_outlier_detection_job_source"] }
+  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
+  - match: { data_frame_analytics.0.dest.index: "mixed_cluster_outlier_detection_job_results" }
+  - match: { data_frame_analytics.0.analysis: {
+    "outlier_detection":{
+      "compute_feature_influence": true,
+      "outlier_fraction": 0.05,
+      "standardization_enabled": true
+    }
+  }}
+
+---
+"Get mixed cluster outlier_detection job stats":
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "mixed_cluster_outlier_detection_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "mixed_cluster_outlier_detection_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }