[7.x][Transforms] backport BWC tests for transforms crud (#46452)

backport 8.0 transform tests to 7.x

x-pack/qa/rolling-upgrade/build.gradle

@@ -160,11 +160,18 @@ for (Version version : bwcVersions.wireCompatible) {
     oldClusterTestRunner.configure {
         systemProperty 'tests.rest.suite', 'old_cluster'
         systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
+
+        def toBlackList = []
         // Dataframe transforms were not added until 7.2.0
         if (version.before('7.2.0')) {
-            systemProperty 'tests.rest.blacklist', [
-                    'old_cluster/80_data_frame_jobs_crud/Test put batch data frame transforms on old cluster'
-            ].join(',')
+            toBlackList << 'old_cluster/80_data_frame_jobs_crud/Test put batch data frame transforms on old cluster'
+        }
+        // continuous Dataframe transforms were not added until 7.3.0
+        if (version.before('7.3.0')) {
+            toBlackList << 'old_cluster/80_data_frame_jobs_crud/Test put continuous data frame transform on old cluster'
+        }
+        if (!toBlackList.empty) {
+            systemProperty 'tests.rest.blacklist', toBlackList.join(',')
         }
     }
 
@@ -223,12 +230,17 @@ for (Version version : bwcVersions.wireCompatible) {
                 'mixed_cluster/30_ml_jobs_crud/Create a job in the mixed cluster and write some data',
                 'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed without aggs in mixed cluster',
                 'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed with aggs in mixed cluster',
-                'mixed_cluster/80_data_frame_jobs_crud/Test put batch data frame transforms on mixed cluster'
+                'mixed_cluster/80_data_frame_jobs_crud/Test put batch data frame transforms on mixed cluster',
+                'mixed_cluster/80_data_frame_jobs_crud/Test put continuous data frame transform on mixed cluster'
         ]
         // Dataframe transforms were not added until 7.2.0
         if (version.before('7.2.0')) {
             toBlackList << 'mixed_cluster/80_data_frame_jobs_crud/Test GET, start, and stop old cluster batch transforms'
         }
+        // continuous Dataframe transforms were not added until 7.3.0
+        if (version.before('7.3.0')) {
+            toBlackList << 'mixed_cluster/80_data_frame_jobs_crud/Test GET, stop, start, old continuous transforms'
+        }
         systemProperty 'tests.rest.blacklist', toBlackList.join(',')
         finalizedBy "${baseName}#oldClusterTestCluster#node1.stop"
     }
@@ -245,13 +257,19 @@ for (Version version : bwcVersions.wireCompatible) {
         systemProperty 'tests.first_round', 'false'
         systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
         finalizedBy "${baseName}#oldClusterTestCluster#node2.stop"
+        def toBlackList = []
         // Dataframe transforms were not added until 7.2.0
         if (version.before('7.2.0')) {
-            systemProperty 'tests.rest.blacklist', [
-                    'mixed_cluster/80_data_frame_jobs_crud/Test put batch data frame transforms on mixed cluster',
-                    'mixed_cluster/80_data_frame_jobs_crud/Test GET, start, and stop old cluster batch transforms'
-
-            ].join(',')
+            toBlackList << 'mixed_cluster/80_data_frame_jobs_crud/Test put batch data frame transforms on mixed cluster'
+            toBlackList << 'mixed_cluster/80_data_frame_jobs_crud/Test GET, start, and stop old cluster batch transforms'
+        }
+        // continuous Dataframe transforms were not added until 7.3.0
+        if (version.before('7.3.0')) {
+            toBlackList << 'mixed_cluster/80_data_frame_jobs_crud/Test put continuous data frame transform on mixed cluster'
+            toBlackList << 'mixed_cluster/80_data_frame_jobs_crud/Test GET, stop, start, old continuous transforms'
+        }
+        if (!toBlackList.empty) {
+            systemProperty 'tests.rest.blacklist', toBlackList.join(',')
         }
     }
 
@@ -288,6 +306,10 @@ for (Version version : bwcVersions.wireCompatible) {
         if (version.before('7.2.0')) {
             toBlackList << 'upgraded_cluster/80_data_frame_jobs_crud/Get start, stop, and delete old and mixed cluster batch data frame transforms'
         }
+        // continuous Dataframe transforms were not added until 7.3.0
+        if (version.before('7.3.0')) {
+            toBlackList << 'upgraded_cluster/80_data_frame_jobs_crud/Test GET, stop, delete, old and mixed continuous transforms'
+        }
         if (!toBlackList.empty) {
             systemProperty 'tests.rest.blacklist', toBlackList.join(',')
         }

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_data_frame_jobs_crud.yml

@@ -31,7 +31,6 @@
   - match: { transforms.0.id: "mixed-simple-transform" }
   # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we cannot
   # assert on state in the mixed cluster as it could be state at the top level or state.task_state
-  # TODO: uncomment this assertion in master
   #- match: { transforms.0.state: "/started|indexing|stopping|stopped/" }
 
   - do:
@@ -47,7 +46,6 @@
   - match: { transforms.0.id: "mixed-simple-transform" }
   # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we cannot
   # assert on state in the mixed cluster as it could be state at the top level or state.task_state
-  # TODO: uncomment this assertion in master
   #- match: { transforms.0.state: "stopped" }
 
   - do:
@@ -94,7 +92,6 @@
   - match: { transforms.0.id: "mixed-complex-transform" }
   # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we cannot
   # assert on state in the mixed cluster as it could be state at the top level or state.task_state
-  # TODO: uncomment this assertion in master
   #- match: { transforms.0.state: "/started|indexing|stopping|stopped/" }
 
   - do:
@@ -110,14 +107,72 @@
   - match: { transforms.0.id: "mixed-complex-transform" }
   # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we cannot
   # assert on state in the mixed cluster as it could be state at the top level or state.task_state
-  # TODO: uncomment this assertion in master
   #- match: { transforms.0.state: "stopped" }
 
+---
+"Test put continuous data frame transform on mixed cluster":
+  - do:
+      cluster.health:
+        index: "dataframe-transform-airline-data-cont"
+        wait_for_status: green
+        timeout: 70s
+
+  - do:
+      data_frame.put_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+        body: >
+          {
+            "source": { "index": "dataframe-transform-airline-data-cont" },
+            "dest": { "index": "mixed-simple-continuous-transform-idx" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            },
+            "sync": {
+              "time": {
+                "field": "time",
+                "delay": "90m"
+              }
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.sync.time.field: "time" }
+  - match: { transforms.0.sync.time.delay: "90m" }
+  - is_true: transforms.0.version
+  - is_true: transforms.0.create_time
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.state: "/started|indexing/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.state: "stopped" }
+
 ---
 "Test GET, start, and stop old cluster batch transforms":
-  - skip:
-      reason: "https://github.com/elastic/elasticsearch/issues/44808"
-      version: "all"
   - do:
       cluster.health:
         index: "dataframe-transform-airline-data"
@@ -145,7 +200,6 @@
   - match: { transforms.0.id: "old-simple-transform" }
   # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we cannot
   # assert on state in the mixed cluster as it could be state at the top level or state.task_state
-  # TODO: uncomment this assertion in master
   #- match: { transforms.0.state: "/started|indexing|stopping|stopped/" }
 
   - do:
@@ -160,7 +214,6 @@
   - match: { transforms.0.id: "old-simple-transform" }
   # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we cannot
   # assert on state in the mixed cluster as it could be state at the top level or state.task_state
-  # TODO: uncomment this assertion in master
   #- match: { transforms.0.state: "stopped" }
 
   - do:
@@ -186,7 +239,6 @@
   - match: { transforms.0.id: "old-complex-transform" }
   # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we cannot
   # assert on state in the mixed cluster as it could be state at the top level or state.task_state
-  # TODO: uncomment this assertion in master
   #- match: { transforms.0.state: "/started|indexing|stopping|stopped/" }
 
   - do:
@@ -201,5 +253,46 @@
   - match: { transforms.0.id: "old-complex-transform" }
   # Since we are breaking the stats format between 7.3 and 7.4 (allowed because we're beta) we cannot
   # assert on state in the mixed cluster as it could be state at the top level or state.task_state
-  # TODO: uncomment this assertion in master
   #- match: { transforms.0.state: "stopped" }
+
+---
+"Test GET, stop, start, old continuous transforms":
+  - do:
+      cluster.health:
+        index: "dataframe-transform-airline-data-cont"
+        wait_for_status: green
+        timeout: 70s
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.sync.time.field: "time" }
+  - match: { transforms.0.sync.time.delay: "90m" }
+  - is_true: transforms.0.version
+  - is_true: transforms.0.create_time
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.state: "/started|indexing/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.state: "stopped" }

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_data_frame_jobs_crud.yml

@@ -116,3 +116,77 @@
         transform_id: "old-complex-transform"
   - match: { count: 1 }
   - match: { transforms.0.id: "old-complex-transform" }
+
+---
+"Test put continuous data frame transform on old cluster":
+  - do:
+      indices.create:
+        index: dataframe-transform-airline-data-cont
+        body:
+          mappings:
+            properties:
+              time:
+                type: date
+              airline:
+                type: keyword
+              responsetime:
+                type: float
+              event_rate:
+                type: integer
+  - do:
+      cluster.health:
+        index: "dataframe-transform-airline-data-cont"
+        wait_for_status: green
+        timeout: 70s
+
+  - do:
+      data_frame.put_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+        body: >
+          {
+            "source": { "index": "dataframe-transform-airline-data-cont" },
+            "dest": { "index": "old-simple-continuous-transform-idx" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            },
+            "sync": {
+              "time": {
+                "field": "time",
+                "delay": "90m"
+              }
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.sync.time.field: "time" }
+  - match: { transforms.0.sync.time.delay: "90m" }
+  - is_true: transforms.0.version
+  - is_true: transforms.0.create_time
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_data_frame_jobs_crud.yml

@@ -159,6 +159,97 @@ setup:
   - match: { count: 0 }
 
 ---
+"Test GET, stop, delete, old and mixed continuous transforms":
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.sync.time.field: "time" }
+  - match: { transforms.0.sync.time.delay: "90m" }
+  - is_true: transforms.0.version
+  - is_true: transforms.0.create_time
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.state: "/started|indexing/" }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.state: "/started|indexing/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.state: "stopped" }
+
+  - do:
+      data_frame.delete_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.sync.time.field: "time" }
+  - match: { transforms.0.sync.time.delay: "90m" }
+  - is_true: transforms.0.version
+  - is_true: transforms.0.create_time
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.state: "/started|indexing/" }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.state: "/started|indexing/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.state: "stopped" }
+
+  - do:
+      data_frame.delete_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+---
 "Test index mappings for latest internal index":
   - do:
       data_frame.put_data_frame_transform: