From 844f350774fc0077d1705a2ad00a06168907881a Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 3 Mar 2020 10:30:55 +0100 Subject: [PATCH] [Transform] restructure transform yaml tests (#52956) restructure transform yaml tests to run cleanup in teardown phase relates #52428 --- .../test/transform/transforms_crud.yml | 43 ----- .../transform/transforms_force_delete.yml | 70 ++++++++ .../test/transform/transforms_stats.yml | 151 ++++-------------- .../transform/transforms_stats_continuous.yml | 74 +++++++++ 4 files changed, 178 insertions(+), 160 deletions(-) create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_force_delete.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_stats_continuous.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml index e66982aadec..9b9cb6ae21b 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml @@ -605,49 +605,6 @@ setup: "version": "7.3.0" } --- -"Test force deleting a running transform": - - do: - transform.put_transform: - transform_id: "airline-transform-start-delete" - body: > - { - "source": { "index": "airline-data" }, - "dest": { "index": "airline-data-by-airline-start-delete" }, - "pivot": { - "group_by": { "airline": {"terms": {"field": "airline"}}}, - "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} - }, - "sync": { - "time": { - "field": "time", - "delay": "90m" - } - } - } - - match: { acknowledged: true } - - do: - transform.start_transform: - transform_id: "airline-transform-start-delete" - - match: { acknowledged: true } - - - do: - transform.get_transform_stats: - transform_id: "airline-transform-start-delete" - - match: { count: 1 } - - match: { transforms.0.id: "airline-transform-start-delete" } - - match: { transforms.0.state: "/started|indexing/" } - - - do: - catch: /Cannot delete transform \[airline-transform-start-delete\] as the task is running/ - transform.delete_transform: - transform_id: "airline-transform-start-delete" - - - do: - transform.delete_transform: - transform_id: "airline-transform-start-delete" - force: true - - match: { acknowledged: true } ---- "Test put transform with missing pipeline": - do: catch: /Pipeline with id \[missing-transform-pipeline\] could not be found/ diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_force_delete.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_force_delete.yml new file mode 100644 index 00000000000..b763fcaf478 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_force_delete.yml @@ -0,0 +1,70 @@ +setup: + - do: + indices.create: + index: airline-data + body: + mappings: + properties: + time: + type: date + airline: + type: keyword + responsetime: + type: float + event_rate: + type: integer + - do: + transform.put_transform: + transform_id: "airline-transform-start-delete" + body: > + { + "source": { "index": "airline-data" }, + "dest": { "index": "airline-data-by-airline-start-delete" }, + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + }, + "sync": { + "time": { + "field": "time", + "delay": "90m" + } + } + } + +--- +teardown: + - do: + transform.stop_transform: + wait_for_checkpoint: false + transform_id: "airline-transform-start-delete" + wait_for_completion: true + ignore: 404 + - do: + transform.delete_transform: + transform_id: "airline-transform-start-delete" + ignore: 404 +--- +"Test force deleting a running transform": + - do: + transform.start_transform: + transform_id: "airline-transform-start-delete" + - match: { acknowledged: true } + + - do: + transform.get_transform_stats: + transform_id: "airline-transform-start-delete" + - match: { count: 1 } + - match: { transforms.0.id: "airline-transform-start-delete" } + - match: { transforms.0.state: "/started|indexing/" } + + - do: + catch: /Cannot delete transform \[airline-transform-start-delete\] as the task is running/ + transform.delete_transform: + transform_id: "airline-transform-start-delete" + + - do: + transform.delete_transform: + transform_id: "airline-transform-start-delete" + force: true + - match: { acknowledged: true } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_stats.yml index f0c55455638..e2385257204 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_stats.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_stats.yml @@ -25,6 +25,30 @@ setup: "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} } } + - do: + transform.put_transform: + transform_id: "airline-transform-stats-dos" + body: > + { + "source": { "index": "airline-data" }, + "dest": { "index": "airline-data-by-airline-stats-dos" }, + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + } + } + - do: + transform.put_transform: + transform_id: "airline-transform-stats-the-third" + body: > + { + "source": { "index": "airline-data" }, + "dest": { "index": "airline-data-by-airline-stats-the-third" }, + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + } + } - do: transform.start_transform: transform_id: "airline-transform-stats" @@ -36,11 +60,16 @@ teardown: wait_for_checkpoint: false transform_id: "airline-transform-stats" wait_for_completion: true - - do: transform.delete_transform: transform_id: "airline-transform-stats" + - do: + transform.delete_transform: + transform_id: "airline-transform-stats-dos" + - do: + transform.delete_transform: + transform_id: "airline-transform-stats-the-third" --- "Test get transform stats": - do: @@ -71,30 +100,6 @@ teardown: --- "Test get multiple transform stats": - - do: - transform.put_transform: - transform_id: "airline-transform-stats-dos" - body: > - { - "source": { "index": "airline-data" }, - "dest": { "index": "airline-data-by-airline-stats-dos" }, - "pivot": { - "group_by": { "airline": {"terms": {"field": "airline"}}}, - "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} - } - } - - do: - transform.put_transform: - transform_id: "airline-transform-stats-the-third" - body: > - { - "source": { "index": "airline-data" }, - "dest": { "index": "airline-data-by-airline-stats-the-third" }, - "pivot": { - "group_by": { "airline": {"terms": {"field": "airline"}}}, - "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} - } - } - do: transform.get_transform_stats: transform_id: "*" @@ -135,58 +140,26 @@ teardown: - match: { transforms.0.id: "airline-transform-stats-dos" } - match: { transforms.1.id: "airline-transform-stats-the-third" } - - do: - transform.delete_transform: - transform_id: "airline-transform-stats-dos" - - - do: - transform.delete_transform: - transform_id: "airline-transform-stats-the-third" - - --- "Test get multiple transform stats where one does not have a task": - - do: - transform.put_transform: - transform_id: "airline-transform-stats-dos" - body: > - { - "source": { "index": "airline-data" }, - "dest": { "index": "airline-data-by-airline-stats-dos" }, - "pivot": { - "group_by": { "airline": {"terms": {"field": "airline"}}}, - "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} - } - } - do: transform.get_transform_stats: transform_id: "*" - - match: { count: 2 } + - match: { count: 3 } - match: { transforms.0.id: "airline-transform-stats" } - match: { transforms.1.id: "airline-transform-stats-dos" } + - match: { transforms.2.id: "airline-transform-stats-the-third" } - do: transform.get_transform_stats: transform_id: "_all" - - match: { count: 2 } + - match: { count: 3 } - match: { transforms.0.id: "airline-transform-stats" } - match: { transforms.1.id: "airline-transform-stats-dos" } + - match: { transforms.2.id: "airline-transform-stats-the-third" } --- "Test get single transform stats when it does not have a task": - - - do: - transform.put_transform: - transform_id: "airline-transform-stats-dos" - body: > - { - "source": { "index": "airline-data" }, - "dest": { "index": "airline-data-by-airline-stats-dos" }, - "pivot": { - "group_by": { "airline": {"terms": {"field": "airline"}}}, - "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} - } - } - do: transform.get_transform_stats: transform_id: "airline-transform-stats-dos" @@ -204,59 +177,3 @@ teardown: - match: { transforms.0.stats.search_time_in_ms: 0 } - match: { transforms.0.stats.search_total: 0 } - match: { transforms.0.stats.search_failures: 0 } ---- -"Test get continuous transform stats": - - do: - transform.put_transform: - transform_id: "airline-transform-stats-continuous" - body: > - { - "source": { "index": "airline-data" }, - "dest": { "index": "airline-data-by-airline-stats-continuous" }, - "pivot": { - "group_by": { "airline": {"terms": {"field": "airline"}}}, - "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} - }, - "sync": { "time": { "field": "time", "delay": "1m" } } - } - - do: - transform.start_transform: - transform_id: "airline-transform-stats-continuous" - - do: - transform.get_transform_stats: - transform_id: "airline-transform-stats-continuous" - - match: { count: 1 } - - match: { transforms.0.id: "airline-transform-stats-continuous" } - # Since this is continuous, there is no worry of it automatically stopping - - match: { transforms.0.state: "/started|indexing/" } - - lte: { transforms.0.checkpointing.last.checkpoint: 1 } - # Since this is continuous, and _start does not return until it is assigned - # we should see a node assignment - - is_true: transforms.0.node - - is_true: transforms.0.node.id - - is_true: transforms.0.node.name - - is_true: transforms.0.node.ephemeral_id - - is_true: transforms.0.node.transport_address - - lte: { transforms.0.stats.pages_processed: 1 } - - match: { transforms.0.stats.documents_processed: 0 } - - match: { transforms.0.stats.documents_indexed: 0 } - - lte: { transforms.0.stats.trigger_count: 1 } - - match: { transforms.0.stats.index_time_in_ms: 0 } - - match: { transforms.0.stats.index_total: 0 } - - match: { transforms.0.stats.index_failures: 0 } - - gte: { transforms.0.stats.search_time_in_ms: 0 } - - lte: { transforms.0.stats.search_total: 1 } - - match: { transforms.0.stats.search_failures: 0 } - - is_true: transforms.0.stats.exponential_avg_checkpoint_duration_ms - - match: { transforms.0.stats.exponential_avg_documents_indexed: 0.0 } - - match: { transforms.0.stats.exponential_avg_documents_processed: 0.0 } - - - do: - transform.stop_transform: - wait_for_checkpoint: false - transform_id: "airline-transform-stats-continuous" - wait_for_completion: true - - - do: - transform.delete_transform: - transform_id: "airline-transform-stats-continuous" diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_stats_continuous.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_stats_continuous.yml new file mode 100644 index 00000000000..f4042271518 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_stats_continuous.yml @@ -0,0 +1,74 @@ +setup: + - do: + indices.create: + index: airline-data + body: + mappings: + properties: + time: + type: date + airline: + type: keyword + responsetime: + type: float + event_rate: + type: integer + - do: + transform.put_transform: + transform_id: "airline-transform-stats-continuous" + body: > + { + "source": { "index": "airline-data" }, + "dest": { "index": "airline-data-by-airline-stats-continuous" }, + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + }, + "sync": { "time": { "field": "time", "delay": "1m" } } + } + - do: + transform.start_transform: + transform_id: "airline-transform-stats-continuous" + +--- +teardown: + - do: + transform.stop_transform: + wait_for_checkpoint: false + transform_id: "airline-transform-stats-continuous" + wait_for_completion: true + + - do: + transform.delete_transform: + transform_id: "airline-transform-stats-continuous" + +--- +"Test get continuous transform stats": + - do: + transform.get_transform_stats: + transform_id: "airline-transform-stats-continuous" + - match: { count: 1 } + - match: { transforms.0.id: "airline-transform-stats-continuous" } + # Since this is continuous, there is no worry of it automatically stopping + - match: { transforms.0.state: "/started|indexing/" } + - lte: { transforms.0.checkpointing.last.checkpoint: 1 } + # Since this is continuous, and _start does not return until it is assigned + # we should see a node assignment + - is_true: transforms.0.node + - is_true: transforms.0.node.id + - is_true: transforms.0.node.name + - is_true: transforms.0.node.ephemeral_id + - is_true: transforms.0.node.transport_address + - lte: { transforms.0.stats.pages_processed: 1 } + - match: { transforms.0.stats.documents_processed: 0 } + - match: { transforms.0.stats.documents_indexed: 0 } + - lte: { transforms.0.stats.trigger_count: 1 } + - match: { transforms.0.stats.index_time_in_ms: 0 } + - match: { transforms.0.stats.index_total: 0 } + - match: { transforms.0.stats.index_failures: 0 } + - gte: { transforms.0.stats.search_time_in_ms: 0 } + - lte: { transforms.0.stats.search_total: 1 } + - match: { transforms.0.stats.search_failures: 0 } + - is_true: transforms.0.stats.exponential_avg_checkpoint_duration_ms + - match: { transforms.0.stats.exponential_avg_documents_indexed: 0.0 } + - match: { transforms.0.stats.exponential_avg_documents_processed: 0.0 }