From 3d4952372670a1d99193fc444c949b56ec6ec919 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Mon, 25 Feb 2019 11:19:49 -0600 Subject: [PATCH] [DATA-FRAME] adds specs and yml tests for existing endpoints (#39326) (#39363) * [DATA-FRAME] adds specs and yml tests for existing endpoints * removing bad URL, adding test for _all --- ...ata_frame.delete_data_frame_transform.json | 20 ++++ .../data_frame.get_data_frame_transform.json | 18 +++ ..._frame.get_data_frame_transform_stats.json | 18 +++ ...ta_frame.preview_data_frame_transform.json | 14 +++ .../data_frame.put_data_frame_transform.json | 21 ++++ ...data_frame.start_data_frame_transform.json | 18 +++ .../data_frame.stop_data_frame_transform.json | 30 +++++ .../test/data_frame/preview_transforms.yml | 90 +++++++++++++++ .../test/data_frame/transforms_crud.yml | 106 ++++++++++++++++++ .../test/data_frame/transforms_start_stop.yml | 78 +++++++++++++ .../test/data_frame/transforms_stats.yml | 89 +++++++++++++++ 11 files changed, 502 insertions(+) create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.delete_data_frame_transform.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform_stats.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.preview_data_frame_transform.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.put_data_frame_transform.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.start_data_frame_transform.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.stop_data_frame_transform.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_stats.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.delete_data_frame_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.delete_data_frame_transform.json new file mode 100644 index 00000000000..78332c1d30f --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.delete_data_frame_transform.json @@ -0,0 +1,20 @@ +{ + "data_frame.delete_data_frame_transform": { + "documentation": "TODO", + "methods": [ "DELETE" ], + "url": { + "path": "/_data_frame/transforms/{transform_id}", + "paths": [ + "/_data_frame/transforms/{transform_id}" + ], + "parts": { + "transform_id": { + "type": "string", + "required": true, + "description": "The id of the transform to delete" + } + } + }, + "body": null + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform.json new file mode 100644 index 00000000000..e3653fecb98 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform.json @@ -0,0 +1,18 @@ +{ + "data_frame.get_data_frame_transform": { + "documentation": "TODO", + "methods": [ "GET" ], + "url": { + "path": "/_data_frame/transforms/{transform_id}", + "paths": [ "/_data_frame/transforms/{transform_id}"], + "parts": { + "transform_id": { + "type": "string", + "required": false, + "description": "The id of the transforms to get, '_all' or '*' implies get all transforms" + } + } + }, + "body": null + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform_stats.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform_stats.json new file mode 100644 index 00000000000..7af9f1f80b8 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform_stats.json @@ -0,0 +1,18 @@ +{ + "data_frame.get_data_frame_transform_stats": { + "documentation": "TODO", + "methods": [ "GET" ], + "url": { + "path": "/_data_frame/transforms/{transform_id}/_stats", + "paths": [ "/_data_frame/transforms/{transform_id}/_stats" ], + "parts": { + "transform_id": { + "type": "string", + "required": false, + "description": "The id of the transform for which to get stats. '_all' or '*' implies all transforms" + } + } + }, + "body": null + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.preview_data_frame_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.preview_data_frame_transform.json new file mode 100644 index 00000000000..e5e9b1be949 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.preview_data_frame_transform.json @@ -0,0 +1,14 @@ +{ + "data_frame.preview_data_frame_transform": { + "documentation": "TODO", + "methods": [ "POST" ], + "url": { + "path": "/_data_frame/transforms/_preview", + "paths": [ "/_data_frame/transforms/_preview" ] + }, + "body": { + "description" : "The definition for the data_frame transform to preview", + "required": true + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.put_data_frame_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.put_data_frame_transform.json new file mode 100644 index 00000000000..8f424bb445e --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.put_data_frame_transform.json @@ -0,0 +1,21 @@ +{ + "data_frame.put_data_frame_transform": { + "documentation": "TODO", + "methods": [ "PUT" ], + "url": { + "path": "/_data_frame/transforms/{transform_id}", + "paths": [ "/_data_frame/transforms/{transform_id}" ], + "parts": { + "transform_id": { + "type": "string", + "required": true, + "description": "The id of the new transform." + } + } + }, + "body": { + "description" : "The data frame transform definition", + "required": true + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.start_data_frame_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.start_data_frame_transform.json new file mode 100644 index 00000000000..9bcd311ed52 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.start_data_frame_transform.json @@ -0,0 +1,18 @@ +{ + "data_frame.start_data_frame_transform": { + "documentation": "TODO", + "methods": [ "POST" ], + "url": { + "path": "/_data_frame/transforms/{transform_id}/_start", + "paths": [ "/_data_frame/transforms/{transform_id}/_start" ], + "parts": { + "transform_id": { + "type": "string", + "required": true, + "description": "The id of the transform to start" + } + } + }, + "body": null + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.stop_data_frame_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.stop_data_frame_transform.json new file mode 100644 index 00000000000..9d64ba7082a --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.stop_data_frame_transform.json @@ -0,0 +1,30 @@ +{ + "data_frame.stop_data_frame_transform": { + "documentation": "TODO", + "methods": [ "POST" ], + "url": { + "path": "/_data_frame/transforms/{transform_id}/_stop", + "paths": [ "/_data_frame/transforms/{transform_id}/_stop" ], + "parts": { + "transform_id": { + "type": "string", + "required": true, + "description": "The id of the transform to stop" + } + }, + "params": { + "wait_for_completion": { + "type": "boolean", + "required": false, + "description": "Whether to wait for the transform to fully stop before returning or not. Default to false" + }, + "timeout": { + "type": "time", + "required": false, + "description": "Controls the time to wait until the transform has stopped. Default to 30 seconds" + } + } + }, + "body": null + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml new file mode 100644 index 00000000000..3bad56e66bd --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml @@ -0,0 +1,90 @@ +setup: + - do: + indices.create: + index: airline-data + body: + mappings: + properties: + time: + type: date + airline: + type: keyword + responsetime: + type: float + event_rate: + type: integer + - do: + index: + index: airline-data + id: 1 + body: > + { + "time": "2017-02-18T00:00:00Z", + "airline": "foo", + "responsetime": 1.0, + "event_rate": 5 + } + + - do: + index: + index: airline-data + id: 2 + body: > + { + "time": "2017-02-18T00:30:00Z", + "airline": "foo", + "responsetime": 1.0, + "event_rate": 6 + } + + - do: + index: + index: airline-data + id: 3 + body: > + { + "time": "2017-02-18T01:00:00Z", + "airline": "bar", + "responsetime": 42.0, + "event_rate": 8 + } + + - do: + index: + index: airline-data + id: 4 + body: > + { + "time": "2017-02-18T01:01:00Z", + "airline": "foo", + "responsetime": 42.0, + "event_rate": 7 + } + + - do: + indices.refresh: + index: airline-data + +--- +"Test preview transform": + - do: + data_frame.preview_data_frame_transform: + body: > + { + "source": "airline-data", + "pivot": { + "group_by": { + "airline": {"terms": {"field": "airline"}}, + "by-hour": {"date_histogram": {"interval": "1h", "field": "time", "format": "yyyy-MM-DD HH"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + } + } + - match: { preview.0.airline: foo } + - match: { preview.0.by-hour: "2017-02-49 00" } + - match: { preview.0.avg_response: 1.0 } + - match: { preview.1.airline: bar } + - match: { preview.1.by-hour: "2017-02-49 01" } + - match: { preview.1.avg_response: 42.0 } + - match: { preview.2.airline: foo } + - match: { preview.2.by-hour: "2017-02-49 01" } + - match: { preview.2.avg_response: 42.0 } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml new file mode 100644 index 00000000000..179ee908086 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml @@ -0,0 +1,106 @@ +setup: + - do: + indices.create: + index: airline-data + body: + mappings: + properties: + time: + type: date + airline: + type: keyword + responsetime: + type: float + event_rate: + type: integer + +--- +"Test get all transforms when there are none": + - do: + data_frame.get_data_frame_transform: + transform_id: "*" + - match: { count: 0 } + - match: { transforms: [] } + +--- +"Test delete transform when it does not exist": + - do: + catch: missing + data_frame.delete_data_frame_transform: + transform_id: "missing transform" + +--- +"Test put transform with invalid source index": + - do: + catch: /Failed to validate data frame configuration/ + data_frame.put_data_frame_transform: + transform_id: "missing-source-transform" + body: > + { + "source": "missing-index", + "dest": "missing-source-dest", + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + } + } +--- +"Test basic transform crud": + - do: + data_frame.put_data_frame_transform: + transform_id: "airline-transform" + body: > + { + "source": "airline-data", + "dest": "airline-data-by-airline", + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + } + } + - match: { acknowledged: true } + + - do: + data_frame.put_data_frame_transform: + transform_id: "airline-transform-dos" + body: > + { + "source": "airline-data", + "dest": "airline-data-by-airline-again", + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + } + } + - match: { acknowledged: true } + + - do: + data_frame.get_data_frame_transform: + transform_id: "airline-transform" + - match: { count: 1 } + - match: { transforms.0.id: "airline-transform" } + - match: { transforms.0.source: "airline-data" } + - match: { transforms.0.dest: "airline-data-by-airline" } + - is_true: transforms.0.query.match_all + - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" } + - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" } + + - do: + data_frame.get_data_frame_transform: + transform_id: "*" + - match: { count: 2 } + + - do: + data_frame.get_data_frame_transform: + transform_id: "_all" + - match: { count: 2 } + + - do: + data_frame.delete_data_frame_transform: + transform_id: "airline-transform" + - match: { acknowledged: true } + + - do: + data_frame.delete_data_frame_transform: + transform_id: "airline-transform-dos" + - match: { acknowledged: true } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml new file mode 100644 index 00000000000..94281e2eb76 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml @@ -0,0 +1,78 @@ +setup: + - do: + indices.create: + index: airline-data + body: + mappings: + properties: + time: + type: date + airline: + type: keyword + responsetime: + type: float + event_rate: + type: integer + - do: + data_frame.put_data_frame_transform: + transform_id: "airline-transform-start-stop" + body: > + { + "source": "airline-data", + "dest": "airline-data-by-airline-start-stop", + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + } + } + +--- +teardown: + - do: + data_frame.stop_data_frame_transform: + transform_id: "airline-transform-start-stop" + timeout: "10m" + wait_for_completion: true + - do: + data_frame.delete_data_frame_transform: + transform_id: "airline-transform-start-stop" + +--- +"Test start transform": + - do: + data_frame.start_data_frame_transform: + transform_id: "airline-transform-start-stop" + - match: { started: true } + +--- +"Test start missing transform": + - do: + catch: missing + data_frame.start_data_frame_transform: + transform_id: "missing-transform" + +--- +"Test start already started transform": + - do: + data_frame.start_data_frame_transform: + transform_id: "airline-transform-start-stop" + - match: { started: true } + + - do: + catch: /Cannot start task for data frame transform \[airline-transform-start-stop\], because state was \[STARTED\]/ + data_frame.start_data_frame_transform: + transform_id: "airline-transform-start-stop" + +--- +"Test stop missing transform": + - do: + catch: missing + data_frame.stop_data_frame_transform: + transform_id: "missing-transform" + +--- +"Test stop already stopped transform": + - do: + data_frame.stop_data_frame_transform: + transform_id: "airline-transform-start-stop" + - match: { stopped: true } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_stats.yml new file mode 100644 index 00000000000..fbf3ceec8f0 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_stats.yml @@ -0,0 +1,89 @@ +setup: + - do: + indices.create: + index: airline-data + body: + mappings: + properties: + time: + type: date + airline: + type: keyword + responsetime: + type: float + event_rate: + type: integer + - do: + data_frame.put_data_frame_transform: + transform_id: "airline-transform-stats" + body: > + { + "source": "airline-data", + "dest": "airline-data-by-airline-stats", + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + } + } + +--- +teardown: + - do: + data_frame.delete_data_frame_transform: + transform_id: "airline-transform-stats" + +--- +"Test get transform stats": + - do: + data_frame.get_data_frame_transform_stats: + transform_id: "airline-transform-stats" + - match: { count: 1 } + - match: { transforms.0.id: "airline-transform-stats" } + - match: { transforms.0.state.transform_state: "stopped" } + - match: { transforms.0.state.generation: 0 } + - match: { transforms.0.stats.pages_processed: 0 } + - match: { transforms.0.stats.documents_processed: 0 } + - match: { transforms.0.stats.documents_indexed: 0 } + - match: { transforms.0.stats.trigger_count: 0 } + - match: { transforms.0.stats.index_time_in_ms: 0 } + - match: { transforms.0.stats.index_total: 0 } + - match: { transforms.0.stats.index_failures: 0 } + - match: { transforms.0.stats.search_time_in_ms: 0 } + - match: { transforms.0.stats.search_total: 0 } + - match: { transforms.0.stats.search_failures: 0 } + +--- +"Test get transform stats on missing transform": + - do: + data_frame.get_data_frame_transform_stats: + transform_id: "missing-transform" + - match: { count: 0 } + - match: { transforms: [] } + +--- +"Test get multiple transform stats": + - do: + data_frame.put_data_frame_transform: + transform_id: "airline-transform-stats-dos" + body: > + { + "source": "airline-data", + "dest": "airline-data-by-airline-stats-dos", + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + } + } + - do: + data_frame.get_data_frame_transform_stats: + transform_id: "*" + - match: { count: 2 } + + - do: + data_frame.get_data_frame_transform_stats: + transform_id: "_all" + - match: { count: 2 } + + - do: + data_frame.delete_data_frame_transform: + transform_id: "airline-transform-stats-dos"