From 9ba017f6999040ea814e29d659f1c1afb9b44097 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Wed, 22 Jul 2020 11:22:57 -0700 Subject: [PATCH] [DOCS] Changes level offset of transform pages (#60066) (#60075) --- docs/reference/data-rollup-transform.asciidoc | 2 +- .../reference/transform/api-quickref.asciidoc | 20 ++-- .../transform/apis/delete-transform.asciidoc | 12 +-- .../apis/get-transform-stats.asciidoc | 18 ++-- .../transform/apis/get-transform.asciidoc | 18 ++-- docs/reference/transform/apis/index.asciidoc | 30 ++---- .../transform/apis/preview-transform.asciidoc | 14 +-- .../transform/apis/put-transform.asciidoc | 16 ++-- .../transform/apis/start-transform.asciidoc | 12 +-- .../transform/apis/stop-transform.asciidoc | 16 ++-- .../transform/apis/transform-apis.asciidoc | 13 +++ .../transform/apis/update-transform.asciidoc | 18 ++-- docs/reference/transform/checkpoints.asciidoc | 4 +- .../transform/ecommerce-tutorial.asciidoc | 2 +- docs/reference/transform/examples.asciidoc | 10 +- docs/reference/transform/index.asciidoc | 43 +++------ docs/reference/transform/limitations.asciidoc | 96 +++++++++---------- docs/reference/transform/overview.asciidoc | 6 +- .../transform/painless-examples.asciidoc | 31 ++---- docs/reference/transform/setup.asciidoc | 6 +- docs/reference/transform/transforms.asciidoc | 21 ++++ .../transform/troubleshooting.asciidoc | 11 +-- docs/reference/transform/usage.asciidoc | 2 +- 23 files changed, 199 insertions(+), 222 deletions(-) create mode 100644 docs/reference/transform/apis/transform-apis.asciidoc create mode 100644 docs/reference/transform/transforms.asciidoc diff --git a/docs/reference/data-rollup-transform.asciidoc b/docs/reference/data-rollup-transform.asciidoc index 413b7d89d82..81ed4a50788 100644 --- a/docs/reference/data-rollup-transform.asciidoc +++ b/docs/reference/data-rollup-transform.asciidoc @@ -11,7 +11,7 @@ include::rollup/index.asciidoc[tag=rollup-intro] * <> + -include::transform/index.asciidoc[tag=transform-intro] +include::transform/transforms.asciidoc[tag=transform-intro] -- diff --git a/docs/reference/transform/api-quickref.asciidoc b/docs/reference/transform/api-quickref.asciidoc index 877ae240969..1db336b5beb 100644 --- a/docs/reference/transform/api-quickref.asciidoc +++ b/docs/reference/transform/api-quickref.asciidoc @@ -1,6 +1,6 @@ [role="xpack"] [[transform-api-quickref]] -=== API quick reference += API quick reference All {transform} endpoints have the following base: @@ -10,13 +10,13 @@ _transform/ ---- // NOTCONSOLE -* {ref}/put-transform.html[Create {transforms}] -* {ref}/delete-transform.html[Delete {transforms}] -* {ref}/get-transform.html[Get {transforms}] -* {ref}/get-transform-stats.html[Get {transforms} statistics] -* {ref}/preview-transform.html[Preview {transforms}] -* {ref}/start-transform.html[Start {transforms}] -* {ref}/stop-transform.html[Stop {transforms}] -* {ref}/update-transform.html[Update {transforms}] +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> -For the full list, see {ref}/transform-apis.html[{transform-cap} APIs]. +For the full list, see <>. diff --git a/docs/reference/transform/apis/delete-transform.asciidoc b/docs/reference/transform/apis/delete-transform.asciidoc index 021a21d8525..a9911f74932 100644 --- a/docs/reference/transform/apis/delete-transform.asciidoc +++ b/docs/reference/transform/apis/delete-transform.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[delete-transform]] -=== Delete {transform} API += Delete {transform} API [subs="attributes"] ++++ @@ -11,12 +11,12 @@ Deletes an existing {transform}. [[delete-transform-request]] -==== {api-request-title} +== {api-request-title} `DELETE _transform/` [[delete-transform-prereqs]] -==== {api-prereq-title} +== {api-prereq-title} * Before you can delete the {transform}, you must stop it. @@ -31,14 +31,14 @@ For more information, see <> and <>. [[delete-transform-path-parms]] -==== {api-path-parms-title} +== {api-path-parms-title} ``:: (Required, string) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] [[delete-transform-query-parms]] -==== {api-query-parms-title} +== {api-query-parms-title} `force`:: (Optional, boolean) When `true`, the {transform} is deleted regardless of its @@ -46,7 +46,7 @@ current state. The default value is `false`, meaning that the {transform} must b `stopped` before it can be deleted. [[delete-transform-examples]] -==== {api-examples-title} +== {api-examples-title} [source,console] -------------------------------------------------- diff --git a/docs/reference/transform/apis/get-transform-stats.asciidoc b/docs/reference/transform/apis/get-transform-stats.asciidoc index 12897fd7405..ceb1a7d194d 100644 --- a/docs/reference/transform/apis/get-transform-stats.asciidoc +++ b/docs/reference/transform/apis/get-transform-stats.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[get-transform-stats]] -=== Get {transform} statistics API += Get {transform} statistics API [subs="attributes"] ++++ @@ -12,7 +12,7 @@ Retrieves usage information for {transforms}. [[get-transform-stats-request]] -==== {api-request-title} +== {api-request-title} `GET _transform//_stats` @@ -26,7 +26,7 @@ Retrieves usage information for {transforms}. [[get-transform-stats-prereqs]] -==== {api-prereq-title} +== {api-prereq-title} If the {es} {security-features} are enabled, you must have the following privileges: @@ -39,7 +39,7 @@ For more information, see <> and <>. [[get-transform-stats-desc]] -==== {api-description-title} +== {api-description-title} You can get statistics for multiple {transforms} in a single API request by using a comma-separated list of identifiers or a wildcard expression. @@ -49,7 +49,7 @@ specifying `*` as the ``, or by omitting the [[get-transform-stats-path-parms]] -==== {api-path-parms-title} +== {api-path-parms-title} ``:: (Optional, string) @@ -57,7 +57,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard] [[get-transform-stats-query-parms]] -==== {api-query-parms-title} +== {api-query-parms-title} `allow_no_match`:: (Optional, boolean) @@ -73,7 +73,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=size-transforms] [role="child_attributes"] [[get-transform-stats-response]] -==== {api-response-body-title} +== {api-response-body-title} The API returns an array of statistics objects for {transforms}, which are sorted by the `id` value in ascending order. All of these properties are @@ -251,14 +251,14 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=trigger-count] //End stats [[get-transform-stats-response-codes]] -==== {api-response-codes-title} +== {api-response-codes-title} `404` (Missing resources):: If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[get-transform-stats-example]] -==== Examples +== Examples The following example skips for the first five {transforms} and gets usage information for a maximum of ten results: diff --git a/docs/reference/transform/apis/get-transform.asciidoc b/docs/reference/transform/apis/get-transform.asciidoc index 2a9ec595546..939bc59413d 100644 --- a/docs/reference/transform/apis/get-transform.asciidoc +++ b/docs/reference/transform/apis/get-transform.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[get-transform]] -=== Get {transforms} API += Get {transforms} API [subs="attributes"] ++++ @@ -11,7 +11,7 @@ Retrieves configuration information for {transforms}. [[get-transform-request]] -==== {api-request-title} +== {api-request-title} `GET _transform/` + @@ -24,7 +24,7 @@ Retrieves configuration information for {transforms}. `GET _transform/*` [[get-transform-prereqs]] -==== {api-prereq-title} +== {api-prereq-title} If the {es} {security-features} are enabled, you must have the following privileges: @@ -36,7 +36,7 @@ The built-in `transform_user` role has this privilege. For more information, see <> and <>. [[get-transform-desc]] -==== {api-description-title} +== {api-description-title} You can get information for multiple {transforms} in a single API request by using a comma-separated list of identifiers or a wildcard expression. @@ -44,14 +44,14 @@ You can get information for all {transforms} by using `_all`, by specifying `*` as the ``, or by omitting the ``. [[get-transform-path-parms]] -==== {api-path-parms-title} +== {api-path-parms-title} ``:: (Optional, string) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard] [[get-transform-query-parms]] -==== {api-query-parms-title} +== {api-query-parms-title} `allow_no_match`:: (Optional, boolean) @@ -66,7 +66,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from-transforms] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=size-transforms] [[get-transform-response]] -==== {api-response-body-title} +== {api-response-body-title} The API returns an array of {transform} resources, which are sorted by the `id` value in ascending order. For the full list of properties, see @@ -81,14 +81,14 @@ This property is informational; you cannot change its value. created. [[get-transform-response-codes]] -==== {api-response-codes-title} +== {api-response-codes-title} `404` (Missing resources):: If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[get-transform-example]] -==== {api-examples-title} +== {api-examples-title} The following example retrieves information about a maximum of ten {transforms}: diff --git a/docs/reference/transform/apis/index.asciidoc b/docs/reference/transform/apis/index.asciidoc index c2c7374deec..b0948a7adb3 100644 --- a/docs/reference/transform/apis/index.asciidoc +++ b/docs/reference/transform/apis/index.asciidoc @@ -1,29 +1,17 @@ -[role="xpack"] -[testenv="basic"] -[[transform-apis]] -== {transform-cap} APIs - -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> +include::transform-apis.asciidoc[leveloffset=+1] //CREATE -include::put-transform.asciidoc[] +include::put-transform.asciidoc[leveloffset=+2] //DELETE -include::delete-transform.asciidoc[] +include::delete-transform.asciidoc[leveloffset=+2] //GET -include::get-transform.asciidoc[] -include::get-transform-stats.asciidoc[] +include::get-transform.asciidoc[leveloffset=+2] +include::get-transform-stats.asciidoc[leveloffset=+2] //PREVIEW -include::preview-transform.asciidoc[] +include::preview-transform.asciidoc[leveloffset=+2] //START -include::start-transform.asciidoc[] +include::start-transform.asciidoc[leveloffset=+2] //STOP -include::stop-transform.asciidoc[] +include::stop-transform.asciidoc[leveloffset=+2] //UPDATE -include::update-transform.asciidoc[] \ No newline at end of file +include::update-transform.asciidoc[leveloffset=+2] \ No newline at end of file diff --git a/docs/reference/transform/apis/preview-transform.asciidoc b/docs/reference/transform/apis/preview-transform.asciidoc index 2ef5daa72ae..8c9b2407f19 100644 --- a/docs/reference/transform/apis/preview-transform.asciidoc +++ b/docs/reference/transform/apis/preview-transform.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[preview-transform]] -=== Preview {transform} API += Preview {transform} API [subs="attributes"] ++++ @@ -11,12 +11,12 @@ Previews a {transform}. [[preview-transform-request]] -==== {api-request-title} +== {api-request-title} `POST _transform/_preview` [[preview-transform-prereq]] -==== {api-prereq-title} +== {api-prereq-title} If the {es} {security-features} are enabled, you must have the following privileges: @@ -30,7 +30,7 @@ For more information, see <> and <>. [[preview-transform-desc]] -==== {api-description-title} +== {api-description-title} This API generates a preview of the results that you will get when you run the <> with the same @@ -49,7 +49,7 @@ or an index template with your preferred mappings before you start the [role="child_attributes"] [[preview-transform-request-body]] -==== {api-request-body-title} +== {api-request-body-title} `description`:: @@ -164,7 +164,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max [role="child_attributes"] [[preview-transform-response]] -==== {api-response-body-title} +== {api-response-body-title} `preview`:: (array) An array of documents. In particular, they are the JSON representation @@ -190,7 +190,7 @@ of the documents that would be created in the destination index by the ==== //End generated_dest_index -==== {api-examples-title} +== {api-examples-title} [source,console] -------------------------------------------------- diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc index 60daff036ab..86dcead55a0 100644 --- a/docs/reference/transform/apis/put-transform.asciidoc +++ b/docs/reference/transform/apis/put-transform.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[put-transform]] -=== Create {transform} API += Create {transform} API [subs="attributes"] ++++ @@ -11,12 +11,12 @@ Instantiates a {transform}. [[put-transform-request]] -==== {api-request-title} +== {api-request-title} `PUT _transform/` [[put-transform-prereqs]] -==== {api-prereq-title} +== {api-prereq-title} If the {es} {security-features} are enabled, you must have the following built-in roles and privileges: @@ -32,7 +32,7 @@ For more information, see <> and <>. [[put-transform-desc]] -==== {api-description-title} +== {api-description-title} This API defines a {transform}, which copies data from source indices, transforms it, and persists it into an entity-centric destination index. The @@ -64,14 +64,14 @@ IMPORTANT: You must use {kib} or this API to create a {transform}. `.data-frame-internal*` indices. [[put-transform-path-parms]] -==== {api-path-parms-title} +== {api-path-parms-title} ``:: (Required, string) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] [[put-transform-query-parms]] -==== {api-query-parms-title} +== {api-query-parms-title} `defer_validation`:: (Optional, boolean) When `true`, deferrable validations are not run. This @@ -80,7 +80,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] [role="child_attributes"] [[put-transform-request-body]] -==== {api-request-body-title} +== {api-request-body-title} `description`:: (Optional, string) Free text description of the {transform}. @@ -204,7 +204,7 @@ delays. [[put-transform-example]] -==== {api-examples-title} +== {api-examples-title} [source,console] -------------------------------------------------- diff --git a/docs/reference/transform/apis/start-transform.asciidoc b/docs/reference/transform/apis/start-transform.asciidoc index 16d47c06aba..085539c6835 100644 --- a/docs/reference/transform/apis/start-transform.asciidoc +++ b/docs/reference/transform/apis/start-transform.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[start-transform]] -=== Start {transform} API += Start {transform} API [subs="attributes"] ++++ @@ -11,12 +11,12 @@ Starts one or more {transforms}. [[start-transform-request]] -==== {api-request-title} +== {api-request-title} `POST _transform//_start` [[start-transform-prereqs]] -==== {api-prereq-title} +== {api-prereq-title} If the {es} {security-features} are enabled, you must have the following built-in roles and privileges: @@ -28,7 +28,7 @@ For more information, see <> and <>. [[start-transform-desc]] -==== {api-description-title} +== {api-description-title} When you start a {transform}, it creates the destination index if it does not already exist. The `number_of_shards` is set to `1` and the @@ -51,14 +51,14 @@ required privileges on the source and destination indices, the {transform} fails when it attempts unauthorized operations. [[start-transform-path-parms]] -==== {api-path-parms-title} +== {api-path-parms-title} ``:: (Required, string) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] [[start-transform-example]] -==== {api-examples-title} +== {api-examples-title} [source,console] -------------------------------------------------- diff --git a/docs/reference/transform/apis/stop-transform.asciidoc b/docs/reference/transform/apis/stop-transform.asciidoc index 9ee11a87058..b02f441fced 100644 --- a/docs/reference/transform/apis/stop-transform.asciidoc +++ b/docs/reference/transform/apis/stop-transform.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[stop-transform]] -=== Stop {transforms} API += Stop {transforms} API [subs="attributes"] ++++ @@ -12,7 +12,7 @@ Stops one or more {transforms}. [[stop-transform-request]] -==== {api-request-title} +== {api-request-title} `POST _transform//_stop` + @@ -22,7 +22,7 @@ Stops one or more {transforms}. [[stop-transform-prereq]] -==== {api-prereq-title} +== {api-prereq-title} If the {es} {security-features} are enabled, you must have the following built-in roles and privileges: @@ -35,7 +35,7 @@ For more information, see <> and <>. [[stop-transform-desc]] -==== {api-description-title} +== {api-description-title} You can stop multiple {transforms} in a single API request by using a comma-separated list of {transforms} or a wildcard expression. @@ -44,14 +44,14 @@ All {transforms} can be stopped by using `_all` or `*` as the [[stop-transform-path-parms]] -==== {api-path-parms-title} +== {api-path-parms-title} ``:: (Required, string) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] [[stop-transform-query-parms]] -==== {api-query-parms-title} +== {api-query-parms-title} `allow_no_match`:: (Optional, boolean) @@ -81,7 +81,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-match-transfo stops as soon as possible. Defaults to `false`. [[stop-transform-response-codes]] -==== {api-response-codes-title} +== {api-response-codes-title} `404` (Missing resources):: If `allow_no_match` is `false`, this code indicates that there are no @@ -89,7 +89,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-match-transfo [[stop-transform-example]] -==== {api-examples-title} +== {api-examples-title} [source,console] -------------------------------------------------- diff --git a/docs/reference/transform/apis/transform-apis.asciidoc b/docs/reference/transform/apis/transform-apis.asciidoc new file mode 100644 index 00000000000..b44c5f4970b --- /dev/null +++ b/docs/reference/transform/apis/transform-apis.asciidoc @@ -0,0 +1,13 @@ +[role="xpack"] +[testenv="basic"] +[[transform-apis]] += {transform-cap} APIs + +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> diff --git a/docs/reference/transform/apis/update-transform.asciidoc b/docs/reference/transform/apis/update-transform.asciidoc index 58195226063..e67a138896e 100644 --- a/docs/reference/transform/apis/update-transform.asciidoc +++ b/docs/reference/transform/apis/update-transform.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[update-transform]] -=== Update {transform} API += Update {transform} API [subs="attributes"] ++++ @@ -11,12 +11,12 @@ Updates certain properties of a {transform}. [[update-transform-request]] -==== {api-request-title} +== {api-request-title} `POST _transform//_update` [[update-transform-prereqs]] -==== {api-prereq-title} +== {api-prereq-title} If the {es} {security-features} are enabled, you must have the following built-in roles and privileges: @@ -31,7 +31,7 @@ For more information, see <> and <>. [[update-transform-desc]] -==== {api-description-title} +== {api-description-title} This API updates an existing {transform}. The list of properties that you can update is a subset of the list that you can define when you create a {transform}. @@ -58,14 +58,14 @@ give users any privileges on `.data-frame-internal*` indices. ==== [[update-transform-path-parms]] -==== {api-path-parms-title} +== {api-path-parms-title} ``:: (Required, string) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] [[update-transform-query-parms]] -==== {api-query-parms-title} +== {api-query-parms-title} `defer_validation`:: (Optional, boolean) When `true`, deferrable validations are not run. This @@ -74,7 +74,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] [role="child_attributes"] [[update-transform-request-body]] -==== {api-request-body-title} +== {api-request-body-title} `description`:: (Optional, string) Free text description of the {transform}. @@ -126,7 +126,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-transforms] + .Properties of `source` [%collapsible%open] -==== +==== `index`::: (Required, string or array) @@ -177,7 +177,7 @@ delays. //End sync [[update-transform-example]] -==== {api-examples-title} +== {api-examples-title} [source,console] -------------------------------------------------- diff --git a/docs/reference/transform/checkpoints.asciidoc b/docs/reference/transform/checkpoints.asciidoc index 84d033d6202..b11909734d7 100644 --- a/docs/reference/transform/checkpoints.asciidoc +++ b/docs/reference/transform/checkpoints.asciidoc @@ -1,6 +1,6 @@ [role="xpack"] [[transform-checkpoints]] -=== How {transform} checkpoints work += How {transform} checkpoints work ++++ How checkpoints work ++++ @@ -53,7 +53,7 @@ TIP: If the cluster experiences unsuitable performance degradation due to the [discrete] [[ml-transform-checkpoint-errors]] -==== Error handling +== Error handling Failures in {transforms} tend to be related to searching or indexing. To increase the resiliency of {transforms}, the cursor positions of diff --git a/docs/reference/transform/ecommerce-tutorial.asciidoc b/docs/reference/transform/ecommerce-tutorial.asciidoc index 16b45fde1ee..3a166b0a526 100644 --- a/docs/reference/transform/ecommerce-tutorial.asciidoc +++ b/docs/reference/transform/ecommerce-tutorial.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[ecommerce-transforms]] -=== Tutorial: Transforming the eCommerce sample data += Tutorial: Transforming the eCommerce sample data <> enable you to retrieve information from an {es} index, transform it, and store it in another index. Let's use the diff --git a/docs/reference/transform/examples.asciidoc b/docs/reference/transform/examples.asciidoc index 7af5d8bf58c..3a73f696e3a 100644 --- a/docs/reference/transform/examples.asciidoc +++ b/docs/reference/transform/examples.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[transform-examples]] -=== {transform-cap} examples += {transform-cap} examples ++++ Examples ++++ @@ -17,7 +17,7 @@ step-by-step example, see * <> [[example-best-customers]] -==== Finding your best customers +== Finding your best customers In this example, we use the eCommerce orders sample dataset to find the customers who spent the most in our hypothetical webshop. Let's transform the @@ -101,14 +101,14 @@ navigate data from a customer centric perspective. In some cases, it can even make creating visualizations much simpler. [[example-airline]] -==== Finding air carriers with the most delays +== Finding air carriers with the most delays In this example, we use the Flights sample dataset to find out which air carrier had the most delays. First, we filter the source data such that it excludes all the cancelled flights by using a query filter. Then we transform the data to contain the distinct number of flights, the sum of delayed minutes, and the sum of the flight minutes by air carrier. Finally, we use a -{ref}/search-aggregations-pipeline-bucket-script-aggregation.html[`bucket_script`] +<> to determine what percentage of the flight time was actually delay. [source,console] @@ -188,7 +188,7 @@ or flight stats for any of the featured destination or origin airports. [[example-clientips]] -==== Finding suspicious client IPs +== Finding suspicious client IPs In this example, we use the web log sample dataset to identify suspicious client IPs. We transform the data such that the new index contains the sum of bytes and diff --git a/docs/reference/transform/index.asciidoc b/docs/reference/transform/index.asciidoc index 84d3afd9e9c..d3c162c972e 100644 --- a/docs/reference/transform/index.asciidoc +++ b/docs/reference/transform/index.asciidoc @@ -1,32 +1,11 @@ -[role="xpack"] -[[transforms]] -== Transforming data - -// tag::transform-intro[] -{transforms-cap} enable you to convert existing {es} indices into summarized -indices, which provide opportunities for new insights and analytics. -// end::transform-intro[] -For example, you can use {transforms} to pivot your data into entity-centric -indices that summarize the behavior of users or sessions or other entities in -your data. - -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> - -include::overview.asciidoc[] -include::setup.asciidoc[] -include::usage.asciidoc[] -include::checkpoints.asciidoc[] -include::api-quickref.asciidoc[] -include::ecommerce-tutorial.asciidoc[] -include::examples.asciidoc[] -include::painless-examples.asciidoc[] -include::troubleshooting.asciidoc[] -include::limitations.asciidoc[] \ No newline at end of file +include::transforms.asciidoc[leveloffset=+1] +include::overview.asciidoc[leveloffset=+2] +include::setup.asciidoc[leveloffset=+2] +include::usage.asciidoc[leveloffset=+2] +include::checkpoints.asciidoc[leveloffset=+2] +include::api-quickref.asciidoc[leveloffset=+2] +include::ecommerce-tutorial.asciidoc[leveloffset=+2] +include::examples.asciidoc[leveloffset=+2] +include::painless-examples.asciidoc[leveloffset=+2] +include::troubleshooting.asciidoc[leveloffset=+2] +include::limitations.asciidoc[leveloffset=+2] \ No newline at end of file diff --git a/docs/reference/transform/limitations.asciidoc b/docs/reference/transform/limitations.asciidoc index 88430cf37db..6c12aea11e6 100644 --- a/docs/reference/transform/limitations.asciidoc +++ b/docs/reference/transform/limitations.asciidoc @@ -1,6 +1,6 @@ [role="xpack"] [[transform-limitations]] -=== {transform-cap} limitations += {transform-cap} limitations [subs="attributes"] ++++ Limitations @@ -9,77 +9,73 @@ The following limitations and known problems apply to the {version} release of the Elastic {transform} feature: - -[float] +[discrete] [[transform-ui-limitation]] -==== {transforms-cap} UI will not work during a rolling upgrade from 7.2 +== {transforms-cap} UI will not work during a rolling upgrade from 7.2 If your cluster contains mixed version nodes, for example during a rolling upgrade from 7.2 to a newer version, and {transforms} have been created in 7.2, the {transforms} UI (earler {dataframe} UI) will not work. Please wait until all nodes have been upgraded to the newer version before using the {transforms} UI. -[float] +[discrete] [[transform-rolling-upgrade-limitation]] -==== {transforms-cap} reassignment suspended during a rolling upgrade from 7.2 and 7.3 +== {transforms-cap} reassignment suspended during a rolling upgrade from 7.2 and 7.3 If your cluster contains mixed version nodes, for example during a rolling upgrade from 7.2 or 7.3 to a newer version, {transforms} whose nodes are stopped will not be reassigned until the upgrade is complete. After the upgrade is done, {transforms} resume automatically; no action is required. -[float] +[discrete] [[transform-datatype-limitations]] -==== {dataframe-cap} data type limitation +== {dataframe-cap} data type limitation {dataframes-cap} do not (yet) support fields containing arrays – in the UI or the API. If you try to create one, the UI will fail to show the source index table. - -[float] +[discrete] [[transform-kibana-limitations]] -==== Up to 1,000 {transforms} are supported +== Up to 1,000 {transforms} are supported A single cluster will support up to 1,000 {transforms}. When using the -{ref}/get-transform.html[GET {transforms} API] a total `count` of {transforms} +<> a total `count` of {transforms} is returned. Use the `size` and `from` parameters to enumerate through the full list. -[float] +[discrete] [[transform-aggresponse-limitations]] -==== Aggregation responses may be incompatible with destination index mappings +== Aggregation responses may be incompatible with destination index mappings When a {transform} is first started, it will deduce the mappings required for the destination index. This process is based on the field types of the source index and the aggregations used. If the fields are derived from -{ref}/search-aggregations-metrics-scripted-metric-aggregation.html[`scripted_metrics`] -or {ref}/search-aggregations-pipeline-bucket-script-aggregation.html[`bucket_scripts`], -{ref}/dynamic-mapping.html[dynamic mappings] will be used. In some instances the +<> +or <>, +<> will be used. In some instances the deduced mappings may be incompatible with the actual data. For example, numeric overflows might occur or dynamically mapped fields might contain both numbers and strings. Please check {es} logs if you think this may have occurred. As a workaround, you may define custom mappings prior to starting the {transform}. For example, -{ref}/indices-create-index.html[create a custom destination index] or -{ref}/indices-templates.html[define an index template]. +<> or +<>. - -[float] +[discrete] [[transform-batch-limitations]] -==== Batch {transforms} may not account for changed documents +== Batch {transforms} may not account for changed documents A batch {transform} uses a -{ref}/search-aggregations-bucket-composite-aggregation.html[composite aggregation] +<> which allows efficient pagination through all buckets. Composite aggregations do not yet support a search context, therefore if the source data is changed (deleted, updated, added) while the batch {dataframe} is in progress, then the results may not include these changes. - -[float] +[discrete] [[transform-consistency-limitations]] -==== {ctransform-cap} consistency does not account for deleted or updated documents +== {ctransform-cap} consistency does not account for deleted or updated documents While the process for {transforms} allows the continual recalculation of the {transform} as new data is being ingested, it does also have some limitations. @@ -101,19 +97,17 @@ archiving, you may wish to include a max ingest timestamp in your aggregation. This will allow you to exclude results that have not been recently updated when viewing the destination index. - -[float] +[discrete] [[transform-deletion-limitations]] -==== Deleting a {transform} does not delete the destination index or {kib} index pattern +== Deleting a {transform} does not delete the destination index or {kib} index pattern When deleting a {transform} using `DELETE _transform/index` neither the destination index nor the {kib} index pattern, should one have been created, are deleted. These objects must be deleted separately. - -[float] +[discrete] [[transform-aggregation-page-limitations]] -==== Handling dynamic adjustment of aggregation page size +== Handling dynamic adjustment of aggregation page size During the development of {transforms}, control was favoured over performance. In the design considerations, it is preferred for the {transform} to take longer @@ -121,9 +115,9 @@ to complete quietly in the background rather than to finish quickly and take precedence in resource consumption. Composite aggregations are well suited for high cardinality data enabling -pagination through results. If a {ref}/circuit-breaker.html[circuit breaker] -memory exception occurs when performing the composite aggregated search then we -try again reducing the number of buckets requested. This circuit breaker is +pagination through results. If a <> memory +exception occurs when performing the composite aggregated search then we try +again reducing the number of buckets requested. This circuit breaker is calculated based upon all activity within the cluster, not just activity from {transforms}, so it therefore may only be a temporary resource availability issue. @@ -140,23 +134,21 @@ default can be changed using `max_page_search_size` and the minimum value is 10. If failures still occur once the number of buckets requested has been reduced to its minimum, then the {transform} will be set to a failed state. - -[float] +[discrete] [[transform-dynamic-adjustments-limitations]] -==== Handling dynamic adjustments for many terms +== Handling dynamic adjustments for many terms For each checkpoint, entities are identified that have changed since the last time the check was performed. This list of changed entities is supplied as a -{ref}/query-dsl-terms-query.html[terms query] to the {transform} composite -aggregation, one page at a time. Then updates are applied to the destination -index for each page of entities. +<> to the {transform} composite aggregation, +one page at a time. Then updates are applied to the destination index for each +page of entities. The page `size` is defined by `max_page_search_size` which is also used to define the number of buckets returned by the composite aggregation search. The default value is 500, the minimum is 10. -The index setting -{ref}/index-modules.html#dynamic-index-settings[`index.max_terms_count`] defines +The index setting <> defines the maximum number of terms that can be used in a terms query. The default value is 65536. If `max_page_search_size` exceeds `index.max_terms_count` the {transform} will fail. @@ -164,10 +156,9 @@ is 65536. If `max_page_search_size` exceeds `index.max_terms_count` the Using smaller values for `max_page_search_size` may result in a longer duration for the {transform} checkpoint to complete. - -[float] +[discrete] [[transform-scheduling-limitations]] -==== {ctransform-cap} scheduling limitations +== {ctransform-cap} scheduling limitations A {ctransform} periodically checks for changes to source data. The functionality of the scheduler is currently limited to a basic periodic timer which can be @@ -177,10 +168,9 @@ your ingest rate along with the impact that the {transform} search/index operations has other users in your cluster. Also note that retries occur at `frequency` interval. - -[float] +[discrete] [[transform-failed-limitations]] -==== Handling of failed {transforms} +== Handling of failed {transforms} Failed {transforms} remain as a persistent task and should be handled appropriately, either by deleting it or by resolving the root cause of the @@ -189,10 +179,9 @@ failure and re-starting. When using the API to delete a failed {transform}, first stop it using `_stop?force=true`, then delete it. - -[float] +[discrete] [[transform-availability-limitations]] -==== {ctransforms-cap} may give incorrect results if documents are not yet available to search +== {ctransforms-cap} may give incorrect results if documents are not yet available to search After a document is indexed, there is a very small delay until it is available to search. @@ -207,8 +196,9 @@ If using a `sync.time.field` that represents the data ingest time and using a zero second or very small `sync.time.delay`, then it is more likely that this issue will occur. +[discrete] [[transform-date-nanos]] -==== Support for date nanoseconds data type +== Support for date nanoseconds data type If your data uses the <>, aggregations are nonetheless on millisecond resolution. This limitation also affects the diff --git a/docs/reference/transform/overview.asciidoc b/docs/reference/transform/overview.asciidoc index 58eba19c8a1..bb00a394167 100644 --- a/docs/reference/transform/overview.asciidoc +++ b/docs/reference/transform/overview.asciidoc @@ -1,6 +1,6 @@ [role="xpack"] [[transform-overview]] -=== {transform-cap} overview += {transform-cap} overview ++++ Overview ++++ @@ -60,9 +60,9 @@ image::images/pivot-preview.jpg["Example of a {transform} pivot in {kib}"] IMPORTANT: The {transform} leaves your source index intact. It creates a new index that is dedicated to the transformed data. - +[discrete] [[transform-performance]] -==== Performance considerations +== Performance considerations {transforms-cap} perform search aggregations on the source indices then index the results into the destination index. Therefore, a {transform} never takes diff --git a/docs/reference/transform/painless-examples.asciidoc b/docs/reference/transform/painless-examples.asciidoc index c2c24614ef1..db3550422d1 100644 --- a/docs/reference/transform/painless-examples.asciidoc +++ b/docs/reference/transform/painless-examples.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[transform-painless-examples]] -=== Painless examples for {transforms} += Painless examples for {transforms} ++++ Painless examples for {transforms} ++++ @@ -22,10 +22,8 @@ NOTE: While the context of the following examples is the {transform} use case, the Painless scripts in the snippets below can be used in other {es} search aggregations, too. - -[discrete] [[painless-top-hits]] -==== Getting top hits by using scripted metric aggregation +== Getting top hits by using scripted metric aggregation This snippet shows how to find the latest document, in other words the document with the earliest timestamp. From a technical perspective, it helps to achieve @@ -103,10 +101,8 @@ You can retrieve the last value in a similar way: -------------------------------------------------- // NOTCONSOLE - -[discrete] [[painless-time-features]] -==== Getting time features by using aggregations +== Getting time features by using aggregations This snippet shows how to extract time based features by using Painless in a {transform}. The snippet uses an index where `@timestamp` is defined as a `date` @@ -149,10 +145,8 @@ type field. <7> Sets `date` based on the timestamp of the document. <8> Returns the month value from `date`. - -[discrete] [[painless-group-by]] -==== Using Painless in `group_by` +== Using Painless in `group_by` It is possible to base the `group_by` property of a {transform} on the output of a script. The following example uses the {kib} sample web logs dataset. The goal @@ -287,14 +281,12 @@ them. The table below shows how normalization modifies the output of the | "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1" | "firefox" |=== - -[discrete] [[painless-bucket-script]] -==== Getting duration by using bucket script +== Getting duration by using bucket script This example shows you how to get the duration of a session by client IP from a data log by using -{ref}/search-aggregations-pipeline-bucket-script-aggregation.html[bucket script]. +<>. The example uses the {kib} sample web logs dataset. [source,console] @@ -345,10 +337,8 @@ the buckets you want to use for the variable. In this particular case, `min` and <3> Finally, the script substracts the start date of the session from the end date which results in the duration of the session. - -[discrete] [[painless-count-http]] -==== Counting HTTP responses by using scripted metric aggregation +== Counting HTTP responses by using scripted metric aggregation You can count the different HTTP response types in a web log data set by using scripted metric aggregation as part of the {transform}. The example below @@ -405,10 +395,8 @@ properties of the `counts` object; error responses to the error counts, success responses to the success counts, and other responses to the other counts. Finally, returns the `counts` array with the response counts. - -[discrete] [[painless-compare]] -==== Comparing indices by using scripted metric aggregations +== Comparing indices by using scripted metric aggregations This example shows how to compare the content of two indices by a {transform} that uses a scripted metric aggregation. @@ -475,9 +463,8 @@ not equal, than it reports back a `count_mismatch`. Then it iterates through all the values of the two indices and compare them. If the values are equal, then it returns a `match`, otherwise returns a `mismatch`. -[discrete] [[painless-web-session]] -==== Getting web session details by using scripted metric aggregation +== Getting web session details by using scripted metric aggregation This example shows how to derive multiple features from a single transaction. Let's take a look on the example source document from the data: diff --git a/docs/reference/transform/setup.asciidoc b/docs/reference/transform/setup.asciidoc index dcd7b96626c..1d54af93082 100644 --- a/docs/reference/transform/setup.asciidoc +++ b/docs/reference/transform/setup.asciidoc @@ -1,6 +1,6 @@ [role="xpack"] [[transform-setup]] -=== Set up {transforms} += Set up {transforms} ++++ Setup ++++ @@ -13,7 +13,7 @@ To use the {transforms}, you must have the [discrete] [[transform-setup-nodes]] -==== {transform-cap} nodes +== {transform-cap} nodes To use {transforms}, there must be at least one {transform} node in your cluster. If you want to control which nodes run {transforms}, add or remove `transform` @@ -22,7 +22,7 @@ from the `node.roles` setting on some nodes. For more information, see [discrete] [[transform-privileges]] -==== Security privileges +== Security privileges The {es} {security-features} provide <> and <> that make it easier to control diff --git a/docs/reference/transform/transforms.asciidoc b/docs/reference/transform/transforms.asciidoc new file mode 100644 index 00000000000..46cdf67b24e --- /dev/null +++ b/docs/reference/transform/transforms.asciidoc @@ -0,0 +1,21 @@ +[role="xpack"] +[[transforms]] += Transforming data + +// tag::transform-intro[] +{transforms-cap} enable you to convert existing {es} indices into summarized +indices, which provide opportunities for new insights and analytics. +// end::transform-intro[] +For example, you can use {transforms} to pivot your data into entity-centric +indices that summarize the behavior of users or sessions or other entities in +your data. + +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> diff --git a/docs/reference/transform/troubleshooting.asciidoc b/docs/reference/transform/troubleshooting.asciidoc index 2198687d29f..01fdb8f6129 100644 --- a/docs/reference/transform/troubleshooting.asciidoc +++ b/docs/reference/transform/troubleshooting.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[transform-troubleshooting]] -=== Troubleshooting {transforms} += Troubleshooting {transforms} [subs="attributes"] ++++ Troubleshooting @@ -20,16 +20,15 @@ information from the following files and APIs: * Lightweight audit messages are stored in `.transform-notifications-read`. Search by your `transform_id`. -* The -{ref}/get-transform-stats.html[get {transform} statistics API] -provides information about the {transform} status and failures. +* The <> provides +information about the {transform} status and failures. * If the {transform} exists as a task, you can use the -{ref}/tasks.html[task management API] to gather task information. For example: +<> to gather task information. For example: `GET _tasks?actions=data_frame/transforms*&detailed`. Typically, the task exists when the {transform} is in a started or failed state. * The {es} logs from the node that was running the {transform} might also contain useful information. You can identify the node from the notification messages. Alternatively, if the task still exists, you can get that information from the get {transform} statistics API. For more information, see -{ref}/logging.html[Logging configuration]. +<>. diff --git a/docs/reference/transform/usage.asciidoc b/docs/reference/transform/usage.asciidoc index f78a0388bcb..fdf6450f4d6 100644 --- a/docs/reference/transform/usage.asciidoc +++ b/docs/reference/transform/usage.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[transform-usage]] -=== When to use {transforms} += When to use {transforms} {es} aggregations are a powerful and flexible feature that enable you to summarize and retrieve complex insights about your data. You can summarize