From d66084dcaf800bc7619ed8594db28c744491abf1 Mon Sep 17 00:00:00 2001 From: James Rodewig Date: Mon, 6 Jul 2020 11:58:43 -0400 Subject: [PATCH] [DOCS] Update data stream mapping and setting docs (#58874) (#59067) --- .../change-mappings-and-settings.asciidoc | 453 ++++++++++++------ 1 file changed, 304 insertions(+), 149 deletions(-) diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc index d27a38f2fc1..1c35d5603cf 100644 --- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc +++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc @@ -1,8 +1,5 @@ [[data-streams-change-mappings-and-settings]] -== Change mappings and settings of backing indices -++++ -Change mappings and settings -++++ +== Change mappings and settings for a data stream Each data stream has a <>. Mappings and index settings from this template are applied to new @@ -10,26 +7,22 @@ backing indices created for the stream. This includes the stream's first backing index, which is auto-generated when the stream is created. Before creating a data stream, we recommend you carefully consider which -mappings and settings to include in this template. However, if you later need to -change the mappings or settings of a data stream's backing indices, you have a -couple options: +mappings and settings to include in this template. -* To apply changes to future backing indices, simply update the index -template used by the data stream. Mapping and setting changes will be -automatically applied to any backing indices created after the update. -+ -.*Example* -[%collapsible] -==== -`logs_data_stream` is an existing index template used by the -`logs` data stream. +If you later need to change the mappings or settings for a data stream, you have +a few options: -The following <> makes several -changes to the `logs_data_stream` template: +* <> +* <> +* <> +* <> -* It changes the `@timestamp` field mapping from the `date` field datatype to - the `date_nanos` datatype. -* It adds new `sort.field` and `sort.order` index settings. +TIP: If your changes include modifications to existing field mappings or +<>, a reindex is often required to +apply the changes to a data stream's backing indices. If you are already +performing a reindex, you can use the same process to add new field +mappings and change <>. See +<>. //// [source,console] @@ -68,23 +61,61 @@ PUT /_index_template/logs_data_stream "type": "date" } } - }, - "settings": { - "index.lifecycle.name": "logs_policy" } } } -PUT /logs/_bulk?refresh -{"create":{"_index" : "logs"}} -{ "@timestamp": "2020-12-08T11:04:05.000Z" } -{"create":{"_index" : "logs"}} -{ "@timestamp": "2020-12-08T11:06:07.000Z" } -{"create":{"_index" : "logs"}} -{ "@timestamp": "2020-12-09T11:07:08.000Z" } +PUT /_index_template/new_logs_data_stream +{ + "index_patterns": [ "new_logs*" ], + "data_stream": { + "timestamp_field": "@timestamp" + }, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + } + } + } + } +} + +PUT /_data_stream/logs + +PUT /_data_stream/new_logs ---- +// TESTSETUP + +[source,console] +---- +DELETE /_data_stream/* + +DELETE /_index_template/* + +DELETE /_ilm/policy/logs_policy +---- +// TEARDOWN //// +[discrete] +[[add-new-field-mapping-to-a-data-stream]] +=== Add a new field mapping to a data stream + +To add a mapping for a new field to a data stream, following these steps: + +. Update the index template used by the data stream. This ensures the new +field mapping is added to future backing indices created for the stream. ++ +.*Example* +[%collapsible] +==== +`logs_data_stream` is an existing index template used by the `logs` data stream. + +The following <> request adds a mapping +for a new field, `message`, to the template. + [source,console] ---- PUT /_index_template/logs_data_stream @@ -97,62 +128,265 @@ PUT /_index_template/logs_data_stream "mappings": { "properties": { "@timestamp": { - "type": "date_nanos" <1> + "type": "date" + }, + "message": { <1> + "type": "text" } } - }, - "settings": { - "index.lifecycle.name": "logs_policy", - "sort.field" : [ "@timestamp"], <2> - "sort.order" : [ "desc"] <3> } } } ---- -// TEST[continued] - -<1> Changes the `@timestamp` field mapping to the `date_nanos` datatype. -<2> Adds the `sort.field` index setting. -<3> Adds the `sort.order` index setting. +<1> Adds a mapping for the new `message` field. ==== -+ -If wanted, you can <> to immediately apply the new mappings and settings to the data stream's -write index. This affects any new data added to the stream after the rollover. + +. Use the <> to add the new field mapping +to the data stream. By default, this adds the mapping to the stream's existing +backing indices, including the write index. + .*Example* [%collapsible] ==== -The following <> request rolls over the -`logs` data stream. This creates a new write index with mappings and index -settings from the recently updated `logs_data_stream` template. +The following put mapping API request adds the new `message` field mapping to +the `logs` data stream. [source,console] ---- -POST /logs/_rollover/ +PUT /logs/_mapping +{ + "properties": { + "message": { + "type": "text" + } + } +} ---- -// TEST[continued] ==== + +[discrete] +[[change-existing-field-mapping-in-a-data-stream]] +=== Change an existing field mapping in a data stream + +The documentation for each <> indicates +whether you can update it for an existing field using the +<>. To update these parameters for an +existing field, follow these steps: + +. Update the index template used by the data stream. This ensures the updated +field mapping is added to future backing indices created for the stream. + -IMPORTANT: You cannot use these methods to change the mapping of a data stream's -<>. To change the timestamp -field's mapping, you must reindex the data stream. See +.*Example* +[%collapsible] +==== +`logs_data_stream` is an existing index template used by the `logs` data stream. + +The following <> request changes the +argument for the `host.ip` field's <> +mapping parameter to `true`. + +[source,console] +---- +PUT /_index_template/logs_data_stream +{ + "index_patterns": [ "logs*" ], + "data_stream": { + "timestamp_field": "@timestamp" + }, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "properties": { + "ip": { + "type": "ip", + "ignore_malformed": true <1> + } + } + } + } + } + } +} +---- +<1> Changes the `host.ip` field's `ignore_malformed` value to `true`. +==== + +. Use the <> to apply the mapping changes +to the data stream. By default, this applies the changes to the stream's +existing backing indices, including the write index. ++ +.*Example* +[%collapsible] +==== +The following <> request targets the `logs` +data stream. The request changes the argument for the `host.ip` field's +`ignore_malformed` mapping parameter to `true`. + +[source,console] +---- +PUT /logs/_mapping +{ + "properties": { + "host": { + "properties": { + "ip": { + "type": "ip", + "ignore_malformed": true + } + } + } + } +} +---- +==== + +Except for supported mapping parameters, we don't recommend you change the +mapping or field datatype of existing fields, even in a data stream's matching +index template or its backing indices. Changing the mapping of an existing +field could invalidate any data that’s already indexed. + +If you need to change the mapping of an existing field, create a new +data stream and reindex your data into it. See <>. -* To apply mapping and setting changes to all existing backing indices and -future ones, you must create a new data stream and reindex your data into it. -See <>. +[discrete] +[[change-dynamic-index-setting-for-a-data-stream]] +=== Change a dynamic index setting for a data stream + +To change a <> for a data stream, +follow these steps: + +. Update the index template used by the data stream. This ensures the setting is +applied to future backing indices created for the stream. ++ +.*Example* +[%collapsible] +==== +`logs_data_stream` is an existing index template used by the `logs` data stream. + +The following <> request changes the +template's `index.refresh_interval` index setting to `30s` (30 seconds). + +[source,console] +---- +PUT /_index_template/logs_data_stream +{ + "index_patterns": [ "logs*" ], + "data_stream": { + "timestamp_field": "@timestamp" + }, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + } + } + }, + "settings": { + "index.refresh_interval": "30s" <1> + } + } +} +---- +<1> Changes the `index.refresh_interval` setting to `30s` (30 seconds). +==== + +. Use the <> to update the +index setting for the data stream. By default, this applies the setting to +the stream's existing backing indices, including the write index. ++ +.*Example* +[%collapsible] +==== +The following update index settings API request updates the +`index.refresh_interval` setting for the `logs` data stream. + +[source,console] +---- +PUT /logs/_settings +{ + "index": { + "refresh_interval": "30s" + } +} +---- +==== + +[discrete] +[[change-static-index-setting-for-a-data-stream]] +=== Change a static index setting for a data stream + +<> can only be set when a backing +index is created. You cannot update static index settings using the +<>. + +To apply a new static setting to future backing indices, update the index +template used by the data stream. The setting is automatically applied to any +backing index created after the update. + +.*Example* +[%collapsible] +==== +`logs_data_stream` is an existing index template used by the `logs` data stream. + +The following <> requests adds new +`sort.field` and `sort.order index` settings to the template. + +[source,console] +---- +PUT /_index_template/logs_data_stream +{ + "index_patterns": [ "logs*" ], + "data_stream": { + "timestamp_field": "@timestamp" + }, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + } + } + }, + "settings": { + "sort.field": [ "@timestamp"], <1> + "sort.order": [ "desc"] <2> + } + } +} +---- +<1> Adds the `sort.field` index setting. +<2> Adds the `sort.order` index setting. +==== + +If wanted, you can <> to immediately apply the setting to the data stream’s write index. This +affects any new data added to the stream after the rollover. However, it does +not affect the data stream's existing backing indices or existing data. + +To apply static setting changes to existing backing indices, you must create a +new data stream and reindex your data into it. See +<>. [discrete] [[data-streams-use-reindex-to-change-mappings-settings]] === Use reindex to change mappings or settings -To change the mappings or settings for every backing index in a data stream, you -must first create or update an index template so that it contains the -changes. You can then reindex the existing data stream into a new one matching -the template. This applies the mapping and setting changes in the template -to each document and backing index of the data stream destination. These changes -also affect any future backing index created by the new stream. +You can use a reindex to change the mappings or settings of a data stream. This +is often required to change the datatype of an existing field or update static +index settings for backing indices. + +To reindex a data stream, first create or update an index template so that it +contains the wanted mapping or setting changes. You can then reindex the +existing data stream into a new stream matching the template. This applies the +mapping and setting changes in the template to each document and backing index +added to the new data stream. These changes also affect any future backing +index created by the new stream. Follow these steps: @@ -174,7 +408,6 @@ wildcard pattern can be used to create a new data stream. ---- GET /_resolve/index/new_logs* ---- -// TEST[continued] The API returns the following response, indicating no existing targets match this pattern. @@ -182,11 +415,12 @@ this pattern. [source,console-result] ---- { - "indices" : [ ], - "aliases" : [ ], - "data_streams" : [ ] + "indices": [ ], + "aliases": [ ], + "data_streams": [ ] } ---- +// TESTRESPONSE[s/"data_streams": \[ \]/"data_streams": $body.data_streams/] ==== . Create or update an index template. This template should contain the @@ -235,15 +469,12 @@ PUT /_index_template/new_logs_data_stream } }, "settings": { - "index.lifecycle.name": "logs_policy", - "sort.field" : [ "@timestamp"], <2> - "sort.order" : [ "desc"] <3> + "sort.field": [ "@timestamp"], <2> + "sort.order": [ "desc"] <3> } } } ---- -// TEST[continued] - <1> Changes the `@timestamp` field mapping to the `date_nanos` field datatype. <2> Adds the `sort.field` index setting. <3> Adds the `sort.order` index setting. @@ -280,7 +511,7 @@ existing index or data stream uses this name, this request creates the ---- PUT /_data_stream/new_logs ---- -// TEST[continued] +// TEST[s/new_logs/new_logs_two/] ==== . If you do not want to mix new and old data in your new data stream, pause the @@ -308,23 +539,6 @@ PUT /_cluster/settings } } ---- -// TEST[continued] - -//// -[source,console] ----- -DELETE /_data_stream/logs - -DELETE /_data_stream/new_logs - -DELETE /_index_template/logs_data_stream - -DELETE /_index_template/new_logs_data_stream - -DELETE /_ilm/policy/logs_policy ----- -// TEST[continued] -//// ==== . Reindex your data to the new data stream using an `op_type` of `create`. @@ -383,49 +597,6 @@ The following <> request copies documents from `.ds-logs-000001` to the `new_logs` data stream. Note the request's `op_type` is `create`. -//// -[source,console] ----- -PUT /_index_template/logs_data_stream -{ - "index_patterns": [ "logs*" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, - "template": { - "mappings": { - "properties": { - "@timestamp": { - "type": "date" - } - } - } - } -} - -PUT /_index_template/new_logs_data_stream -{ - "index_patterns": [ "new_logs*" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, - "template": { - "mappings": { - "properties": { - "@timestamp": { - "type": "date" - } - } - } - } -} - -PUT /_data_stream/logs - -PUT /_data_stream/new_logs ----- -//// - [source,console] ---- POST /_reindex @@ -439,7 +610,6 @@ POST /_reindex } } ---- -// TEST[continued] ==== + You can also use a query to reindex only a subset of documents with each @@ -474,7 +644,6 @@ POST /_reindex } } ---- -// TEST[continued] ==== . If you previously changed your {ilm-init} poll interval, change it back to its @@ -496,7 +665,6 @@ PUT /_cluster/settings } } ---- -// TEST[continued] ==== . Resume indexing using the new data stream. Searches on this stream will now @@ -516,17 +684,4 @@ indices and any data they contain. ---- DELETE /_data_stream/logs ---- -// TEST[continued] ==== - -//// -[source,console] ----- -DELETE /_data_stream/new_logs - -DELETE /_index_template/logs_data_stream - -DELETE /_index_template/new_logs_data_stream ----- -// TEST[continued] -////