diff --git a/docs/reference/analysis/tokenizers/keyword-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/keyword-tokenizer.asciidoc index ebbb42baa5d..57f3fbfd9f9 100644 --- a/docs/reference/analysis/tokenizers/keyword-tokenizer.asciidoc +++ b/docs/reference/analysis/tokenizers/keyword-tokenizer.asciidoc @@ -44,6 +44,54 @@ The above sentence would produce the following term: [ New York ] --------------------------- +[discrete] +[[analysis-keyword-tokenizer-token-filters]] +=== Combine with token filters +You can combine the `keyword` tokenizer with token filters to normalise +structured data, such as product IDs or email addresses. + +For example, the following <> request uses the +`keyword` tokenizer and <> filter to +convert an email address to lowercase. + +[source,console] +--------------------------- +POST _analyze +{ + "tokenizer": "keyword", + "filter": [ "lowercase" ], + "text": "john.SMITH@example.COM" +} +--------------------------- + +///////////////////// + +[source,console-result] +---------------------------- +{ + "tokens": [ + { + "token": "john.smith@example.com", + "start_offset": 0, + "end_offset": 22, + "type": "word", + "position": 0 + } + ] +} +---------------------------- + +///////////////////// + + +The request produces the following token: + +[source,text] +--------------------------- +[ john.smith@example.com ] +--------------------------- + + [float] === Configuration