From c80f86e3e4d94dc4220a7b268514282bc5acbef2 Mon Sep 17 00:00:00 2001 From: clement-tourriere Date: Fri, 19 Apr 2019 22:17:00 +0200 Subject: [PATCH] Add ignore_above in ICUCollationKeywordFieldMapper (#40414) Add the possibility to use ignore_above parameter in ICUCollationKeywordFieldMapper. Close #40413 --- docs/plugins/analysis-icu.asciidoc | 8 ++++ .../ICUCollationKeywordFieldMapper.java | 29 ++++++++++-- .../ICUCollationKeywordFieldMapperTests.java | 47 +++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/docs/plugins/analysis-icu.asciidoc b/docs/plugins/analysis-icu.asciidoc index 51be1907c98..9dc88967485 100644 --- a/docs/plugins/analysis-icu.asciidoc +++ b/docs/plugins/analysis-icu.asciidoc @@ -413,6 +413,14 @@ The following parameters are accepted by `icu_collation_keyword` fields: Accepts a string value which is substituted for any explicit `null` values. Defaults to `null`, which means the field is treated as missing. +<>:: + + Strings longer than the `ignore_above` setting will be ignored. + Checking is performed on the original string before the collation. + The `ignore_above` setting can be updated on existing fields + using the {ref}/indices-put-mapping.html[PUT mapping API]. + By default, there is no limit and all values will be indexed. + `store`:: Whether the field value should be stored and retrievable separately from diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index a228283527d..4b29d314356 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -70,6 +70,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { } public static final String NULL_VALUE = null; + public static final int IGNORE_ABOVE = Integer.MAX_VALUE; } public static final class CollationFieldType extends StringFieldType { @@ -226,6 +227,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { private boolean numeric = false; private String variableTop = null; private boolean hiraganaQuaternaryMode = false; + protected int ignoreAbove = Defaults.IGNORE_ABOVE; public Builder(String name) { super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE); @@ -247,6 +249,14 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { return super.indexOptions(indexOptions); } + public Builder ignoreAbove(int ignoreAbove) { + if (ignoreAbove < 0) { + throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove); + } + this.ignoreAbove = ignoreAbove; + return this; + } + public String rules() { return rules; } @@ -458,7 +468,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { setupFieldType(context); return new ICUCollationKeywordFieldMapper(name, fieldType, defaultFieldType, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo, rules, language, country, variant, strength, decomposition, - alternate, caseLevel, caseFirst, numeric, variableTop, hiraganaQuaternaryMode, collator); + alternate, caseLevel, caseFirst, numeric, variableTop, hiraganaQuaternaryMode, ignoreAbove, collator); } } @@ -480,6 +490,10 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { builder.nullValue(fieldNode.toString()); iterator.remove(); break; + case "ignore_above": + builder.ignoreAbove(XContentMapValues.nodeIntegerValue(fieldNode, -1)); + iterator.remove(); + break; case "norms": builder.omitNorms(!XContentMapValues.nodeBooleanValue(fieldNode, "norms")); iterator.remove(); @@ -553,13 +567,15 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { private final boolean numeric; private final String variableTop; private final boolean hiraganaQuaternaryMode; + private int ignoreAbove; private final Collator collator; protected ICUCollationKeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings, MultiFields multiFields, CopyTo copyTo, String rules, String language, String country, String variant, String strength, String decomposition, String alternate, boolean caseLevel, String caseFirst, - boolean numeric, String variableTop, boolean hiraganaQuaternaryMode, Collator collator) { + boolean numeric, String variableTop, boolean hiraganaQuaternaryMode, + int ignoreAbove, Collator collator) { super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); assert collator.isFrozen(); this.rules = rules; @@ -574,6 +590,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { this.numeric = numeric; this.variableTop = variableTop; this.hiraganaQuaternaryMode = hiraganaQuaternaryMode; + this.ignoreAbove = ignoreAbove; this.collator = collator; } @@ -642,6 +659,8 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { conflicts.add("Cannot update hiragana_quaternary_mode setting for [" + CONTENT_TYPE + "]"); } + this.ignoreAbove = icuMergeWith.ignoreAbove; + if (!conflicts.isEmpty()) { throw new IllegalArgumentException("Can't merge because of conflicts: " + conflicts); } @@ -702,6 +721,10 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { if (includeDefaults || hiraganaQuaternaryMode) { builder.field("hiragana_quaternary_mode", hiraganaQuaternaryMode); } + + if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) { + builder.field("ignore_above", ignoreAbove); + } } @Override @@ -718,7 +741,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { } } - if (value == null) { + if (value == null || value.length() > ignoreAbove) { return; } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index 103098d5a46..058bd7dbc89 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -403,4 +403,51 @@ public class ICUCollationKeywordFieldMapperTests extends ESSingleNodeTestCase { assertEquals("Can't merge because of conflicts: [Cannot update language setting for [" + FIELD_TYPE + "], Cannot update strength setting for [" + FIELD_TYPE + "]]", e.getMessage()); } + + + public void testIgnoreAbove() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("field").field("type", FIELD_TYPE) + .field("ignore_above", 5).endObject().endObject() + .endObject().endObject()); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + + assertEquals(mapping, mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference + .bytes(XContentFactory.jsonBuilder() + .startObject() + .field("field", "elk") + .endObject()), + XContentType.JSON)); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(2, fields.length); + + doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference + .bytes(XContentFactory.jsonBuilder() + .startObject() + .field("field", "elasticsearch") + .endObject()), + XContentType.JSON)); + + fields = doc.rootDoc().getFields("field"); + assertEquals(0, fields.length); + } + + public void testUpdateIgnoreAbove() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("field").field("type", FIELD_TYPE).endObject().endObject() + .endObject().endObject()); + + indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("field").field("type", FIELD_TYPE) + .field("ignore_above", 5).endObject().endObject() + .endObject().endObject()); + indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + } + }