honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add ignore_above in ICUCollationKeywordFieldMapper (#40414) 

Add the possibility to use ignore_above parameter in ICUCollationKeywordFieldMapper.

Close #40413
This commit is contained in:
clement-tourriere 2019-04-19 22:17:00 +02:00 committed by Julie Tibshirani
parent 38b43c4a56
commit c80f86e3e4
3 changed files with 81 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/plugins/analysis-icu.asciidoc
View File

@ -413,6 +413,14 @@ The following parameters are accepted by `icu_collation_keyword` fields:
Accepts a string value which is substituted for any explicit `null`
values. Defaults to `null`, which means the field is treated as missing.
<<ignore-above,`ignore_above`>>::
Strings longer than the `ignore_above` setting will be ignored.
Checking is performed on the original string before the collation.
The `ignore_above` setting can be updated on existing fields
using the {ref}/indices-put-mapping.html[PUT mapping API].
By default, there is no limit and all values will be indexed.
`store`::
Whether the field value should be stored and retrievable separately from

29
plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java
View File

@ -70,6 +70,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
}
public static final String NULL_VALUE = null;
public static final int IGNORE_ABOVE = Integer.MAX_VALUE;
}
public static final class CollationFieldType extends StringFieldType {
@ -226,6 +227,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
private boolean numeric = false;
private String variableTop = null;
private boolean hiraganaQuaternaryMode = false;
protected int ignoreAbove = Defaults.IGNORE_ABOVE;
public Builder(String name) {
super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
@ -247,6 +249,14 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
return super.indexOptions(indexOptions);
}
public Builder ignoreAbove(int ignoreAbove) {
if (ignoreAbove < 0) {
throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove);
}
this.ignoreAbove = ignoreAbove;
return this;
}
public String rules() {
return rules;
}
@ -458,7 +468,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
setupFieldType(context);
return new ICUCollationKeywordFieldMapper(name, fieldType, defaultFieldType, context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo, rules, language, country, variant, strength, decomposition,
alternate, caseLevel, caseFirst, numeric, variableTop, hiraganaQuaternaryMode, collator);
alternate, caseLevel, caseFirst, numeric, variableTop, hiraganaQuaternaryMode, ignoreAbove, collator);
}
}
@ -480,6 +490,10 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
builder.nullValue(fieldNode.toString());
iterator.remove();
break;
case "ignore_above":
builder.ignoreAbove(XContentMapValues.nodeIntegerValue(fieldNode, -1));
iterator.remove();
break;
case "norms":
builder.omitNorms(!XContentMapValues.nodeBooleanValue(fieldNode, "norms"));
iterator.remove();
@ -553,13 +567,15 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
private final boolean numeric;
private final String variableTop;
private final boolean hiraganaQuaternaryMode;
private int ignoreAbove;
private final Collator collator;
protected ICUCollationKeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Settings indexSettings, MultiFields multiFields, CopyTo copyTo, String rules, String language,
String country, String variant,
String strength, String decomposition, String alternate, boolean caseLevel, String caseFirst,
boolean numeric, String variableTop, boolean hiraganaQuaternaryMode, Collator collator) {
boolean numeric, String variableTop, boolean hiraganaQuaternaryMode,
int ignoreAbove, Collator collator) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
assert collator.isFrozen();
this.rules = rules;
@ -574,6 +590,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
this.numeric = numeric;
this.variableTop = variableTop;
this.hiraganaQuaternaryMode = hiraganaQuaternaryMode;
this.ignoreAbove = ignoreAbove;
this.collator = collator;
}
@ -642,6 +659,8 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
conflicts.add("Cannot update hiragana_quaternary_mode setting for [" + CONTENT_TYPE + "]");
}
this.ignoreAbove = icuMergeWith.ignoreAbove;
if (!conflicts.isEmpty()) {
throw new IllegalArgumentException("Can't merge because of conflicts: " + conflicts);
}
@ -702,6 +721,10 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
if (includeDefaults || hiraganaQuaternaryMode) {
builder.field("hiragana_quaternary_mode", hiraganaQuaternaryMode);
}
if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
builder.field("ignore_above", ignoreAbove);
}
}
@Override
@ -718,7 +741,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
}
}
if (value == null) {
if (value == null || value.length() > ignoreAbove) {
return;
}

47
plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java
View File

@ -403,4 +403,51 @@ public class ICUCollationKeywordFieldMapperTests extends ESSingleNodeTestCase {
assertEquals("Can't merge because of conflicts: [Cannot update language setting for [" + FIELD_TYPE
+ "], Cannot update strength setting for [" + FIELD_TYPE + "]]", e.getMessage());
}
public void testIgnoreAbove() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", FIELD_TYPE)
.field("ignore_above", 5).endObject().endObject()
.endObject().endObject());
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, mapper.mappingSource().toString());
ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", "elk")
.endObject()),
XContentType.JSON));
IndexableField[] fields = doc.rootDoc().getFields("field");
assertEquals(2, fields.length);
doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", "elasticsearch")
.endObject()),
XContentType.JSON));
fields = doc.rootDoc().getFields("field");
assertEquals(0, fields.length);
}
public void testUpdateIgnoreAbove() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", FIELD_TYPE).endObject().endObject()
.endObject().endObject());
indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", FIELD_TYPE)
.field("ignore_above", 5).endObject().endObject()
.endObject().endObject());
indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
}
}