honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-25 14:26:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cut over IPFieldMapper to parametrized form (#60602)

Browse Source 
This commit makes IpFieldMapper extend ParametrizedFieldMapper. It also
updates the IpFieldMapper docs to add the ignore_malformed parameter,
which was not previously documented.
This commit is contained in:
Alan Woodward 2020-08-10 10:57:06 +01:00 committed by Alan Woodward
parent a2d5bfca2f
commit e8d9185045
4 changed files with 73 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/reference/mapping/types/ip.asciidoc
View File

@ -56,13 +56,18 @@ The following parameters are accepted by `ip` fields:
can later be used for sorting, aggregations, or scripting? Accepts `true`
(default) or `false`.
<<ignore-malformed,`ignore_malformed`>>::
If `true`, malformed IP addresses are ignored. If `false` (default), malformed
IP addresses throw an exception and reject the whole document.
<<mapping-index,`index`>>::
Should the field be searchable? Accepts `true` (default) and `false`.
<<null-value,`null_value`>>::
Accepts an IPv4 value which is substituted for any explicit `null` values.
Accepts an IPv4 or IPv6 value which is substituted for any explicit `null` values.
Defaults to `null`, which means the field is treated as missing.
<<mapping-store,`store`>>::

165
server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
View File

@ -19,11 +19,9 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.InetAddressPoint;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
@ -32,12 +30,9 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
@ -50,93 +45,69 @@ import java.net.InetAddress;
import java.time.ZoneId;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
/** A {@link FieldMapper} for ip addresses. */
public class IpFieldMapper extends FieldMapper {
public class IpFieldMapper extends ParametrizedFieldMapper {
public static final String CONTENT_TYPE = "ip";
public static class Defaults {
public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<>(false, false);
public static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setDimensions(1, Integer.BYTES);
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
FIELD_TYPE.freeze();
}
private static IpFieldMapper toType(FieldMapper in) {
return (IpFieldMapper) in;
}
public static class Builder extends FieldMapper.Builder<Builder> {
public static class Builder extends ParametrizedFieldMapper.Builder {
private Boolean ignoreMalformed;
private InetAddress nullValue;
private final Parameter<Boolean> indexed = Parameter.indexParam(m -> toType(m).indexed, true);
private final Parameter<Boolean> hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
public Builder(String name) {
super(name, Defaults.FIELD_TYPE);
builder = this;
private final Parameter<Boolean> ignoreMalformed;
private final Parameter<InetAddress> nullValue = new Parameter<>("null_value", false, () -> null,
(n, c, o) -> InetAddresses.forString(o.toString()), m -> toType(m).nullValue)
.setSerializer((b, f, v) -> {
if (v == null) {
b.nullField(f);
} else {
b.field(f, InetAddresses.toAddrString(v));
}
});
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
private final boolean ignoreMalformedByDefault;
public Builder(String name, boolean ignoreMalformedByDefault) {
super(name);
this.ignoreMalformedByDefault = ignoreMalformedByDefault;
this.ignoreMalformed
= Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
}
public Builder ignoreMalformed(boolean ignoreMalformed) {
this.ignoreMalformed = ignoreMalformed;
return builder;
Builder nullValue(InetAddress nullValue) {
this.nullValue.setValue(nullValue);
return this;
}
public Builder nullValue(InetAddress nullValue) {
this.nullValue = nullValue;
return builder;
}
protected Explicit<Boolean> ignoreMalformed(BuilderContext context) {
if (ignoreMalformed != null) {
return new Explicit<>(ignoreMalformed, true);
}
if (context.indexSettings() != null) {
return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false);
}
return Defaults.IGNORE_MALFORMED;
@Override
protected List<Parameter<?>> getParameters() {
return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, nullValue);
}
@Override
public IpFieldMapper build(BuilderContext context) {
return new IpFieldMapper(name, fieldType, new IpFieldType(buildFullName(context), indexed, hasDocValues, meta),
ignoreMalformed(context), nullValue,
multiFieldsBuilder.build(this, context), copyTo);
return new IpFieldMapper(name,
new IpFieldType(buildFullName(context), indexed.getValue(), hasDocValues.getValue(), meta.getValue()),
multiFieldsBuilder.build(this, context), copyTo.build(), this);
}
}
public static class TypeParser implements Mapper.TypeParser {
public TypeParser() {
}
@Override
public Mapper.Builder<?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
Builder builder = new Builder(name);
TypeParsers.parseField(builder, name, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
String propName = entry.getKey();
Object propNode = entry.getValue();
if (propName.equals("null_value")) {
if (propNode == null) {
throw new MapperParsingException("Property [null_value] cannot be null.");
}
builder.nullValue(InetAddresses.forString(propNode.toString()));
iterator.remove();
} else if (propName.equals("ignore_malformed")) {
builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(propNode, name + ".ignore_malformed"));
iterator.remove();
} else if (TypeParsers.parseMultiField(builder::addMultiField, name, parserContext, propName, propNode)) {
iterator.remove();
}
}
return builder;
}
}
public static final TypeParser PARSER = new TypeParser((n, c) -> {
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
return new Builder(n, ignoreMalformedByDefault);
});
public static final class IpFieldType extends SimpleMappedFieldType {
@ -322,20 +293,27 @@ public class IpFieldMapper extends FieldMapper {
}
}
private Explicit<Boolean> ignoreMalformed;
private final boolean indexed;
private final boolean hasDocValues;
private final boolean stored;
private final boolean ignoreMalformed;
private final InetAddress nullValue;
private final boolean ignoreMalformedByDefault;
private IpFieldMapper(
String simpleName,
FieldType fieldType,
MappedFieldType mappedFieldType,
Explicit<Boolean> ignoreMalformed,
InetAddress nullValue,
MultiFields multiFields,
CopyTo copyTo) {
super(simpleName, fieldType, mappedFieldType, multiFields, copyTo);
this.ignoreMalformed = ignoreMalformed;
this.nullValue = nullValue;
CopyTo copyTo,
Builder builder) {
super(simpleName, mappedFieldType, multiFields, copyTo);
this.ignoreMalformedByDefault = builder.ignoreMalformedByDefault;
this.indexed = builder.indexed.getValue();
this.hasDocValues = builder.hasDocValues.getValue();
this.stored = builder.stored.getValue();
this.ignoreMalformed = builder.ignoreMalformed.getValue();
this.nullValue = builder.nullValue.getValue();
}
@Override
@ -383,7 +361,7 @@ public class IpFieldMapper extends FieldMapper {
try {
address = InetAddresses.forString(addressAsString);
} catch (IllegalArgumentException e) {
if (ignoreMalformed.value()) {
if (ignoreMalformed) {
context.addIgnoredField(fieldType().name());
return;
} else {
@ -392,15 +370,15 @@ public class IpFieldMapper extends FieldMapper {
}
}
if (fieldType().isSearchable()) {
if (indexed) {
context.doc().add(new InetAddressPoint(fieldType().name(), address));
}
if (fieldType().hasDocValues()) {
if (hasDocValues) {
context.doc().add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address))));
} else if (fieldType.stored() || fieldType().isSearchable()) {
} else if (stored || indexed) {
createFieldNamesField(context);
}
if (fieldType.stored()) {
if (stored) {
context.doc().add(new StoredField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address))));
}
}
@ -421,26 +399,7 @@ public class IpFieldMapper extends FieldMapper {
}
@Override
protected void mergeOptions(FieldMapper other, List<String> conflicts) {
IpFieldMapper mergeWith = (IpFieldMapper) other;
if (mergeWith.nullValue != this.nullValue) {
conflicts.add("mapper [" + name() + "] has different [null_value] values");
}
if (mergeWith.ignoreMalformed.explicit()) {
this.ignoreMalformed = mergeWith.ignoreMalformed;
}
}
@Override
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);
if (nullValue != null) {
builder.field("null_value", InetAddresses.toAddrString(nullValue));
}
if (includeDefaults || ignoreMalformed.explicit()) {
builder.field("ignore_malformed", ignoreMalformed.value());
}
public ParametrizedFieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), ignoreMalformedByDefault).init(this);
}
}

2
server/src/main/java/org/elasticsearch/indices/IndicesModule.java
View File

@ -125,7 +125,7 @@ public class IndicesModule extends AbstractModule {
mappers.put(milliseconds.type(), DateFieldMapper.MILLIS_PARSER);
DateFieldMapper.Resolution nanoseconds = DateFieldMapper.Resolution.NANOSECONDS;
mappers.put(nanoseconds.type(), DateFieldMapper.NANOS_PARSER);
mappers.put(IpFieldMapper.CONTENT_TYPE, new IpFieldMapper.TypeParser());
mappers.put(IpFieldMapper.CONTENT_TYPE, IpFieldMapper.PARSER);
mappers.put(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser());
mappers.put(KeywordFieldMapper.CONTENT_TYPE, new KeywordFieldMapper.TypeParser());
mappers.put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser());

20
server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
View File

@ -42,6 +42,7 @@ import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.termvectors.TermVectorsService;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.lookup.SourceLookup;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;
import org.junit.Before;
@ -49,11 +50,10 @@ import java.io.IOException;
import java.net.InetAddress;
import java.util.Collection;
import java.util.Collections;
import java.util.Set;
import static org.hamcrest.Matchers.containsString;
public class IpFieldMapperTests extends FieldMapperTestCase<IpFieldMapper.Builder> {
public class IpFieldMapperTests extends ESSingleNodeTestCase {
IndexService indexService;
DocumentMapperParser parser;
@ -62,14 +62,6 @@ public class IpFieldMapperTests extends FieldMapperTestCase<IpFieldMapper.Builde
public void setup() {
indexService = createIndex("test");
parser = indexService.mapperService().documentMapperParser();
addModifier("null_value", false, (a, b) -> {
a.nullValue(InetAddresses.forString("::1"));
});
}
@Override
protected Set<String> unsupportedProperties() {
return org.elasticsearch.common.collect.Set.of("analyzer", "similarity");
}
@Override
@ -307,12 +299,12 @@ public class IpFieldMapperTests extends FieldMapperTestCase<IpFieldMapper.Builde
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
IpFieldMapper mapper = new IpFieldMapper.Builder("field").build(context);
IpFieldMapper mapper = new IpFieldMapper.Builder("field", true).build(context);
assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8::2:1", null));
assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1", null));
assertEquals("::1", mapper.parseSourceValue("0:0:0:0:0:0:0:1", null));
IpFieldMapper nullValueMapper = new IpFieldMapper.Builder("field")
IpFieldMapper nullValueMapper = new IpFieldMapper.Builder("field", true)
.nullValue(InetAddresses.forString("2001:db8:0:0:0:0:2:7"))
.build(context);
SourceLookup sourceLookup = new SourceLookup();
@ -320,8 +312,4 @@ public class IpFieldMapperTests extends FieldMapperTestCase<IpFieldMapper.Builde
assertEquals(List.of("2001:db8::2:7"), nullValueMapper.lookupValues(sourceLookup, null));
}
@Override
protected IpFieldMapper.Builder newBuilder() {
return new IpFieldMapper.Builder("ip");
}
}