mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-03-28 02:48:38 +00:00
Warn on badly-formed null values for date and IP field mappers (#62487)
In #57666 we changed when null_value was parsed for ip and date fields. Previously, the null value was stored as a string, and parsed into a date or InetAddress whenever a document containing a null value was encountered. Now, the values are parsed when the mappings are built, which means that bad values are detected up front; if you try and add a mapping with a badly-parsed ip or date for a null_value, the mapping will be rejected. This causes problems for upgrades in the case when you have a badly-formed null_value in a pre-7.9 cluster. This commit fixes the upgrade case by changing the logic to only logging a warning on the badly formed value, replicating the earlier behaviour. Fixes #62363
This commit is contained in:
Alan Woodward
Alan Woodward
parent
4d272a2a00
commit
91e2330529
server/src
main/java/org/elasticsearch/index/mapper
test/java/org/elasticsearch
action/admin/indices/rollover
index/mapper
test/framework/src/main/java/org/elasticsearch/index/mapper
@ -36,6 +36,7 @@ import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.Nullable;
|
||||
import org.elasticsearch.common.geo.ShapeRelation;
|
||||
import org.elasticsearch.common.joda.Joda;
|
||||
import org.elasticsearch.common.logging.DeprecationLogger;
|
||||
import org.elasticsearch.common.lucene.BytesRefs;
|
||||
import org.elasticsearch.common.time.DateFormatter;
|
||||
import org.elasticsearch.common.time.DateFormatters;
|
||||
@ -73,6 +74,8 @@ import static org.elasticsearch.common.time.DateUtils.toLong;
|
||||
/** A {@link FieldMapper} for dates. */
|
||||
public final class DateFieldMapper extends ParametrizedFieldMapper {
|
||||
|
||||
private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(DateFieldMapper.class);
|
||||
|
||||
public static final String CONTENT_TYPE = "date";
|
||||
public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos";
|
||||
public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis");
|
||||
@ -205,8 +208,8 @@ public final class DateFieldMapper extends ParametrizedFieldMapper {
|
||||
private final Resolution resolution;
|
||||
private final Version indexCreatedVersion;
|
||||
|
||||
public Builder(String name, Version indexCreatedVersion, Resolution resolution,
|
||||
DateFormatter dateFormatter, boolean ignoreMalformedByDefault) {
|
||||
public Builder(String name, Resolution resolution, DateFormatter dateFormatter,
|
||||
boolean ignoreMalformedByDefault, Version indexCreatedVersion) {
|
||||
super(name);
|
||||
this.resolution = resolution;
|
||||
this.indexCreatedVersion = indexCreatedVersion;
|
||||
@ -241,9 +244,10 @@ public final class DateFieldMapper extends ParametrizedFieldMapper {
|
||||
}
|
||||
try {
|
||||
return fieldType.parse(nullValue.getValue());
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e);
|
||||
} catch (Exception e) {
|
||||
DEPRECATION_LOGGER.deprecate("date_mapper_null_field", "Error parsing [" + nullValue.getValue()
|
||||
+ "] as date in [null_value] on field [" + name() + "]); [null_value] will be ignored");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@ -254,18 +258,18 @@ public final class DateFieldMapper extends ParametrizedFieldMapper {
|
||||
ft.setBoost(boost.getValue());
|
||||
Long nullTimestamp = parseNullValue(ft);
|
||||
return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context),
|
||||
copyTo.build(), nullTimestamp, resolution, indexCreatedVersion, this);
|
||||
copyTo.build(), nullTimestamp, resolution, this);
|
||||
}
|
||||
}
|
||||
|
||||
public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> {
|
||||
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
|
||||
return new Builder(n, c.indexVersionCreated(), Resolution.MILLISECONDS, c.getDateFormatter(), ignoreMalformedByDefault);
|
||||
return new Builder(n, Resolution.MILLISECONDS, c.getDateFormatter(), ignoreMalformedByDefault, c.indexVersionCreated());
|
||||
});
|
||||
|
||||
public static final TypeParser NANOS_PARSER = new TypeParser((n, c) -> {
|
||||
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
|
||||
return new Builder(n, c.indexVersionCreated(), Resolution.NANOSECONDS, c.getDateFormatter(), ignoreMalformedByDefault);
|
||||
return new Builder(n, Resolution.NANOSECONDS, c.getDateFormatter(), ignoreMalformedByDefault, c.indexVersionCreated());
|
||||
});
|
||||
|
||||
public static final class DateFieldType extends MappedFieldType {
|
||||
@ -525,9 +529,9 @@ public final class DateFieldMapper extends ParametrizedFieldMapper {
|
||||
private final Long nullValue;
|
||||
private final String nullValueAsString;
|
||||
private final Resolution resolution;
|
||||
private final Version indexCreatedVersion;
|
||||
|
||||
private final boolean ignoreMalformedByDefault;
|
||||
private final Version indexCreatedVersion;
|
||||
|
||||
private DateFieldMapper(
|
||||
String simpleName,
|
||||
@ -536,7 +540,6 @@ public final class DateFieldMapper extends ParametrizedFieldMapper {
|
||||
CopyTo copyTo,
|
||||
Long nullValue,
|
||||
Resolution resolution,
|
||||
Version indexCreatedVersion,
|
||||
Builder builder) {
|
||||
super(simpleName, mappedFieldType, multiFields, copyTo);
|
||||
this.store = builder.store.getValue();
|
||||
@ -548,13 +551,13 @@ public final class DateFieldMapper extends ParametrizedFieldMapper {
|
||||
this.nullValueAsString = builder.nullValue.getValue();
|
||||
this.nullValue = nullValue;
|
||||
this.resolution = resolution;
|
||||
this.indexCreatedVersion = indexCreatedVersion;
|
||||
this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue();
|
||||
this.indexCreatedVersion = builder.indexCreatedVersion;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ParametrizedFieldMapper.Builder getMergeBuilder() {
|
||||
return new Builder(simpleName(), indexCreatedVersion, resolution, null, ignoreMalformedByDefault).init(this);
|
||||
return new Builder(simpleName(), resolution, null, ignoreMalformedByDefault, indexCreatedVersion).init(this);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -696,9 +696,8 @@ final class DocumentParser {
|
||||
= context.root().findTemplateBuilder(context, currentFieldName, dateTimeFormatter);
|
||||
if (builder == null) {
|
||||
boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
|
||||
Version indexCreatedVersion = context.indexSettings().getIndexVersionCreated();
|
||||
builder = new DateFieldMapper.Builder(currentFieldName, indexCreatedVersion,
|
||||
DateFieldMapper.Resolution.MILLISECONDS, dateTimeFormatter, ignoreMalformed);
|
||||
builder = new DateFieldMapper.Builder(currentFieldName, DateFieldMapper.Resolution.MILLISECONDS,
|
||||
dateTimeFormatter, ignoreMalformed, Version.indexCreated(context.indexSettings().getSettings()));
|
||||
}
|
||||
return builder;
|
||||
|
||||
|
||||
@ -30,10 +30,11 @@ import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.util.ArrayUtil;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.Nullable;
|
||||
import org.elasticsearch.common.collect.Tuple;
|
||||
import org.elasticsearch.common.logging.DeprecationLogger;
|
||||
import org.elasticsearch.common.network.InetAddresses;
|
||||
import org.elasticsearch.common.network.NetworkAddress;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.ScriptDocValues;
|
||||
import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
|
||||
@ -55,6 +56,8 @@ import java.util.function.Supplier;
|
||||
/** A {@link FieldMapper} for ip addresses. */
|
||||
public class IpFieldMapper extends ParametrizedFieldMapper {
|
||||
|
||||
private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(IpFieldMapper.class);
|
||||
|
||||
public static final String CONTENT_TYPE = "ip";
|
||||
|
||||
private static IpFieldMapper toType(FieldMapper in) {
|
||||
@ -68,36 +71,44 @@ public class IpFieldMapper extends ParametrizedFieldMapper {
|
||||
private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
|
||||
|
||||
private final Parameter<Boolean> ignoreMalformed;
|
||||
private final Parameter<InetAddress> nullValue = new Parameter<>("null_value", false, () -> null,
|
||||
(n, c, o) -> o == null ? null : InetAddresses.forString(o.toString()), m -> toType(m).nullValue)
|
||||
.setSerializer((b, f, v) -> {
|
||||
if (v == null) {
|
||||
b.nullField(f);
|
||||
} else {
|
||||
b.field(f, InetAddresses.toAddrString(v));
|
||||
}
|
||||
}, NetworkAddress::format)
|
||||
.acceptsNull();
|
||||
private final Parameter<String> nullValue
|
||||
= Parameter.stringParam("null_value", false, m -> toType(m).nullValueAsString, null).acceptsNull();
|
||||
|
||||
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
|
||||
|
||||
private final boolean ignoreMalformedByDefault;
|
||||
private final Version indexCreatedVersion;
|
||||
|
||||
public Builder(String name, boolean ignoreMalformedByDefault) {
|
||||
public Builder(String name, boolean ignoreMalformedByDefault, Version indexCreatedVersion) {
|
||||
super(name);
|
||||
this.ignoreMalformedByDefault = ignoreMalformedByDefault;
|
||||
this.indexCreatedVersion = indexCreatedVersion;
|
||||
this.ignoreMalformed
|
||||
= Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
|
||||
}
|
||||
|
||||
Builder nullValue(InetAddress nullValue) {
|
||||
Builder nullValue(String nullValue) {
|
||||
this.nullValue.setValue(nullValue);
|
||||
return this;
|
||||
}
|
||||
|
||||
private InetAddress parseNullValue() {
|
||||
String nullValueAsString = nullValue.getValue();
|
||||
if (nullValueAsString == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return InetAddresses.forString(nullValueAsString);
|
||||
} catch (Exception e) {
|
||||
DEPRECATION_LOGGER.deprecate("ip_mapper_null_field", "Error parsing [" + nullValue.getValue()
|
||||
+ "] as IP in [null_value] on field [" + name() + "]); [null_value] will be ignored");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<Parameter<?>> getParameters() {
|
||||
return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, nullValue);
|
||||
return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, nullValue, meta);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -111,7 +122,7 @@ public class IpFieldMapper extends ParametrizedFieldMapper {
|
||||
|
||||
public static final TypeParser PARSER = new TypeParser((n, c) -> {
|
||||
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
|
||||
return new Builder(n, ignoreMalformedByDefault);
|
||||
return new Builder(n, ignoreMalformedByDefault, c.indexVersionCreated());
|
||||
});
|
||||
|
||||
public static final class IpFieldType extends SimpleMappedFieldType {
|
||||
@ -322,9 +333,12 @@ public class IpFieldMapper extends ParametrizedFieldMapper {
|
||||
private final boolean hasDocValues;
|
||||
private final boolean stored;
|
||||
private final boolean ignoreMalformed;
|
||||
|
||||
private final InetAddress nullValue;
|
||||
private final String nullValueAsString;
|
||||
|
||||
private final boolean ignoreMalformedByDefault;
|
||||
private final Version indexCreatedVersion;
|
||||
|
||||
private IpFieldMapper(
|
||||
String simpleName,
|
||||
@ -338,7 +352,9 @@ public class IpFieldMapper extends ParametrizedFieldMapper {
|
||||
this.hasDocValues = builder.hasDocValues.getValue();
|
||||
this.stored = builder.stored.getValue();
|
||||
this.ignoreMalformed = builder.ignoreMalformed.getValue();
|
||||
this.nullValue = builder.nullValue.getValue();
|
||||
this.nullValue = builder.parseNullValue();
|
||||
this.nullValueAsString = builder.nullValue.getValue();
|
||||
this.indexCreatedVersion = builder.indexCreatedVersion;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -424,6 +440,6 @@ public class IpFieldMapper extends ParametrizedFieldMapper {
|
||||
|
||||
@Override
|
||||
public ParametrizedFieldMapper.Builder getMergeBuilder() {
|
||||
return new Builder(simpleName(), ignoreMalformedByDefault).init(this);
|
||||
return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion).init(this);
|
||||
}
|
||||
}
|
||||
|
||||
@ -554,7 +554,7 @@ public class MetadataRolloverServiceTests extends ESTestCase {
|
||||
try {
|
||||
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(Settings.EMPTY, new ContentPath(0));
|
||||
DateFieldMapper dateFieldMapper
|
||||
= new DateFieldMapper.Builder("@timestamp", Version.CURRENT, DateFieldMapper.Resolution.MILLISECONDS, null, false)
|
||||
= new DateFieldMapper.Builder("@timestamp", DateFieldMapper.Resolution.MILLISECONDS, null, false, Version.CURRENT)
|
||||
.build(builderContext);
|
||||
MetadataFieldMapper mockedTimestampField = mock(MetadataFieldMapper.class);
|
||||
when(mockedTimestampField.name()).thenReturn("_data_stream_timestamp");
|
||||
|
||||
@ -40,7 +40,6 @@ import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.notNullValue;
|
||||
|
||||
public class DateFieldMapperTests extends MapperTestCase {
|
||||
@ -221,14 +220,10 @@ public class DateFieldMapperTests extends MapperTestCase {
|
||||
assertFalse(dvField.fieldType().stored());
|
||||
}
|
||||
|
||||
public void testBadNullValue() {
|
||||
public void testBadNullValue() throws IOException {
|
||||
createDocumentMapper(fieldMapping(b -> b.field("type", "date").field("null_value", "foo")));
|
||||
|
||||
MapperParsingException e = expectThrows(MapperParsingException.class,
|
||||
() -> createDocumentMapper(fieldMapping(b -> b.field("type", "date").field("null_value", ""))));
|
||||
|
||||
assertThat(e.getMessage(),
|
||||
equalTo("Failed to parse mapping [_doc]: Error parsing [null_value] on field [field]: cannot parse empty date"));
|
||||
}
|
||||
assertWarnings("Error parsing [foo] as date in [null_value] on field [field]); [null_value] will be ignored"); }
|
||||
|
||||
public void testNullConfigValuesFail() {
|
||||
Exception e = expectThrows(MapperParsingException.class,
|
||||
@ -364,7 +359,8 @@ public class DateFieldMapperTests extends MapperTestCase {
|
||||
mapping.put("null_value", nullValue);
|
||||
}
|
||||
|
||||
DateFieldMapper.Builder builder = new DateFieldMapper.Builder("field", Version.CURRENT, resolution, null, false);
|
||||
DateFieldMapper.Builder builder
|
||||
= new DateFieldMapper.Builder("field", resolution, null, false, Version.CURRENT);
|
||||
builder.parse("field", null, mapping);
|
||||
return builder.build(context);
|
||||
}
|
||||
|
||||
@ -28,61 +28,28 @@ import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.cluster.metadata.IndexMetadata;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.bytes.BytesReference;
|
||||
import org.elasticsearch.common.collect.List;
|
||||
import org.elasticsearch.common.compress.CompressedXContent;
|
||||
import org.elasticsearch.common.network.InetAddresses;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.ToXContent;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.elasticsearch.index.IndexService;
|
||||
import org.elasticsearch.index.termvectors.TermVectorsService;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.test.ESSingleNodeTestCase;
|
||||
import org.elasticsearch.test.InternalSettingsPlugin;
|
||||
import org.junit.Before;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.util.Collection;
|
||||
|
||||
import static org.elasticsearch.index.mapper.FieldMapperTestCase.fetchSourceValue;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
|
||||
public class IpFieldMapperTests extends ESSingleNodeTestCase {
|
||||
|
||||
IndexService indexService;
|
||||
DocumentMapperParser parser;
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
indexService = createIndex("test");
|
||||
parser = indexService.mapperService().documentMapperParser();
|
||||
}
|
||||
public class IpFieldMapperTests extends MapperTestCase {
|
||||
|
||||
@Override
|
||||
protected Collection<Class<? extends Plugin>> getPlugins() {
|
||||
return pluginList(InternalSettingsPlugin.class);
|
||||
protected void minimalMapping(XContentBuilder b) throws IOException {
|
||||
b.field("type", "ip");
|
||||
}
|
||||
|
||||
public void testDefaults() throws Exception {
|
||||
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
|
||||
.startObject("properties").startObject("field").field("type", "ip").endObject().endObject()
|
||||
.endObject().endObject());
|
||||
DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
|
||||
|
||||
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
|
||||
|
||||
assertEquals(mapping, mapper.mappingSource().toString());
|
||||
|
||||
ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
|
||||
.bytes(XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.field("field", "::1")
|
||||
.endObject()),
|
||||
XContentType.JSON));
|
||||
ParsedDocument doc = mapper.parse(source(b -> b.field("field", "::1")));
|
||||
|
||||
IndexableField[] fields = doc.rootDoc().getFields("field");
|
||||
assertEquals(2, fields.length);
|
||||
@ -98,20 +65,13 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
|
||||
}
|
||||
|
||||
public void testNotIndexed() throws Exception {
|
||||
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
|
||||
.startObject("properties").startObject("field").field("type", "ip").field("index", false).endObject().endObject()
|
||||
.endObject().endObject());
|
||||
|
||||
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
|
||||
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
|
||||
b.field("type", "ip");
|
||||
b.field("index", false);
|
||||
}));
|
||||
|
||||
assertEquals(mapping, mapper.mappingSource().toString());
|
||||
|
||||
ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
|
||||
.bytes(XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.field("field", "::1")
|
||||
.endObject()),
|
||||
XContentType.JSON));
|
||||
ParsedDocument doc = mapper.parse(source(b -> b.field("field", "::1")));
|
||||
|
||||
IndexableField[] fields = doc.rootDoc().getFields("field");
|
||||
assertEquals(1, fields.length);
|
||||
@ -120,20 +80,13 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
|
||||
}
|
||||
|
||||
public void testNoDocValues() throws Exception {
|
||||
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
|
||||
.startObject("properties").startObject("field").field("type", "ip").field("doc_values", false).endObject().endObject()
|
||||
.endObject().endObject());
|
||||
|
||||
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
|
||||
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
|
||||
b.field("type", "ip");
|
||||
b.field("doc_values", false);
|
||||
}));
|
||||
|
||||
assertEquals(mapping, mapper.mappingSource().toString());
|
||||
|
||||
ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
|
||||
.bytes(XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.field("field", "::1")
|
||||
.endObject()),
|
||||
XContentType.JSON));
|
||||
ParsedDocument doc = mapper.parse(source(b -> b.field("field", "::1")));
|
||||
|
||||
IndexableField[] fields = doc.rootDoc().getFields("field");
|
||||
assertEquals(1, fields.length);
|
||||
@ -151,20 +104,13 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
|
||||
}
|
||||
|
||||
public void testStore() throws Exception {
|
||||
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
|
||||
.startObject("properties").startObject("field").field("type", "ip").field("store", true).endObject().endObject()
|
||||
.endObject().endObject());
|
||||
|
||||
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
|
||||
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
|
||||
b.field("type", "ip");
|
||||
b.field("store", true);
|
||||
}));
|
||||
|
||||
assertEquals(mapping, mapper.mappingSource().toString());
|
||||
|
||||
ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
|
||||
.bytes(XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.field("field", "::1")
|
||||
.endObject()),
|
||||
XContentType.JSON));
|
||||
ParsedDocument doc = mapper.parse(source(b -> b.field("field", "::1")));
|
||||
|
||||
IndexableField[] fields = doc.rootDoc().getFields("field");
|
||||
assertEquals(3, fields.length);
|
||||
@ -179,35 +125,19 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
|
||||
}
|
||||
|
||||
public void testIgnoreMalformed() throws Exception {
|
||||
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
|
||||
.startObject("properties").startObject("field").field("type", "ip").endObject().endObject()
|
||||
.endObject().endObject());
|
||||
|
||||
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
|
||||
DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
|
||||
|
||||
assertEquals(mapping, mapper.mappingSource().toString());
|
||||
|
||||
ThrowingRunnable runnable = () -> mapper.parse(new SourceToParse("test", "type", "1", BytesReference
|
||||
.bytes(XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.field("field", ":1")
|
||||
.endObject()),
|
||||
XContentType.JSON));
|
||||
ThrowingRunnable runnable = () -> mapper.parse(source(b -> b.field("field", ":1")));
|
||||
MapperParsingException e = expectThrows(MapperParsingException.class, runnable);
|
||||
assertThat(e.getCause().getMessage(), containsString("':1' is not an IP string literal"));
|
||||
|
||||
mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
|
||||
.startObject("properties").startObject("field").field("type", "ip").field("ignore_malformed", true).endObject().endObject()
|
||||
.endObject().endObject());
|
||||
DocumentMapper mapper2 = createDocumentMapper(fieldMapping(b -> {
|
||||
b.field("type", "ip");
|
||||
b.field("ignore_malformed", true);
|
||||
}));
|
||||
|
||||
DocumentMapper mapper2 = parser.parse("type", new CompressedXContent(mapping));
|
||||
|
||||
ParsedDocument doc = mapper2.parse(new SourceToParse("test", "type", "1", BytesReference
|
||||
.bytes(XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.field("field", ":1")
|
||||
.endObject()),
|
||||
XContentType.JSON));
|
||||
ParsedDocument doc = mapper2.parse(source(b -> b.field("field", ":1")));
|
||||
|
||||
IndexableField[] fields = doc.rootDoc().getFields("field");
|
||||
assertEquals(0, fields.length);
|
||||
@ -215,45 +145,19 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
|
||||
}
|
||||
|
||||
public void testNullValue() throws IOException {
|
||||
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
|
||||
.startObject("type")
|
||||
.startObject("properties")
|
||||
.startObject("field")
|
||||
.field("type", "ip")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject().endObject());
|
||||
|
||||
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
|
||||
assertEquals(mapping, mapper.mappingSource().toString());
|
||||
DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
|
||||
|
||||
ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
|
||||
.bytes(XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.nullField("field")
|
||||
.endObject()),
|
||||
XContentType.JSON));
|
||||
ParsedDocument doc = mapper.parse(source(b -> b.nullField("field")));
|
||||
assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("field"));
|
||||
|
||||
mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
|
||||
.startObject("type")
|
||||
.startObject("properties")
|
||||
.startObject("field")
|
||||
.field("type", "ip")
|
||||
.field("null_value", "::1")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject().endObject());
|
||||
mapper = createDocumentMapper(fieldMapping(b -> {
|
||||
b.field("type", "ip");
|
||||
b.field("null_value", "::1");
|
||||
}));
|
||||
|
||||
mapper = parser.parse("type", new CompressedXContent(mapping));
|
||||
assertEquals(mapping, mapper.mappingSource().toString());
|
||||
doc = mapper.parse(source(b -> b.nullField("field")));
|
||||
|
||||
doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
|
||||
.bytes(XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.nullField("field")
|
||||
.endObject()),
|
||||
XContentType.JSON));
|
||||
IndexableField[] fields = doc.rootDoc().getFields("field");
|
||||
assertEquals(2, fields.length);
|
||||
IndexableField pointField = fields[0];
|
||||
@ -265,46 +169,33 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
|
||||
assertEquals(DocValuesType.SORTED_SET, dvField.fieldType().docValuesType());
|
||||
assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), dvField.binaryValue());
|
||||
assertFalse(dvField.fieldType().stored());
|
||||
}
|
||||
|
||||
public void testSerializeDefaults() throws Exception {
|
||||
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
|
||||
.startObject("properties").startObject("field").field("type", "ip").endObject().endObject()
|
||||
.endObject().endObject());
|
||||
mapper = createDocumentMapper(fieldMapping(b -> {
|
||||
b.field("type", "ip");
|
||||
b.nullField("null_value");
|
||||
}));
|
||||
|
||||
DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping));
|
||||
IpFieldMapper mapper = (IpFieldMapper)docMapper.root().getMapper("field");
|
||||
XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
|
||||
mapper.doXContentBody(builder, true, ToXContent.EMPTY_PARAMS);
|
||||
String got = Strings.toString(builder.endObject());
|
||||
doc = mapper.parse(source(b -> b.nullField("field")));
|
||||
assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("field"));
|
||||
|
||||
// it would be nice to check the entire serialized default mapper, but there are
|
||||
// a whole lot of bogus settings right now it picks up from calling super.doXContentBody...
|
||||
assertTrue(got, got.contains("\"ignore_malformed\":false"));
|
||||
}
|
||||
|
||||
public void testEmptyName() throws IOException {
|
||||
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
|
||||
.startObject("properties").startObject("").field("type", "ip").endObject().endObject()
|
||||
.endObject().endObject());
|
||||
|
||||
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
|
||||
() -> parser.parse("type", new CompressedXContent(mapping))
|
||||
);
|
||||
assertThat(e.getMessage(), containsString("name cannot be empty string"));
|
||||
createDocumentMapper(fieldMapping(b -> {
|
||||
b.field("type", "ip");
|
||||
b.field("null_value", ":1");
|
||||
}));
|
||||
assertWarnings("Error parsing [:1] as IP in [null_value] on field [field]); [null_value] will be ignored");
|
||||
}
|
||||
|
||||
public void testFetchSourceValue() throws IOException {
|
||||
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
|
||||
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
|
||||
|
||||
IpFieldMapper mapper = new IpFieldMapper.Builder("field", true).build(context);
|
||||
IpFieldMapper mapper = new IpFieldMapper.Builder("field", true, Version.CURRENT).build(context);
|
||||
assertEquals(List.of("2001:db8::2:1"), fetchSourceValue(mapper, "2001:db8::2:1"));
|
||||
assertEquals(List.of("2001:db8::2:1"), fetchSourceValue(mapper, "2001:db8:0:0:0:0:2:1"));
|
||||
assertEquals(List.of("::1"), fetchSourceValue(mapper, "0:0:0:0:0:0:0:1"));
|
||||
|
||||
IpFieldMapper nullValueMapper = new IpFieldMapper.Builder("field", true)
|
||||
.nullValue(InetAddresses.forString("2001:db8:0:0:0:0:2:7"))
|
||||
IpFieldMapper nullValueMapper = new IpFieldMapper.Builder("field", true, Version.CURRENT)
|
||||
.nullValue("2001:db8:0:0:0:0:2:7")
|
||||
.build(context);
|
||||
assertEquals(List.of("2001:db8::2:7"), fetchSourceValue(nullValueMapper, null));
|
||||
}
|
||||
|
||||
@ -102,7 +102,7 @@ public abstract class MapperServiceTestCase extends ESTestCase {
|
||||
}
|
||||
|
||||
protected final MapperService createMapperService(XContentBuilder mappings) throws IOException {
|
||||
return createMapperService(getIndexSettings(), mappings);
|
||||
return createMapperService(Version.CURRENT, mappings);
|
||||
}
|
||||
|
||||
protected final MapperService createMapperService(String type, String mappings) throws IOException {
|
||||
@ -114,13 +114,13 @@ public abstract class MapperServiceTestCase extends ESTestCase {
|
||||
/**
|
||||
* Create a {@link MapperService} like we would for an index.
|
||||
*/
|
||||
protected final MapperService createMapperService(Settings settings, XContentBuilder mapping) throws IOException {
|
||||
protected final MapperService createMapperService(Version version, XContentBuilder mapping) throws IOException {
|
||||
IndexMetadata meta = IndexMetadata.builder("index")
|
||||
.settings(Settings.builder().put("index.version.created", Version.CURRENT))
|
||||
.settings(Settings.builder().put("index.version.created", version))
|
||||
.numberOfReplicas(0)
|
||||
.numberOfShards(1)
|
||||
.build();
|
||||
IndexSettings indexSettings = new IndexSettings(meta, Settings.EMPTY);
|
||||
IndexSettings indexSettings = new IndexSettings(meta, getIndexSettings());
|
||||
MapperRegistry mapperRegistry = new IndicesModule(
|
||||
getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList())
|
||||
).getMapperRegistry();
|
||||
@ -128,7 +128,7 @@ public abstract class MapperServiceTestCase extends ESTestCase {
|
||||
Settings.EMPTY,
|
||||
getPlugins().stream().filter(p -> p instanceof ScriptPlugin).map(p -> (ScriptPlugin) p).collect(toList())
|
||||
);
|
||||
ScriptService scriptService = new ScriptService(settings, scriptModule.engines, scriptModule.contexts);
|
||||
ScriptService scriptService = new ScriptService(getIndexSettings(), scriptModule.engines, scriptModule.contexts);
|
||||
SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, emptyMap());
|
||||
MapperService mapperService = new MapperService(
|
||||
indexSettings,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user