From cd0e1226e1788c376ba1030a241d7d83d36a7cf1 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 31 Aug 2012 16:13:43 +0200 Subject: [PATCH] Added a global ignore_malformed index setting. #2220 Also extended the ignore_malformed support to TTL, Ip and timestamp field types. --- .../index/mapper/core/ByteFieldMapper.java | 2 +- .../index/mapper/core/DateFieldMapper.java | 2 +- .../index/mapper/core/DoubleFieldMapper.java | 2 +- .../index/mapper/core/FloatFieldMapper.java | 2 +- .../index/mapper/core/IntegerFieldMapper.java | 2 +- .../index/mapper/core/LongFieldMapper.java | 2 +- .../index/mapper/core/NumberFieldMapper.java | 11 ++- .../index/mapper/core/ShortFieldMapper.java | 2 +- .../index/mapper/internal/TTLFieldMapper.java | 8 +-- .../mapper/internal/TimestampFieldMapper.java | 9 +-- .../index/mapper/ip/IpFieldMapper.java | 6 +- .../test/unit/index/mapper/MapperTests.java | 5 ++ .../mapper/date/SimpleDateMappingTests.java | 67 +++++++++++++++++++ .../mapper/numeric/SimpleNumericTests.java | 25 ++++++- 14 files changed, 125 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java index f82b11f04ac..c8bb5553529 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java @@ -76,7 +76,7 @@ public class ByteFieldMapper extends NumberFieldMapper { @Override public ByteFieldMapper build(BuilderContext context) { ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context), - precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, ignoreMalformed); + precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, ignoreMalformed(context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 404114731d2..f74d6ed8f36 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -105,7 +105,7 @@ public class DateFieldMapper extends NumberFieldMapper { } DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter, precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, - timeUnit, parseUpperInclusive, ignoreMalformed); + timeUnit, parseUpperInclusive, ignoreMalformed(context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index ec244b8450f..f5268c49c78 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -77,7 +77,7 @@ public class DoubleFieldMapper extends NumberFieldMapper { public DoubleFieldMapper build(BuilderContext context) { DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context), precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, - ignoreMalformed); + ignoreMalformed(context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java index 8215d91fa8c..f923c1c9094 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java @@ -78,7 +78,7 @@ public class FloatFieldMapper extends NumberFieldMapper { public FloatFieldMapper build(BuilderContext context) { FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context), precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, - ignoreMalformed); + ignoreMalformed(context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java index b3d6992c805..8a6a897dc51 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java @@ -78,7 +78,7 @@ public class IntegerFieldMapper extends NumberFieldMapper { public IntegerFieldMapper build(BuilderContext context) { IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context), precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, - nullValue, ignoreMalformed); + nullValue, ignoreMalformed(context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java index 7d7a4a95b64..5dad4f62de0 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java @@ -78,7 +78,7 @@ public class LongFieldMapper extends NumberFieldMapper { public LongFieldMapper build(BuilderContext context) { LongFieldMapper fieldMapper = new LongFieldMapper(buildNames(context), precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, - ignoreMalformed); + ignoreMalformed(context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java index 0e44d50c666..fe3ef480183 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java @@ -60,7 +60,7 @@ public abstract class NumberFieldMapper extends AbstractFieldM protected String fuzzyFactor = Defaults.FUZZY_FACTOR; - protected boolean ignoreMalformed = Defaults.IGNORE_MALFORMED; + private Boolean ignoreMalformed; public Builder(String name) { super(name); @@ -104,6 +104,15 @@ public abstract class NumberFieldMapper extends AbstractFieldM return builder; } + protected boolean ignoreMalformed(BuilderContext context) { + if (ignoreMalformed != null) { + return ignoreMalformed; + } + if (context.indexSettings() != null) { + return context.indexSettings().getAsBoolean("index.mapping.ignore_malformed", Defaults.IGNORE_MALFORMED); + } + return Defaults.IGNORE_MALFORMED; + } } protected int precisionStep; diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java index 12bfc2487a1..bc7fac751c3 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java @@ -78,7 +78,7 @@ public class ShortFieldMapper extends NumberFieldMapper { public ShortFieldMapper build(BuilderContext context) { ShortFieldMapper fieldMapper = new ShortFieldMapper(buildNames(context), precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, - ignoreMalformed); + ignoreMalformed(context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index e175e83f247..6f3468ea147 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -75,7 +75,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R @Override public TTLFieldMapper build(BuilderContext context) { - return new TTLFieldMapper(store, index, enabled, defaultTTL); + return new TTLFieldMapper(store, index, enabled, defaultTTL, ignoreMalformed(context)); } } @@ -104,13 +104,13 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R private long defaultTTL; public TTLFieldMapper() { - this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.DEFAULT); + this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.DEFAULT, Defaults.IGNORE_MALFORMED); } - protected TTLFieldMapper(Field.Store store, Field.Index index, boolean enabled, long defaultTTL) { + protected TTLFieldMapper(Field.Store store, Field.Index index, boolean enabled, long defaultTTL, boolean ignoreMalformed) { super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, index, store, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.INDEX_OPTIONS, - Defaults.NULL_VALUE, Defaults.IGNORE_MALFORMED); + Defaults.NULL_VALUE, ignoreMalformed); this.enabled = enabled; this.defaultTTL = defaultTTL; } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index 467ab3f5357..1ba1ac7f5e9 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -89,7 +89,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap if (context.indexSettings() != null) { parseUpperInclusive = context.indexSettings().getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.PARSE_UPPER_INCLUSIVE); } - return new TimestampFieldMapper(store, index, enabled, path, dateTimeFormatter, parseUpperInclusive); + return new TimestampFieldMapper(store, index, enabled, path, dateTimeFormatter, parseUpperInclusive, ignoreMalformed(context)); } } @@ -119,14 +119,15 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap private final String path; public TimestampFieldMapper() { - this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.PARSE_UPPER_INCLUSIVE); + this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.PARSE_UPPER_INCLUSIVE, Defaults.IGNORE_MALFORMED); } - protected TimestampFieldMapper(Field.Store store, Field.Index index, boolean enabled, String path, FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive) { + protected TimestampFieldMapper(Field.Store store, Field.Index index, boolean enabled, String path, + FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive, boolean ignoreMalformed) { super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter, Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, index, store, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.INDEX_OPTIONS, Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/, - parseUpperInclusive, Defaults.IGNORE_MALFORMED); + parseUpperInclusive, ignoreMalformed); this.enabled = enabled; this.path = path; } diff --git a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java index 70047a19471..0500cb6a4e7 100644 --- a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java @@ -105,7 +105,7 @@ public class IpFieldMapper extends NumberFieldMapper { @Override public IpFieldMapper build(BuilderContext context) { IpFieldMapper fieldMapper = new IpFieldMapper(buildNames(context), - precisionStep, index, store, boost, omitNorms, indexOptions, nullValue); + precisionStep, index, store, boost, omitNorms, indexOptions, nullValue, ignoreMalformed(context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } @@ -132,9 +132,9 @@ public class IpFieldMapper extends NumberFieldMapper { protected IpFieldMapper(Names names, int precisionStep, Field.Index index, Field.Store store, float boost, boolean omitNorms, IndexOptions indexOptions, - String nullValue) { + String nullValue, boolean ignoreMalformed) { super(names, precisionStep, null, index, store, boost, omitNorms, indexOptions, - false, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)), + ignoreMalformed, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)), new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE))); this.nullValue = nullValue; } diff --git a/src/test/java/org/elasticsearch/test/unit/index/mapper/MapperTests.java b/src/test/java/org/elasticsearch/test/unit/index/mapper/MapperTests.java index bd642ef69c4..3fdcd5725e1 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/mapper/MapperTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/mapper/MapperTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.test.unit.index.mapper; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.EnvironmentModule; @@ -44,6 +45,10 @@ public class MapperTests { return new DocumentMapperParser(new Index("test"), newAnalysisService()); } + public static DocumentMapperParser newParser(Settings indexSettings) { + return new DocumentMapperParser(new Index("test"), indexSettings, newAnalysisService()); + } + public static MapperService newMapperService() { return new MapperService(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS, new Environment(), newAnalysisService()); } diff --git a/src/test/java/org/elasticsearch/test/unit/index/mapper/date/SimpleDateMappingTests.java b/src/test/java/org/elasticsearch/test/unit/index/mapper/date/SimpleDateMappingTests.java index fb2d4a598ac..0ec0c025db7 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/mapper/date/SimpleDateMappingTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/mapper/date/SimpleDateMappingTests.java @@ -19,14 +19,17 @@ package org.elasticsearch.test.unit.index.mapper.date; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.core.DateFieldMapper; import org.elasticsearch.test.unit.index.mapper.MapperTests; import org.testng.annotations.Test; +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; @@ -97,4 +100,68 @@ public class SimpleDateMappingTests { assertThat(doc.rootDoc().get("date_field"), nullValue()); assertThat(doc.rootDoc().get("date_field_x"), equalTo("2010-01-01")); } + + @Test + public void testIgnoreMalformedOption() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("field1").field("type", "date").field("ignore_malformed", true).endObject() + .startObject("field2").field("type", "date").field("ignore_malformed", false).endObject() + .startObject("field3").field("type", "date").endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = MapperTests.newParser().parse(mapping); + + ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field1", "a") + .field("field2", "2010-01-01") + .endObject() + .bytes()); + assertThat(doc.rootDoc().getFieldable("field1"), nullValue()); + assertThat(doc.rootDoc().getFieldable("field2"), notNullValue()); + + try { + defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field2", "a") + .endObject() + .bytes()); + } catch (MapperParsingException e) { + assertThat(e.getCause(), instanceOf(MapperParsingException.class)); + } + + // Verify that the default is false + try { + defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field3", "a") + .endObject() + .bytes()); + } catch (MapperParsingException e) { + assertThat(e.getCause(), instanceOf(MapperParsingException.class)); + } + + // Unless the global ignore_malformed option is set to true + Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build(); + defaultMapper = MapperTests.newParser(indexSettings).parse(mapping); + doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field3", "a") + .endObject() + .bytes()); + assertThat(doc.rootDoc().getFieldable("field3"), nullValue()); + + // This should still throw an exception, since field2 is specifically set to ignore_malformed=false + try { + defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field2", "a") + .endObject() + .bytes()); + } catch (MapperParsingException e) { + assertThat(e.getCause(), instanceOf(MapperParsingException.class)); + } + } } \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/test/unit/index/mapper/numeric/SimpleNumericTests.java b/src/test/java/org/elasticsearch/test/unit/index/mapper/numeric/SimpleNumericTests.java index dfb92d4ca05..eba387eafac 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/mapper/numeric/SimpleNumericTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/mapper/numeric/SimpleNumericTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.test.unit.index.mapper.numeric; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; @@ -30,6 +31,7 @@ import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.test.unit.index.mapper.MapperTests; import org.testng.annotations.Test; +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; @@ -79,7 +81,7 @@ public class SimpleNumericTests { assertThat(mapper, instanceOf(StringFieldMapper.class)); } - public void testIgnoreMalformedEnabled() throws Exception { + public void testIgnoreMalformedOption() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") .startObject("field1").field("type", "integer").field("ignore_malformed", true).endObject() @@ -119,6 +121,27 @@ public class SimpleNumericTests { } catch (MapperParsingException e) { assertThat(e.getCause(), instanceOf(NumberFormatException.class)); } + + // Unless the global ignore_malformed option is set to true + Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build(); + defaultMapper = MapperTests.newParser(indexSettings).parse(mapping); + doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field3", "a") + .endObject() + .bytes()); + assertThat(doc.rootDoc().getFieldable("field3"), nullValue()); + + // This should still throw an exception, since field2 is specifically set to ignore_malformed=false + try { + defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field2", "a") + .endObject() + .bytes()); + } catch (MapperParsingException e) { + assertThat(e.getCause(), instanceOf(NumberFormatException.class)); + } } }