Added a global ignore_malformed index setting. #2220 Also extended the ignore_malformed support to TTL, Ip and timestamp field types.

src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java

@@ -76,7 +76,7 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
         @Override
         public ByteFieldMapper build(BuilderContext context) {
             ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context),
-                    precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, ignoreMalformed);
+                    precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, ignoreMalformed(context));
             fieldMapper.includeInAll(includeInAll);
             return fieldMapper;
         }

src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java

@@ -105,7 +105,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
             }
             DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter,
                     precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
-                    timeUnit, parseUpperInclusive, ignoreMalformed);
+                    timeUnit, parseUpperInclusive, ignoreMalformed(context));
             fieldMapper.includeInAll(includeInAll);
             return fieldMapper;
         }

src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java

@@ -77,7 +77,7 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
         public DoubleFieldMapper build(BuilderContext context) {
             DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context),
                     precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
-                    ignoreMalformed);
+                    ignoreMalformed(context));
             fieldMapper.includeInAll(includeInAll);
             return fieldMapper;
         }

src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java

@@ -78,7 +78,7 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
         public FloatFieldMapper build(BuilderContext context) {
             FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context),
                     precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
-                    ignoreMalformed);
+                    ignoreMalformed(context));
             fieldMapper.includeInAll(includeInAll);
             return fieldMapper;
         }

src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java

@@ -78,7 +78,7 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
         public IntegerFieldMapper build(BuilderContext context) {
             IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context),
                     precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions,
-                    nullValue, ignoreMalformed);
+                    nullValue, ignoreMalformed(context));
             fieldMapper.includeInAll(includeInAll);
             return fieldMapper;
         }

src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java

@@ -78,7 +78,7 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
         public LongFieldMapper build(BuilderContext context) {
             LongFieldMapper fieldMapper = new LongFieldMapper(buildNames(context),
                     precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
-                    ignoreMalformed);
+                    ignoreMalformed(context));
             fieldMapper.includeInAll(includeInAll);
             return fieldMapper;
         }

src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java

@@ -60,7 +60,7 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
 
         protected String fuzzyFactor = Defaults.FUZZY_FACTOR;
 
-        protected boolean ignoreMalformed = Defaults.IGNORE_MALFORMED;
+        private Boolean ignoreMalformed;
 
         public Builder(String name) {
             super(name);
@@ -104,6 +104,15 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
             return builder;
         }
 
+        protected boolean ignoreMalformed(BuilderContext context) {
+            if (ignoreMalformed != null) {
+                return ignoreMalformed;
+            }
+            if (context.indexSettings() != null) {
+                return context.indexSettings().getAsBoolean("index.mapping.ignore_malformed", Defaults.IGNORE_MALFORMED);
+            }
+            return Defaults.IGNORE_MALFORMED;
+        }
     }
 
     protected int precisionStep;

src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java

@@ -78,7 +78,7 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
         public ShortFieldMapper build(BuilderContext context) {
             ShortFieldMapper fieldMapper = new ShortFieldMapper(buildNames(context),
                     precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
-                    ignoreMalformed);
+                    ignoreMalformed(context));
             fieldMapper.includeInAll(includeInAll);
             return fieldMapper;
         }

src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java

@@ -75,7 +75,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
 
         @Override
         public TTLFieldMapper build(BuilderContext context) {
-            return new TTLFieldMapper(store, index, enabled, defaultTTL);
+            return new TTLFieldMapper(store, index, enabled, defaultTTL, ignoreMalformed(context));
         }
     }
 
@@ -104,13 +104,13 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
     private long defaultTTL;
 
     public TTLFieldMapper() {
-        this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.DEFAULT);
+        this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.DEFAULT, Defaults.IGNORE_MALFORMED);
     }
 
-    protected TTLFieldMapper(Field.Store store, Field.Index index, boolean enabled, long defaultTTL) {
+    protected TTLFieldMapper(Field.Store store, Field.Index index, boolean enabled, long defaultTTL, boolean ignoreMalformed) {
         super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP,
                 Defaults.FUZZY_FACTOR, index, store, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.INDEX_OPTIONS,
-                Defaults.NULL_VALUE, Defaults.IGNORE_MALFORMED);
+                Defaults.NULL_VALUE, ignoreMalformed);
         this.enabled = enabled;
         this.defaultTTL = defaultTTL;
     }

src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java

@@ -89,7 +89,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
             if (context.indexSettings() != null) {
                 parseUpperInclusive = context.indexSettings().getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.PARSE_UPPER_INCLUSIVE);
             }
-            return new TimestampFieldMapper(store, index, enabled, path, dateTimeFormatter, parseUpperInclusive);
+            return new TimestampFieldMapper(store, index, enabled, path, dateTimeFormatter, parseUpperInclusive, ignoreMalformed(context));
         }
     }
 
@@ -119,14 +119,15 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
     private final String path;
 
     public TimestampFieldMapper() {
-        this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.PARSE_UPPER_INCLUSIVE);
+        this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.PARSE_UPPER_INCLUSIVE, Defaults.IGNORE_MALFORMED);
     }
 
-    protected TimestampFieldMapper(Field.Store store, Field.Index index, boolean enabled, String path, FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive) {
+    protected TimestampFieldMapper(Field.Store store, Field.Index index, boolean enabled, String path,
+                                   FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive, boolean ignoreMalformed) {
         super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
                 Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, index, store, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.INDEX_OPTIONS,
                 Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/,
-                parseUpperInclusive, Defaults.IGNORE_MALFORMED);
+                parseUpperInclusive, ignoreMalformed);
         this.enabled = enabled;
         this.path = path;
     }

src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java

@@ -105,7 +105,7 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
         @Override
         public IpFieldMapper build(BuilderContext context) {
             IpFieldMapper fieldMapper = new IpFieldMapper(buildNames(context),
-                    precisionStep, index, store, boost, omitNorms, indexOptions, nullValue);
+                    precisionStep, index, store, boost, omitNorms, indexOptions, nullValue, ignoreMalformed(context));
             fieldMapper.includeInAll(includeInAll);
             return fieldMapper;
         }
@@ -132,9 +132,9 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
     protected IpFieldMapper(Names names, int precisionStep,
                             Field.Index index, Field.Store store,
                             float boost, boolean omitNorms, IndexOptions indexOptions,
-                            String nullValue) {
+                            String nullValue, boolean ignoreMalformed) {
         super(names, precisionStep, null, index, store, boost, omitNorms, indexOptions,
-                false, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)),
+                ignoreMalformed, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)),
                 new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)));
         this.nullValue = nullValue;
     }

src/test/java/org/elasticsearch/test/unit/index/mapper/MapperTests.java

@@ -22,6 +22,7 @@ package org.elasticsearch.test.unit.index.mapper;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.EnvironmentModule;
@@ -44,6 +45,10 @@ public class MapperTests {
         return new DocumentMapperParser(new Index("test"), newAnalysisService());
     }
 
+    public static DocumentMapperParser newParser(Settings indexSettings) {
+        return new DocumentMapperParser(new Index("test"), indexSettings, newAnalysisService());
+    }
+
     public static MapperService newMapperService() {
         return new MapperService(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS, new Environment(), newAnalysisService());
     }

src/test/java/org/elasticsearch/test/unit/index/mapper/date/SimpleDateMappingTests.java

@@ -19,14 +19,17 @@
 
 package org.elasticsearch.test.unit.index.mapper.date;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.core.DateFieldMapper;
 import org.elasticsearch.test.unit.index.mapper.MapperTests;
 import org.testng.annotations.Test;
 
+import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.*;
 
@@ -97,4 +100,68 @@ public class SimpleDateMappingTests {
         assertThat(doc.rootDoc().get("date_field"), nullValue());
         assertThat(doc.rootDoc().get("date_field_x"), equalTo("2010-01-01"));
     }
+
+    @Test
+    public void testIgnoreMalformedOption() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties")
+                .startObject("field1").field("type", "date").field("ignore_malformed", true).endObject()
+                .startObject("field2").field("type", "date").field("ignore_malformed", false).endObject()
+                .startObject("field3").field("type", "date").endObject()
+                .endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper defaultMapper = MapperTests.newParser().parse(mapping);
+
+        ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field1", "a")
+                .field("field2", "2010-01-01")
+                .endObject()
+                .bytes());
+        assertThat(doc.rootDoc().getFieldable("field1"), nullValue());
+        assertThat(doc.rootDoc().getFieldable("field2"), notNullValue());
+
+        try {
+            defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                    .startObject()
+                    .field("field2", "a")
+                    .endObject()
+                    .bytes());
+        } catch (MapperParsingException e) {
+            assertThat(e.getCause(), instanceOf(MapperParsingException.class));
+        }
+
+        // Verify that the default is false
+        try {
+            defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                    .startObject()
+                    .field("field3", "a")
+                    .endObject()
+                    .bytes());
+        } catch (MapperParsingException e) {
+            assertThat(e.getCause(), instanceOf(MapperParsingException.class));
+        }
+
+        // Unless the global ignore_malformed option is set to true
+        Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build();
+        defaultMapper = MapperTests.newParser(indexSettings).parse(mapping);
+        doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field3", "a")
+                .endObject()
+                .bytes());
+        assertThat(doc.rootDoc().getFieldable("field3"), nullValue());
+
+        // This should still throw an exception, since field2 is specifically set to ignore_malformed=false
+        try {
+            defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                    .startObject()
+                    .field("field2", "a")
+                    .endObject()
+                    .bytes());
+        } catch (MapperParsingException e) {
+            assertThat(e.getCause(), instanceOf(MapperParsingException.class));
+        }
+    }
 }

src/test/java/org/elasticsearch/test/unit/index/mapper/numeric/SimpleNumericTests.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.test.unit.index.mapper.numeric;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -30,6 +31,7 @@ import org.elasticsearch.index.mapper.core.StringFieldMapper;
 import org.elasticsearch.test.unit.index.mapper.MapperTests;
 import org.testng.annotations.Test;
 
+import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.*;
 
@@ -79,7 +81,7 @@ public class SimpleNumericTests {
         assertThat(mapper, instanceOf(StringFieldMapper.class));
     }
 
-    public void testIgnoreMalformedEnabled() throws Exception {
+    public void testIgnoreMalformedOption() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties")
                     .startObject("field1").field("type", "integer").field("ignore_malformed", true).endObject()
@@ -119,6 +121,27 @@ public class SimpleNumericTests {
         } catch (MapperParsingException e) {
             assertThat(e.getCause(), instanceOf(NumberFormatException.class));
         }
+
+        // Unless the global ignore_malformed option is set to true
+        Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build();
+        defaultMapper = MapperTests.newParser(indexSettings).parse(mapping);
+        doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field3", "a")
+                .endObject()
+                .bytes());
+        assertThat(doc.rootDoc().getFieldable("field3"), nullValue());
+
+        // This should still throw an exception, since field2 is specifically set to ignore_malformed=false
+        try {
+            defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                    .startObject()
+                    .field("field2", "a")
+                    .endObject()
+                    .bytes());
+        } catch (MapperParsingException e) {
+            assertThat(e.getCause(), instanceOf(NumberFormatException.class));
+        }
     }
 
 }