honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added a global ignore_malformed index setting. #2220 Also extended the ignore_malformed support to TTL, Ip and timestamp field types.

This commit is contained in:
Martijn van Groningen 2012-08-31 16:13:43 +02:00 committed by Shay Banon
parent 888b7cc48f
commit cd0e1226e1
14 changed files with 125 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
View File

@ -76,7 +76,7 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
@Override
public ByteFieldMapper build(BuilderContext context) {
ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, ignoreMalformed);
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue, ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}

2
src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -105,7 +105,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
}
DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter,
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
timeUnit, parseUpperInclusive, ignoreMalformed);
timeUnit, parseUpperInclusive, ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}

2
src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
View File

@ -77,7 +77,7 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
public DoubleFieldMapper build(BuilderContext context) {
DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
ignoreMalformed);
ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}

2
src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
View File

@ -78,7 +78,7 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
public FloatFieldMapper build(BuilderContext context) {
FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
ignoreMalformed);
ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}

2
src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
View File

@ -78,7 +78,7 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
public IntegerFieldMapper build(BuilderContext context) {
IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions,
nullValue, ignoreMalformed);
nullValue, ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}

2
src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
View File

@ -78,7 +78,7 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
public LongFieldMapper build(BuilderContext context) {
LongFieldMapper fieldMapper = new LongFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
ignoreMalformed);
ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}

11
src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
View File

@ -60,7 +60,7 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
protected String fuzzyFactor = Defaults.FUZZY_FACTOR;
protected boolean ignoreMalformed = Defaults.IGNORE_MALFORMED;
private Boolean ignoreMalformed;
public Builder(String name) {
super(name);
@ -104,6 +104,15 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
return builder;
}
protected boolean ignoreMalformed(BuilderContext context) {
if (ignoreMalformed != null) {
return ignoreMalformed;
}
if (context.indexSettings() != null) {
return context.indexSettings().getAsBoolean("index.mapping.ignore_malformed", Defaults.IGNORE_MALFORMED);
}
return Defaults.IGNORE_MALFORMED;
}
}
protected int precisionStep;

2
src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
View File

@ -78,7 +78,7 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
public ShortFieldMapper build(BuilderContext context) {
ShortFieldMapper fieldMapper = new ShortFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, index, store, boost, omitNorms, indexOptions, nullValue,
ignoreMalformed);
ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}

8
src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
View File

@ -75,7 +75,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
@Override
public TTLFieldMapper build(BuilderContext context) {
return new TTLFieldMapper(store, index, enabled, defaultTTL);
return new TTLFieldMapper(store, index, enabled, defaultTTL, ignoreMalformed(context));
}
}
@ -104,13 +104,13 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
private long defaultTTL;
public TTLFieldMapper() {
this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.DEFAULT);
this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.DEFAULT, Defaults.IGNORE_MALFORMED);
}
protected TTLFieldMapper(Field.Store store, Field.Index index, boolean enabled, long defaultTTL) {
protected TTLFieldMapper(Field.Store store, Field.Index index, boolean enabled, long defaultTTL, boolean ignoreMalformed) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP,
Defaults.FUZZY_FACTOR, index, store, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.INDEX_OPTIONS,
Defaults.NULL_VALUE, Defaults.IGNORE_MALFORMED);
Defaults.NULL_VALUE, ignoreMalformed);
this.enabled = enabled;
this.defaultTTL = defaultTTL;
}

9
src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View File

@ -89,7 +89,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
if (context.indexSettings() != null) {
parseUpperInclusive = context.indexSettings().getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.PARSE_UPPER_INCLUSIVE);
}
return new TimestampFieldMapper(store, index, enabled, path, dateTimeFormatter, parseUpperInclusive);
return new TimestampFieldMapper(store, index, enabled, path, dateTimeFormatter, parseUpperInclusive, ignoreMalformed(context));
}
}
@ -119,14 +119,15 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
private final String path;
public TimestampFieldMapper() {
this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.PARSE_UPPER_INCLUSIVE);
this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.PARSE_UPPER_INCLUSIVE, Defaults.IGNORE_MALFORMED);
}
protected TimestampFieldMapper(Field.Store store, Field.Index index, boolean enabled, String path, FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive) {
protected TimestampFieldMapper(Field.Store store, Field.Index index, boolean enabled, String path,
FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive, boolean ignoreMalformed) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, index, store, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.INDEX_OPTIONS,
Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/,
parseUpperInclusive, Defaults.IGNORE_MALFORMED);
parseUpperInclusive, ignoreMalformed);
this.enabled = enabled;
this.path = path;
}

6
src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
View File

@ -105,7 +105,7 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
@Override
public IpFieldMapper build(BuilderContext context) {
IpFieldMapper fieldMapper = new IpFieldMapper(buildNames(context),
precisionStep, index, store, boost, omitNorms, indexOptions, nullValue);
precisionStep, index, store, boost, omitNorms, indexOptions, nullValue, ignoreMalformed(context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -132,9 +132,9 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
protected IpFieldMapper(Names names, int precisionStep,
Field.Index index, Field.Store store,
float boost, boolean omitNorms, IndexOptions indexOptions,
String nullValue) {
String nullValue, boolean ignoreMalformed) {
super(names, precisionStep, null, index, store, boost, omitNorms, indexOptions,
false, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)),
ignoreMalformed, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)),
new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)));
this.nullValue = nullValue;
}

5
src/test/java/org/elasticsearch/test/unit/index/mapper/MapperTests.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.test.unit.index.mapper;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.ModulesBuilder;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsModule;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.EnvironmentModule;
@ -44,6 +45,10 @@ public class MapperTests {
return new DocumentMapperParser(new Index("test"), newAnalysisService());
}
public static DocumentMapperParser newParser(Settings indexSettings) {
return new DocumentMapperParser(new Index("test"), indexSettings, newAnalysisService());
}
public static MapperService newMapperService() {
return new MapperService(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS, new Environment(), newAnalysisService());
}

67
src/test/java/org/elasticsearch/test/unit/index/mapper/date/SimpleDateMappingTests.java
View File

@ -19,14 +19,17 @@
package org.elasticsearch.test.unit.index.mapper.date;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.core.DateFieldMapper;
import org.elasticsearch.test.unit.index.mapper.MapperTests;
import org.testng.annotations.Test;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
@ -97,4 +100,68 @@ public class SimpleDateMappingTests {
assertThat(doc.rootDoc().get("date_field"), nullValue());
assertThat(doc.rootDoc().get("date_field_x"), equalTo("2010-01-01"));
}
@Test
public void testIgnoreMalformedOption() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("field1").field("type", "date").field("ignore_malformed", true).endObject()
.startObject("field2").field("type", "date").field("ignore_malformed", false).endObject()
.startObject("field3").field("type", "date").endObject()
.endObject()
.endObject().endObject().string();
DocumentMapper defaultMapper = MapperTests.newParser().parse(mapping);
ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field1", "a")
.field("field2", "2010-01-01")
.endObject()
.bytes());
assertThat(doc.rootDoc().getFieldable("field1"), nullValue());
assertThat(doc.rootDoc().getFieldable("field2"), notNullValue());
try {
defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field2", "a")
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(MapperParsingException.class));
}
// Verify that the default is false
try {
defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field3", "a")
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(MapperParsingException.class));
}
// Unless the global ignore_malformed option is set to true
Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build();
defaultMapper = MapperTests.newParser(indexSettings).parse(mapping);
doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field3", "a")
.endObject()
.bytes());
assertThat(doc.rootDoc().getFieldable("field3"), nullValue());
// This should still throw an exception, since field2 is specifically set to ignore_malformed=false
try {
defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field2", "a")
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(MapperParsingException.class));
}
}
}

25
src/test/java/org/elasticsearch/test/unit/index/mapper/numeric/SimpleNumericTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.test.unit.index.mapper.numeric;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
@ -30,6 +31,7 @@ import org.elasticsearch.index.mapper.core.StringFieldMapper;
import org.elasticsearch.test.unit.index.mapper.MapperTests;
import org.testng.annotations.Test;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
@ -79,7 +81,7 @@ public class SimpleNumericTests {
assertThat(mapper, instanceOf(StringFieldMapper.class));
}
public void testIgnoreMalformedEnabled() throws Exception {
public void testIgnoreMalformedOption() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("field1").field("type", "integer").field("ignore_malformed", true).endObject()
@ -119,6 +121,27 @@ public class SimpleNumericTests {
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
// Unless the global ignore_malformed option is set to true
Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build();
defaultMapper = MapperTests.newParser(indexSettings).parse(mapping);
doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field3", "a")
.endObject()
.bytes());
assertThat(doc.rootDoc().getFieldable("field3"), nullValue());
// This should still throw an exception, since field2 is specifically set to ignore_malformed=false
try {
defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field2", "a")
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
}
}