From 541059a4d13042819fcdf3e563b2ea60ba0bb604 Mon Sep 17 00:00:00 2001 From: markharwood Date: Fri, 10 Jan 2014 13:50:58 +0000 Subject: [PATCH] Adds a new coerce flag for numeric field mappings which is defaulted to true. When set to false a new strict mode of parsing is employed which a) does not permit numbers to be passed as JSON strings in quotes b) rejects numbers with fractions that are passed to integer, short or long fields. Closes #4117 --- docs/reference/mapping.asciidoc | 10 +- .../mapping/types/core-types.asciidoc | 2 + .../org/elasticsearch/common/Explicit.java | 18 +++- .../common/xcontent/XContentParser.java | 10 ++ .../support/AbstractXContentParser.java | 77 ++++++++++++++- .../index/mapper/core/ByteFieldMapper.java | 15 +-- .../index/mapper/core/DateFieldMapper.java | 11 ++- .../index/mapper/core/DoubleFieldMapper.java | 13 +-- .../index/mapper/core/FloatFieldMapper.java | 12 +-- .../index/mapper/core/IntegerFieldMapper.java | 12 +-- .../index/mapper/core/LongFieldMapper.java | 10 +- .../index/mapper/core/NumberFieldMapper.java | 30 +++++- .../index/mapper/core/ShortFieldMapper.java | 10 +- .../mapper/core/TokenCountFieldMapper.java | 9 +- .../index/mapper/core/TypeParsers.java | 2 + .../mapper/internal/BoostFieldMapper.java | 4 +- .../mapper/internal/SizeFieldMapper.java | 2 +- .../index/mapper/internal/TTLFieldMapper.java | 12 +-- .../mapper/internal/TimestampFieldMapper.java | 8 +- .../index/mapper/ip/IpFieldMapper.java | 7 +- .../mapper/numeric/SimpleNumericTests.java | 99 +++++++++++++++++++ 21 files changed, 307 insertions(+), 66 deletions(-) diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc index ec362fc5771..3c5a225cc05 100644 --- a/docs/reference/mapping.asciidoc +++ b/docs/reference/mapping.asciidoc @@ -51,8 +51,16 @@ index>>. The `index.mapping.ignore_malformed` global setting can be set on the index level to allow to ignore malformed content globally across all -mapping types (malformed content example is trying to index a string +mapping types (malformed content example is trying to index a text string value as a numeric type). + +The `index.mapping.coerce` global setting can be set on the +index level to coerce numeric content globally across all +mapping types (The default setting is true and coercions attempted are +to convert strings with numbers into numeric types and also numeric values +with fractions to any integer/short/long values minus the fraction part). +When the permitted conversions fail in their attempts, the value is considered +malformed and the ignore_malformed setting dictates what will happen next. -- include::mapping/fields.asciidoc[] diff --git a/docs/reference/mapping/types/core-types.asciidoc b/docs/reference/mapping/types/core-types.asciidoc index e0d9aed65d9..e3e7d03f27e 100644 --- a/docs/reference/mapping/types/core-types.asciidoc +++ b/docs/reference/mapping/types/core-types.asciidoc @@ -230,6 +230,8 @@ defaults to `true` or to the parent `object` type setting. |`ignore_malformed` |Ignored a malformed number. Defaults to `false`. +|`coerce` |Try convert strings to numbers and truncate fractions for integers. Defaults to `true`. + |======================================================================= [float] diff --git a/src/main/java/org/elasticsearch/common/Explicit.java b/src/main/java/org/elasticsearch/common/Explicit.java index d2aa495e2b1..3132f22daa0 100644 --- a/src/main/java/org/elasticsearch/common/Explicit.java +++ b/src/main/java/org/elasticsearch/common/Explicit.java @@ -20,12 +20,24 @@ package org.elasticsearch.common; /** + * Holds a value that is either: + * a) set implicitly e.g. through some default value + * b) set explicitly e.g. from a user selection + * + * When merging conflicting configuration settings such as + * field mapping settings it is preferable to preserve an explicit + * choice rather than a choice made only made implicitly by defaults. + * */ public class Explicit { private final T value; private final boolean explicit; - + /** + * Create a value with an indication if this was an explicit choice + * @param value a setting value + * @param explicit true if the value passed is a conscious decision, false if using some kind of default + */ public Explicit(T value, boolean explicit) { this.value = value; this.explicit = explicit; @@ -35,6 +47,10 @@ public class Explicit { return this.value; } + /** + * + * @return true if the value passed is a conscious decision, false if using some kind of default + */ public boolean explicit() { return this.explicit; } diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java index 486f80ee6c0..497dbe7503f 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java @@ -157,6 +157,16 @@ public interface XContentParser extends Closeable { */ boolean estimatedNumberType(); + short shortValue(boolean coerce) throws IOException; + + int intValue(boolean coerce) throws IOException; + + long longValue(boolean coerce) throws IOException; + + float floatValue(boolean coerce) throws IOException; + + double doubleValue(boolean coerce) throws IOException; + short shortValue() throws IOException; int intValue() throws IOException; diff --git a/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java index 679809723af..7d55b6a3f31 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java +++ b/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java @@ -31,6 +31,39 @@ import java.util.*; */ public abstract class AbstractXContentParser implements XContentParser { + //Currently this is not a setting that can be changed and is a policy + // that relates to how parsing of things like "boost" are done across + // the whole of Elasticsearch (eg if String "1.0" is a valid float). + // The idea behind keeping it as a constant is that we can track + // references to this policy decision throughout the codebase and find + // and change any code that needs to apply an alternative policy. + public static final boolean DEFAULT_NUMBER_COEERCE_POLICY = true; + + private static void checkCoerceString(boolean coeerce, Class clazz) { + if (!coeerce) { + //Need to throw type IllegalArgumentException as current catch logic in + //NumberFieldMapper.parseCreateField relies on this for "malformed" value detection + throw new IllegalArgumentException(clazz.getSimpleName() + " value passed as String"); + } + } + + + + // The 3rd party parsers we rely on are known to silently truncate fractions: see + // http://fasterxml.github.io/jackson-core/javadoc/2.3.0/com/fasterxml/jackson/core/JsonParser.html#getShortValue() + // If this behaviour is flagged as undesirable and any truncation occurs + // then this method is called to trigger the"malformed" handling logic + void ensureNumberConversion(boolean coerce, long result, Class clazz) throws IOException { + if (!coerce) { + double fullVal = doDoubleValue(); + if (result != fullVal) { + // Need to throw type IllegalArgumentException as current catch + // logic in NumberFieldMapper.parseCreateField relies on this + // for "malformed" value detection + throw new IllegalArgumentException(fullVal + " cannot be converted to " + clazz.getSimpleName() + " without data loss"); + } + } + } @Override public boolean isBooleanValue() throws IOException { @@ -62,41 +95,72 @@ public abstract class AbstractXContentParser implements XContentParser { @Override public short shortValue() throws IOException { + return shortValue(DEFAULT_NUMBER_COEERCE_POLICY); + } + + @Override + public short shortValue(boolean coerce) throws IOException { Token token = currentToken(); if (token == Token.VALUE_STRING) { + checkCoerceString(coerce, Short.class); return Short.parseShort(text()); } - return doShortValue(); + short result = doShortValue(); + ensureNumberConversion(coerce, result, Short.class); + return result; } protected abstract short doShortValue() throws IOException; @Override public int intValue() throws IOException { + return intValue(DEFAULT_NUMBER_COEERCE_POLICY); + } + + + @Override + public int intValue(boolean coerce) throws IOException { Token token = currentToken(); if (token == Token.VALUE_STRING) { + checkCoerceString(coerce, Integer.class); return Integer.parseInt(text()); } - return doIntValue(); + int result = doIntValue(); + ensureNumberConversion(coerce, result, Integer.class); + return result; } protected abstract int doIntValue() throws IOException; @Override public long longValue() throws IOException { + return longValue(DEFAULT_NUMBER_COEERCE_POLICY); + } + + @Override + public long longValue(boolean coerce) throws IOException { Token token = currentToken(); if (token == Token.VALUE_STRING) { + checkCoerceString(coerce, Long.class); return Long.parseLong(text()); } - return doLongValue(); + long result = doLongValue(); + ensureNumberConversion(coerce, result, Long.class); + return result; } protected abstract long doLongValue() throws IOException; @Override public float floatValue() throws IOException { + return floatValue(DEFAULT_NUMBER_COEERCE_POLICY); + } + + @Override + public float floatValue(boolean coerce) throws IOException { Token token = currentToken(); if (token == Token.VALUE_STRING) { + checkCoerceString(coerce, Float.class); return Float.parseFloat(text()); } return doFloatValue(); @@ -104,10 +168,17 @@ public abstract class AbstractXContentParser implements XContentParser { protected abstract float doFloatValue() throws IOException; + @Override public double doubleValue() throws IOException { + return doubleValue(DEFAULT_NUMBER_COEERCE_POLICY); + } + + @Override + public double doubleValue(boolean coerce) throws IOException { Token token = currentToken(); if (token == Token.VALUE_STRING) { + checkCoerceString(coerce, Double.class); return Double.parseDouble(text()); } return doDoubleValue(); diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java index d42afd7c995..7b975b1941a 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java @@ -92,8 +92,8 @@ public class ByteFieldMapper extends NumberFieldMapper { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), - postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, - context.indexSettings(), multiFieldsBuilder.build(this, context)); + coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, + fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } @@ -120,11 +120,12 @@ public class ByteFieldMapper extends NumberFieldMapper { private String nullValueAsString; protected ByteFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Byte nullValue, Explicit ignoreMalformed, PostingsFormatProvider postingsProvider, + Byte nullValue, Explicit ignoreMalformed, Explicit coerce, + PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { super(names, precisionStep, boost, fieldType, docValues, - ignoreMalformed, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), + ignoreMalformed, coerce, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; @@ -293,7 +294,7 @@ public class ByteFieldMapper extends NumberFieldMapper { } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { - objValue = (byte) parser.shortValue(); + objValue = (byte) parser.shortValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); @@ -308,7 +309,7 @@ public class ByteFieldMapper extends NumberFieldMapper { } value = objValue; } else { - value = (byte) parser.shortValue(); + value = (byte) parser.shortValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } @@ -383,4 +384,4 @@ public class ByteFieldMapper extends NumberFieldMapper { return Byte.toString(number); } } -} \ No newline at end of file +} diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 1a8b8748922..375d21031c2 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -128,7 +128,7 @@ public class DateFieldMapper extends NumberFieldMapper { dateTimeFormatter = new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale); } DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter, - precisionStep, boost, fieldType, docValues, nullValue, timeUnit, roundCeil, ignoreMalformed(context), + precisionStep, boost, fieldType, docValues, nullValue, timeUnit, roundCeil, ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); @@ -202,10 +202,11 @@ public class DateFieldMapper extends NumberFieldMapper { protected final TimeUnit timeUnit; protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - String nullValue, TimeUnit timeUnit, boolean roundCeil, Explicit ignoreMalformed, + String nullValue, TimeUnit timeUnit, boolean roundCeil, Explicit ignoreMalformed,Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, + Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, new NamedAnalyzer("_date/" + precisionStep, + super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, new NamedAnalyzer("_date/" + precisionStep, new NumericDateAnalyzer(precisionStep, dateTimeFormatter.parser())), new NamedAnalyzer("_date/max", new NumericDateAnalyzer(Integer.MAX_VALUE, dateTimeFormatter.parser())), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); @@ -393,7 +394,7 @@ public class DateFieldMapper extends NumberFieldMapper { if (token == XContentParser.Token.VALUE_NULL) { dateAsString = nullValue; } else if (token == XContentParser.Token.VALUE_NUMBER) { - value = parser.longValue(); + value = parser.longValue(coerce.value()); } else if (token == XContentParser.Token.START_OBJECT) { String currentFieldName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -404,7 +405,7 @@ public class DateFieldMapper extends NumberFieldMapper { if (token == XContentParser.Token.VALUE_NULL) { dateAsString = nullValue; } else if (token == XContentParser.Token.VALUE_NUMBER) { - value = parser.longValue(); + value = parser.longValue(coerce.value()); } else { dateAsString = parser.text(); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index 062e1f2d050..b3f9027e420 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -95,8 +95,8 @@ public class DoubleFieldMapper extends NumberFieldMapper { public DoubleFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context), - precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), postingsProvider, docValuesProvider, - similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); + precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), coerce(context), postingsProvider, + docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } @@ -124,11 +124,12 @@ public class DoubleFieldMapper extends NumberFieldMapper { private String nullValueAsString; protected DoubleFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Double nullValue, Explicit ignoreMalformed, + Double nullValue, Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, + SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, + super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, NumericDoubleAnalyzer.buildNamedAnalyzer(precisionStep), NumericDoubleAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; @@ -288,7 +289,7 @@ public class DoubleFieldMapper extends NumberFieldMapper { } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { - objValue = parser.doubleValue(); + objValue = parser.doubleValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); @@ -303,7 +304,7 @@ public class DoubleFieldMapper extends NumberFieldMapper { } value = objValue; } else { - value = parser.doubleValue(); + value = parser.doubleValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java index c2110a49cc7..eb313ee306d 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java @@ -96,8 +96,8 @@ public class FloatFieldMapper extends NumberFieldMapper { public FloatFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context), - precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), postingsProvider, docValuesProvider, - similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); + precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), coerce(context), postingsProvider, + docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } @@ -124,11 +124,11 @@ public class FloatFieldMapper extends NumberFieldMapper { private String nullValueAsString; protected FloatFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Float nullValue, Explicit ignoreMalformed, + Float nullValue, Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, + super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, NumericFloatAnalyzer.buildNamedAnalyzer(precisionStep), NumericFloatAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; @@ -294,7 +294,7 @@ public class FloatFieldMapper extends NumberFieldMapper { } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { - objValue = parser.floatValue(); + objValue = parser.floatValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); @@ -309,7 +309,7 @@ public class FloatFieldMapper extends NumberFieldMapper { } value = objValue; } else { - value = parser.floatValue(); + value = parser.floatValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java index cb1c010e510..485f1097c49 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java @@ -92,8 +92,8 @@ public class IntegerFieldMapper extends NumberFieldMapper { public IntegerFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, - nullValue, ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, normsLoading, - fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); + nullValue, ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, + context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } @@ -120,11 +120,11 @@ public class IntegerFieldMapper extends NumberFieldMapper { private String nullValueAsString; protected IntegerFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Integer nullValue, Explicit ignoreMalformed, + Integer nullValue, Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, + super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, NumericIntegerAnalyzer.buildNamedAnalyzer(precisionStep), NumericIntegerAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; @@ -289,7 +289,7 @@ public class IntegerFieldMapper extends NumberFieldMapper { } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { - objValue = parser.intValue(); + objValue = parser.intValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); @@ -304,7 +304,7 @@ public class IntegerFieldMapper extends NumberFieldMapper { } value = objValue; } else { - value = parser.intValue(); + value = parser.intValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java index 38b38ceacfe..f1acae71553 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java @@ -92,7 +92,7 @@ public class LongFieldMapper extends NumberFieldMapper { public LongFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); LongFieldMapper fieldMapper = new LongFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, nullValue, - ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, normsLoading, + ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -120,11 +120,11 @@ public class LongFieldMapper extends NumberFieldMapper { private String nullValueAsString; protected LongFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Long nullValue, Explicit ignoreMalformed, + Long nullValue, Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, + super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, NumericLongAnalyzer.buildNamedAnalyzer(precisionStep), NumericLongAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; @@ -279,7 +279,7 @@ public class LongFieldMapper extends NumberFieldMapper { } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { - objValue = parser.longValue(); + objValue = parser.longValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); @@ -294,7 +294,7 @@ public class LongFieldMapper extends NumberFieldMapper { } value = objValue; } else { - value = parser.longValue(); + value = parser.longValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java index de56c60dc41..d3d2c0d8062 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java @@ -77,6 +77,7 @@ public abstract class NumberFieldMapper extends AbstractFieldM } public static final Explicit IGNORE_MALFORMED = new Explicit(false, false); + public static final Explicit COERCE = new Explicit(true, false); } public abstract static class Builder extends AbstractFieldMapper.Builder { @@ -85,6 +86,8 @@ public abstract class NumberFieldMapper extends AbstractFieldM private Boolean ignoreMalformed; + private Boolean coerce; + public Builder(String name, FieldType fieldType) { super(name, fieldType); } @@ -108,6 +111,22 @@ public abstract class NumberFieldMapper extends AbstractFieldM } return Defaults.IGNORE_MALFORMED; } + + public T coerce(boolean coerce) { + this.coerce = coerce; + return builder; + } + + protected Explicit coerce(BuilderContext context) { + if (coerce != null) { + return new Explicit(coerce, true); + } + if (context.indexSettings() != null) { + return new Explicit(context.indexSettings().getAsBoolean("index.mapping.coerce", Defaults.COERCE.value()), false); + } + return Defaults.COERCE; + } + } protected int precisionStep; @@ -116,6 +135,8 @@ public abstract class NumberFieldMapper extends AbstractFieldM protected Explicit ignoreMalformed; + protected Explicit coerce; + private ThreadLocal tokenStream = new ThreadLocal() { @Override protected NumericTokenStream initialValue() { @@ -145,7 +166,7 @@ public abstract class NumberFieldMapper extends AbstractFieldM }; protected NumberFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Explicit ignoreMalformed, NamedAnalyzer indexAnalyzer, + Explicit ignoreMalformed, Explicit coerce, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, @@ -159,6 +180,7 @@ public abstract class NumberFieldMapper extends AbstractFieldM this.precisionStep = precisionStep; } this.ignoreMalformed = ignoreMalformed; + this.coerce = coerce; } @Override @@ -334,6 +356,9 @@ public abstract class NumberFieldMapper extends AbstractFieldM if (nfmMergeWith.ignoreMalformed.explicit()) { this.ignoreMalformed = nfmMergeWith.ignoreMalformed; } + if (nfmMergeWith.coerce.explicit()) { + this.coerce = nfmMergeWith.coerce; + } } } @@ -477,6 +502,9 @@ public abstract class NumberFieldMapper extends AbstractFieldM if (includeDefaults || ignoreMalformed.explicit()) { builder.field("ignore_malformed", ignoreMalformed.value()); } + if (includeDefaults || coerce.explicit()) { + builder.field("coerce", coerce.value()); + } } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java index 367dffd6548..1d217f801dd 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java @@ -93,7 +93,7 @@ public class ShortFieldMapper extends NumberFieldMapper { public ShortFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); ShortFieldMapper fieldMapper = new ShortFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, nullValue, - ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, + ignoreMalformed(context), coerce(context),postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -121,11 +121,11 @@ public class ShortFieldMapper extends NumberFieldMapper { private String nullValueAsString; protected ShortFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Short nullValue, Explicit ignoreMalformed, + Short nullValue, Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, new NamedAnalyzer("_short/" + precisionStep, + super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, new NamedAnalyzer("_short/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), new NamedAnalyzer("_short/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; @@ -294,7 +294,7 @@ public class ShortFieldMapper extends NumberFieldMapper { } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { - objValue = parser.shortValue(); + objValue = parser.shortValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); @@ -309,7 +309,7 @@ public class ShortFieldMapper extends NumberFieldMapper { } value = objValue; } else { - value = parser.shortValue(); + value = parser.shortValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java index 20868c9df64..4544ee917f0 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java @@ -79,7 +79,7 @@ public class TokenCountFieldMapper extends IntegerFieldMapper { public TokenCountFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); TokenCountFieldMapper fieldMapper = new TokenCountFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, nullValue, - ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), + ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), analyzer, multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -115,11 +115,12 @@ public class TokenCountFieldMapper extends IntegerFieldMapper { private NamedAnalyzer analyzer; protected TokenCountFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, Integer nullValue, - Explicit ignoreMalformed, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, + Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, Settings fieldDataSettings, Settings indexSettings, NamedAnalyzer analyzer, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed, postingsProvider, docValuesProvider, similarity, - normsLoading, fieldDataSettings, indexSettings, multiFields); + super(names, precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed, coerce, postingsProvider, docValuesProvider, + similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); + this.analyzer = analyzer; } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java index 56f9243b4c8..c142acae2a6 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java @@ -146,6 +146,8 @@ public class TypeParsers { builder.precisionStep(nodeIntegerValue(propNode)); } else if (propName.equals("ignore_malformed")) { builder.ignoreMalformed(nodeBooleanValue(propNode)); + } else if (propName.equals("coerce")) { + builder.coerce(nodeBooleanValue(propNode)); } else if (propName.equals("omit_norms")) { builder.omitNorms(nodeBooleanValue(propNode)); } else if (propName.equals("similarity")) { diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java index 77e12f64f78..c38a3ea113e 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java @@ -125,7 +125,7 @@ public class BoostFieldMapper extends NumberFieldMapper implements Intern protected BoostFieldMapper(String name, String indexName, int precisionStep, float boost, FieldType fieldType, Boolean docValues, Float nullValue, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, @Nullable Settings fieldDataSettings, Settings indexSettings) { - super(new Names(name, indexName, indexName, name), precisionStep, boost, fieldType, docValues, Defaults.IGNORE_MALFORMED, + super(new Names(name, indexName, indexName, name), precisionStep, boost, fieldType, docValues, Defaults.IGNORE_MALFORMED, Defaults.COERCE, NumericFloatAnalyzer.buildNamedAnalyzer(precisionStep), NumericFloatAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE), postingsProvider, docValuesProvider, null, null, fieldDataSettings, indexSettings, MultiFields.empty()); this.nullValue = nullValue; @@ -273,7 +273,7 @@ public class BoostFieldMapper extends NumberFieldMapper implements Intern } value = nullValue; } else { - value = context.parser().floatValue(); + value = context.parser().floatValue(coerce.value()); } return value; } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java index cd619405de4..4333032686b 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java @@ -102,7 +102,7 @@ public class SizeFieldMapper extends IntegerFieldMapper implements RootMapper { public SizeFieldMapper(EnabledAttributeMapper enabled, FieldType fieldType, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, @Nullable Settings fieldDataSettings, Settings indexSettings) { super(new Names(Defaults.NAME), Defaults.PRECISION_STEP, Defaults.BOOST, fieldType, null, Defaults.NULL_VALUE, - Defaults.IGNORE_MALFORMED, postingsProvider, docValuesProvider, null, null, fieldDataSettings, + Defaults.IGNORE_MALFORMED, Defaults.COERCE, postingsProvider, docValuesProvider, null, null, fieldDataSettings, indexSettings, MultiFields.empty()); this.enabledState = enabled; } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index 48be044e91a..d518b885238 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -89,7 +89,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R @Override public TTLFieldMapper build(BuilderContext context) { - return new TTLFieldMapper(fieldType, enabledState, defaultTTL, ignoreMalformed(context), postingsProvider, docValuesProvider, fieldDataSettings, context.indexSettings()); + return new TTLFieldMapper(fieldType, enabledState, defaultTTL, ignoreMalformed(context),coerce(context), postingsProvider, docValuesProvider, fieldDataSettings, context.indexSettings()); } } @@ -119,14 +119,14 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R private long defaultTTL; public TTLFieldMapper() { - this(new FieldType(Defaults.TTL_FIELD_TYPE), Defaults.ENABLED_STATE, Defaults.DEFAULT, Defaults.IGNORE_MALFORMED, null, null, null, ImmutableSettings.EMPTY); + this(new FieldType(Defaults.TTL_FIELD_TYPE), Defaults.ENABLED_STATE, Defaults.DEFAULT, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, ImmutableSettings.EMPTY); } protected TTLFieldMapper(FieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL, Explicit ignoreMalformed, - PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, - @Nullable Settings fieldDataSettings, Settings indexSettings) { + Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, + @Nullable Settings fieldDataSettings, Settings indexSettings) { super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP, - Defaults.BOOST, fieldType, null, Defaults.NULL_VALUE, ignoreMalformed, + Defaults.BOOST, fieldType, null, Defaults.NULL_VALUE, ignoreMalformed, coerce, postingsProvider, docValuesProvider, null, null, fieldDataSettings, indexSettings, MultiFields.empty()); this.enabledState = enabled; this.defaultTTL = defaultTTL; @@ -184,7 +184,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R if (context.parser().currentToken() == XContentParser.Token.VALUE_STRING) { ttl = TimeValue.parseTimeValue(context.parser().text(), null).millis(); } else { - ttl = context.parser().longValue(); + ttl = context.parser().longValue(coerce.value()); } if (ttl <= 0) { throw new MapperParsingException("TTL value must be > 0. Illegal value provided [" + ttl + "]"); diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index 51978b59f58..58fe783f5c5 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -105,7 +105,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap roundCeil = settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL)); } return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, roundCeil, - ignoreMalformed(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings()); + ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings()); } } @@ -137,18 +137,18 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap public TimestampFieldMapper() { this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, - Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, null, null, null, null, ImmutableSettings.EMPTY); + Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, ImmutableSettings.EMPTY); } protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path, FormatDateTimeFormatter dateTimeFormatter, boolean roundCeil, - Explicit ignoreMalformed, PostingsFormatProvider postingsProvider, + Explicit ignoreMalformed,Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings) { super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter, Defaults.PRECISION_STEP, Defaults.BOOST, fieldType, docValues, Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/, - roundCeil, ignoreMalformed, postingsProvider, docValuesProvider, null, normsLoading, fieldDataSettings, + roundCeil, ignoreMalformed, coerce, postingsProvider, docValuesProvider, null, normsLoading, fieldDataSettings, indexSettings, MultiFields.empty()); this.enabledState = enabledState; this.path = path; diff --git a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java index db379e45dc2..f9f31688ec3 100644 --- a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java @@ -122,7 +122,8 @@ public class IpFieldMapper extends NumberFieldMapper { public IpFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); IpFieldMapper fieldMapper = new IpFieldMapper(buildNames(context), - precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, + precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), coerce(context), + postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -148,12 +149,12 @@ public class IpFieldMapper extends NumberFieldMapper { private String nullValue; protected IpFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - String nullValue, Explicit ignoreMalformed, + String nullValue, Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { super(names, precisionStep, boost, fieldType, docValues, - ignoreMalformed, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)), + ignoreMalformed, coerce, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)), new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; diff --git a/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java b/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java index e7f29d0feb6..541a05d17f7 100644 --- a/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java @@ -145,6 +145,105 @@ public class SimpleNumericTests extends ElasticsearchTestCase { } } + @Test + public void testCoerceOption() throws Exception { + String [] nonFractionNumericFieldTypes={"integer","long","short"}; + //Test co-ercion policies on all non-fraction numerics + for (String nonFractionNumericFieldType : nonFractionNumericFieldTypes) { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("noErrorNoCoerceField").field("type", nonFractionNumericFieldType).field("ignore_malformed", true) + .field("coerce", false).endObject() + .startObject("noErrorCoerceField").field("type", nonFractionNumericFieldType).field("ignore_malformed", true) + .field("coerce", true).endObject() + .startObject("errorDefaultCoerce").field("type", nonFractionNumericFieldType).field("ignore_malformed", false).endObject() + .startObject("errorNoCoerce").field("type", nonFractionNumericFieldType).field("ignore_malformed", false) + .field("coerce", false).endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping); + + //Test numbers passed as strings + String invalidJsonNumberAsString="1"; + ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("noErrorNoCoerceField", invalidJsonNumberAsString) + .field("noErrorCoerceField", invalidJsonNumberAsString) + .field("errorDefaultCoerce", invalidJsonNumberAsString) + .endObject() + .bytes()); + assertThat(doc.rootDoc().getField("noErrorNoCoerceField"), nullValue()); + assertThat(doc.rootDoc().getField("noErrorCoerceField"), notNullValue()); + //Default is ignore_malformed=true and coerce=true + assertThat(doc.rootDoc().getField("errorDefaultCoerce"), notNullValue()); + + //Test valid case of numbers passed as numbers + int validNumber=1; + doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("noErrorNoCoerceField", validNumber) + .field("noErrorCoerceField", validNumber) + .field("errorDefaultCoerce", validNumber) + .endObject() + .bytes()); + assertEquals(validNumber,doc.rootDoc().getField("noErrorNoCoerceField").numericValue().intValue()); + assertEquals(validNumber,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue()); + assertEquals(validNumber,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue()); + + //Test valid case of negative numbers passed as numbers + int validNegativeNumber=-1; + doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("noErrorNoCoerceField", validNegativeNumber) + .field("noErrorCoerceField", validNegativeNumber) + .field("errorDefaultCoerce", validNegativeNumber) + .endObject() + .bytes()); + assertEquals(validNegativeNumber,doc.rootDoc().getField("noErrorNoCoerceField").numericValue().intValue()); + assertEquals(validNegativeNumber,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue()); + assertEquals(validNegativeNumber,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue()); + + + try { + defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("errorNoCoerce", invalidJsonNumberAsString) + .endObject() + .bytes()); + } catch (MapperParsingException e) { + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + } + + + //Test questionable case of floats passed to ints + float invalidJsonForInteger=1.9f; + int coercedFloatValue=1; //This is what the JSON parser will do to a float - truncate not round + doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("noErrorNoCoerceField", invalidJsonForInteger) + .field("noErrorCoerceField", invalidJsonForInteger) + .field("errorDefaultCoerce", invalidJsonForInteger) + .endObject() + .bytes()); + assertThat(doc.rootDoc().getField("noErrorNoCoerceField"), nullValue()); + assertEquals(coercedFloatValue,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue()); + //Default is ignore_malformed=true and coerce=true + assertEquals(coercedFloatValue,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue()); + + try { + defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("errorNoCoerce", invalidJsonForInteger) + .endObject() + .bytes()); + } catch (MapperParsingException e) { + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + } + } + } + + public void testDocValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties")