diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc index ec362fc5771..3c5a225cc05 100644 --- a/docs/reference/mapping.asciidoc +++ b/docs/reference/mapping.asciidoc @@ -51,8 +51,16 @@ index>>. The `index.mapping.ignore_malformed` global setting can be set on the index level to allow to ignore malformed content globally across all -mapping types (malformed content example is trying to index a string +mapping types (malformed content example is trying to index a text string value as a numeric type). + +The `index.mapping.coerce` global setting can be set on the +index level to coerce numeric content globally across all +mapping types (The default setting is true and coercions attempted are +to convert strings with numbers into numeric types and also numeric values +with fractions to any integer/short/long values minus the fraction part). +When the permitted conversions fail in their attempts, the value is considered +malformed and the ignore_malformed setting dictates what will happen next. -- include::mapping/fields.asciidoc[] diff --git a/docs/reference/mapping/types/core-types.asciidoc b/docs/reference/mapping/types/core-types.asciidoc index e0d9aed65d9..e3e7d03f27e 100644 --- a/docs/reference/mapping/types/core-types.asciidoc +++ b/docs/reference/mapping/types/core-types.asciidoc @@ -230,6 +230,8 @@ defaults to `true` or to the parent `object` type setting. |`ignore_malformed` |Ignored a malformed number. Defaults to `false`. +|`coerce` |Try convert strings to numbers and truncate fractions for integers. Defaults to `true`. + |======================================================================= [float] diff --git a/src/main/java/org/elasticsearch/common/Explicit.java b/src/main/java/org/elasticsearch/common/Explicit.java index d2aa495e2b1..3132f22daa0 100644 --- a/src/main/java/org/elasticsearch/common/Explicit.java +++ b/src/main/java/org/elasticsearch/common/Explicit.java @@ -20,12 +20,24 @@ package org.elasticsearch.common; /** + * Holds a value that is either: + * a) set implicitly e.g. through some default value + * b) set explicitly e.g. from a user selection + * + * When merging conflicting configuration settings such as + * field mapping settings it is preferable to preserve an explicit + * choice rather than a choice made only made implicitly by defaults. + * */ public class Explicit { private final T value; private final boolean explicit; - + /** + * Create a value with an indication if this was an explicit choice + * @param value a setting value + * @param explicit true if the value passed is a conscious decision, false if using some kind of default + */ public Explicit(T value, boolean explicit) { this.value = value; this.explicit = explicit; @@ -35,6 +47,10 @@ public class Explicit { return this.value; } + /** + * + * @return true if the value passed is a conscious decision, false if using some kind of default + */ public boolean explicit() { return this.explicit; } diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java index 486f80ee6c0..497dbe7503f 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java @@ -157,6 +157,16 @@ public interface XContentParser extends Closeable { */ boolean estimatedNumberType(); + short shortValue(boolean coerce) throws IOException; + + int intValue(boolean coerce) throws IOException; + + long longValue(boolean coerce) throws IOException; + + float floatValue(boolean coerce) throws IOException; + + double doubleValue(boolean coerce) throws IOException; + short shortValue() throws IOException; int intValue() throws IOException; diff --git a/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java index 679809723af..7d55b6a3f31 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java +++ b/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java @@ -31,6 +31,39 @@ import java.util.*; */ public abstract class AbstractXContentParser implements XContentParser { + //Currently this is not a setting that can be changed and is a policy + // that relates to how parsing of things like "boost" are done across + // the whole of Elasticsearch (eg if String "1.0" is a valid float). + // The idea behind keeping it as a constant is that we can track + // references to this policy decision throughout the codebase and find + // and change any code that needs to apply an alternative policy. + public static final boolean DEFAULT_NUMBER_COEERCE_POLICY = true; + + private static void checkCoerceString(boolean coeerce, Class clazz) { + if (!coeerce) { + //Need to throw type IllegalArgumentException as current catch logic in + //NumberFieldMapper.parseCreateField relies on this for "malformed" value detection + throw new IllegalArgumentException(clazz.getSimpleName() + " value passed as String"); + } + } + + + + // The 3rd party parsers we rely on are known to silently truncate fractions: see + // http://fasterxml.github.io/jackson-core/javadoc/2.3.0/com/fasterxml/jackson/core/JsonParser.html#getShortValue() + // If this behaviour is flagged as undesirable and any truncation occurs + // then this method is called to trigger the"malformed" handling logic + void ensureNumberConversion(boolean coerce, long result, Class clazz) throws IOException { + if (!coerce) { + double fullVal = doDoubleValue(); + if (result != fullVal) { + // Need to throw type IllegalArgumentException as current catch + // logic in NumberFieldMapper.parseCreateField relies on this + // for "malformed" value detection + throw new IllegalArgumentException(fullVal + " cannot be converted to " + clazz.getSimpleName() + " without data loss"); + } + } + } @Override public boolean isBooleanValue() throws IOException { @@ -62,41 +95,72 @@ public abstract class AbstractXContentParser implements XContentParser { @Override public short shortValue() throws IOException { + return shortValue(DEFAULT_NUMBER_COEERCE_POLICY); + } + + @Override + public short shortValue(boolean coerce) throws IOException { Token token = currentToken(); if (token == Token.VALUE_STRING) { + checkCoerceString(coerce, Short.class); return Short.parseShort(text()); } - return doShortValue(); + short result = doShortValue(); + ensureNumberConversion(coerce, result, Short.class); + return result; } protected abstract short doShortValue() throws IOException; @Override public int intValue() throws IOException { + return intValue(DEFAULT_NUMBER_COEERCE_POLICY); + } + + + @Override + public int intValue(boolean coerce) throws IOException { Token token = currentToken(); if (token == Token.VALUE_STRING) { + checkCoerceString(coerce, Integer.class); return Integer.parseInt(text()); } - return doIntValue(); + int result = doIntValue(); + ensureNumberConversion(coerce, result, Integer.class); + return result; } protected abstract int doIntValue() throws IOException; @Override public long longValue() throws IOException { + return longValue(DEFAULT_NUMBER_COEERCE_POLICY); + } + + @Override + public long longValue(boolean coerce) throws IOException { Token token = currentToken(); if (token == Token.VALUE_STRING) { + checkCoerceString(coerce, Long.class); return Long.parseLong(text()); } - return doLongValue(); + long result = doLongValue(); + ensureNumberConversion(coerce, result, Long.class); + return result; } protected abstract long doLongValue() throws IOException; @Override public float floatValue() throws IOException { + return floatValue(DEFAULT_NUMBER_COEERCE_POLICY); + } + + @Override + public float floatValue(boolean coerce) throws IOException { Token token = currentToken(); if (token == Token.VALUE_STRING) { + checkCoerceString(coerce, Float.class); return Float.parseFloat(text()); } return doFloatValue(); @@ -104,10 +168,17 @@ public abstract class AbstractXContentParser implements XContentParser { protected abstract float doFloatValue() throws IOException; + @Override public double doubleValue() throws IOException { + return doubleValue(DEFAULT_NUMBER_COEERCE_POLICY); + } + + @Override + public double doubleValue(boolean coerce) throws IOException { Token token = currentToken(); if (token == Token.VALUE_STRING) { + checkCoerceString(coerce, Double.class); return Double.parseDouble(text()); } return doDoubleValue(); diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java index d42afd7c995..7b975b1941a 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java @@ -92,8 +92,8 @@ public class ByteFieldMapper extends NumberFieldMapper { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), - postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, - context.indexSettings(), multiFieldsBuilder.build(this, context)); + coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, + fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } @@ -120,11 +120,12 @@ public class ByteFieldMapper extends NumberFieldMapper { private String nullValueAsString; protected ByteFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Byte nullValue, Explicit ignoreMalformed, PostingsFormatProvider postingsProvider, + Byte nullValue, Explicit ignoreMalformed, Explicit coerce, + PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { super(names, precisionStep, boost, fieldType, docValues, - ignoreMalformed, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), + ignoreMalformed, coerce, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; @@ -293,7 +294,7 @@ public class ByteFieldMapper extends NumberFieldMapper { } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { - objValue = (byte) parser.shortValue(); + objValue = (byte) parser.shortValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); @@ -308,7 +309,7 @@ public class ByteFieldMapper extends NumberFieldMapper { } value = objValue; } else { - value = (byte) parser.shortValue(); + value = (byte) parser.shortValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } @@ -383,4 +384,4 @@ public class ByteFieldMapper extends NumberFieldMapper { return Byte.toString(number); } } -} \ No newline at end of file +} diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 1a8b8748922..375d21031c2 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -128,7 +128,7 @@ public class DateFieldMapper extends NumberFieldMapper { dateTimeFormatter = new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale); } DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter, - precisionStep, boost, fieldType, docValues, nullValue, timeUnit, roundCeil, ignoreMalformed(context), + precisionStep, boost, fieldType, docValues, nullValue, timeUnit, roundCeil, ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); @@ -202,10 +202,11 @@ public class DateFieldMapper extends NumberFieldMapper { protected final TimeUnit timeUnit; protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - String nullValue, TimeUnit timeUnit, boolean roundCeil, Explicit ignoreMalformed, + String nullValue, TimeUnit timeUnit, boolean roundCeil, Explicit ignoreMalformed,Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, + Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, new NamedAnalyzer("_date/" + precisionStep, + super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, new NamedAnalyzer("_date/" + precisionStep, new NumericDateAnalyzer(precisionStep, dateTimeFormatter.parser())), new NamedAnalyzer("_date/max", new NumericDateAnalyzer(Integer.MAX_VALUE, dateTimeFormatter.parser())), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); @@ -393,7 +394,7 @@ public class DateFieldMapper extends NumberFieldMapper { if (token == XContentParser.Token.VALUE_NULL) { dateAsString = nullValue; } else if (token == XContentParser.Token.VALUE_NUMBER) { - value = parser.longValue(); + value = parser.longValue(coerce.value()); } else if (token == XContentParser.Token.START_OBJECT) { String currentFieldName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -404,7 +405,7 @@ public class DateFieldMapper extends NumberFieldMapper { if (token == XContentParser.Token.VALUE_NULL) { dateAsString = nullValue; } else if (token == XContentParser.Token.VALUE_NUMBER) { - value = parser.longValue(); + value = parser.longValue(coerce.value()); } else { dateAsString = parser.text(); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index 062e1f2d050..b3f9027e420 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -95,8 +95,8 @@ public class DoubleFieldMapper extends NumberFieldMapper { public DoubleFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context), - precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), postingsProvider, docValuesProvider, - similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); + precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), coerce(context), postingsProvider, + docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } @@ -124,11 +124,12 @@ public class DoubleFieldMapper extends NumberFieldMapper { private String nullValueAsString; protected DoubleFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Double nullValue, Explicit ignoreMalformed, + Double nullValue, Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, + SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, + super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, NumericDoubleAnalyzer.buildNamedAnalyzer(precisionStep), NumericDoubleAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; @@ -288,7 +289,7 @@ public class DoubleFieldMapper extends NumberFieldMapper { } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { - objValue = parser.doubleValue(); + objValue = parser.doubleValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); @@ -303,7 +304,7 @@ public class DoubleFieldMapper extends NumberFieldMapper { } value = objValue; } else { - value = parser.doubleValue(); + value = parser.doubleValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java index c2110a49cc7..eb313ee306d 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java @@ -96,8 +96,8 @@ public class FloatFieldMapper extends NumberFieldMapper { public FloatFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context), - precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), postingsProvider, docValuesProvider, - similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); + precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), coerce(context), postingsProvider, + docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } @@ -124,11 +124,11 @@ public class FloatFieldMapper extends NumberFieldMapper { private String nullValueAsString; protected FloatFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Float nullValue, Explicit ignoreMalformed, + Float nullValue, Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, + super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, NumericFloatAnalyzer.buildNamedAnalyzer(precisionStep), NumericFloatAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; @@ -294,7 +294,7 @@ public class FloatFieldMapper extends NumberFieldMapper { } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { - objValue = parser.floatValue(); + objValue = parser.floatValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); @@ -309,7 +309,7 @@ public class FloatFieldMapper extends NumberFieldMapper { } value = objValue; } else { - value = parser.floatValue(); + value = parser.floatValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java index cb1c010e510..485f1097c49 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java @@ -92,8 +92,8 @@ public class IntegerFieldMapper extends NumberFieldMapper { public IntegerFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, - nullValue, ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, normsLoading, - fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); + nullValue, ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, + context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; } @@ -120,11 +120,11 @@ public class IntegerFieldMapper extends NumberFieldMapper { private String nullValueAsString; protected IntegerFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Integer nullValue, Explicit ignoreMalformed, + Integer nullValue, Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, + super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, NumericIntegerAnalyzer.buildNamedAnalyzer(precisionStep), NumericIntegerAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; @@ -289,7 +289,7 @@ public class IntegerFieldMapper extends NumberFieldMapper { } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { - objValue = parser.intValue(); + objValue = parser.intValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); @@ -304,7 +304,7 @@ public class IntegerFieldMapper extends NumberFieldMapper { } value = objValue; } else { - value = parser.intValue(); + value = parser.intValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java index 38b38ceacfe..f1acae71553 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java @@ -92,7 +92,7 @@ public class LongFieldMapper extends NumberFieldMapper { public LongFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); LongFieldMapper fieldMapper = new LongFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, nullValue, - ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, normsLoading, + ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -120,11 +120,11 @@ public class LongFieldMapper extends NumberFieldMapper { private String nullValueAsString; protected LongFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Long nullValue, Explicit ignoreMalformed, + Long nullValue, Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, + super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, NumericLongAnalyzer.buildNamedAnalyzer(precisionStep), NumericLongAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; @@ -279,7 +279,7 @@ public class LongFieldMapper extends NumberFieldMapper { } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { - objValue = parser.longValue(); + objValue = parser.longValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); @@ -294,7 +294,7 @@ public class LongFieldMapper extends NumberFieldMapper { } value = objValue; } else { - value = parser.longValue(); + value = parser.longValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java index de56c60dc41..d3d2c0d8062 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java @@ -77,6 +77,7 @@ public abstract class NumberFieldMapper extends AbstractFieldM } public static final Explicit IGNORE_MALFORMED = new Explicit(false, false); + public static final Explicit COERCE = new Explicit(true, false); } public abstract static class Builder extends AbstractFieldMapper.Builder { @@ -85,6 +86,8 @@ public abstract class NumberFieldMapper extends AbstractFieldM private Boolean ignoreMalformed; + private Boolean coerce; + public Builder(String name, FieldType fieldType) { super(name, fieldType); } @@ -108,6 +111,22 @@ public abstract class NumberFieldMapper extends AbstractFieldM } return Defaults.IGNORE_MALFORMED; } + + public T coerce(boolean coerce) { + this.coerce = coerce; + return builder; + } + + protected Explicit coerce(BuilderContext context) { + if (coerce != null) { + return new Explicit(coerce, true); + } + if (context.indexSettings() != null) { + return new Explicit(context.indexSettings().getAsBoolean("index.mapping.coerce", Defaults.COERCE.value()), false); + } + return Defaults.COERCE; + } + } protected int precisionStep; @@ -116,6 +135,8 @@ public abstract class NumberFieldMapper extends AbstractFieldM protected Explicit ignoreMalformed; + protected Explicit coerce; + private ThreadLocal tokenStream = new ThreadLocal() { @Override protected NumericTokenStream initialValue() { @@ -145,7 +166,7 @@ public abstract class NumberFieldMapper extends AbstractFieldM }; protected NumberFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Explicit ignoreMalformed, NamedAnalyzer indexAnalyzer, + Explicit ignoreMalformed, Explicit coerce, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, @@ -159,6 +180,7 @@ public abstract class NumberFieldMapper extends AbstractFieldM this.precisionStep = precisionStep; } this.ignoreMalformed = ignoreMalformed; + this.coerce = coerce; } @Override @@ -334,6 +356,9 @@ public abstract class NumberFieldMapper extends AbstractFieldM if (nfmMergeWith.ignoreMalformed.explicit()) { this.ignoreMalformed = nfmMergeWith.ignoreMalformed; } + if (nfmMergeWith.coerce.explicit()) { + this.coerce = nfmMergeWith.coerce; + } } } @@ -477,6 +502,9 @@ public abstract class NumberFieldMapper extends AbstractFieldM if (includeDefaults || ignoreMalformed.explicit()) { builder.field("ignore_malformed", ignoreMalformed.value()); } + if (includeDefaults || coerce.explicit()) { + builder.field("coerce", coerce.value()); + } } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java index 367dffd6548..1d217f801dd 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java @@ -93,7 +93,7 @@ public class ShortFieldMapper extends NumberFieldMapper { public ShortFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); ShortFieldMapper fieldMapper = new ShortFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, nullValue, - ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, + ignoreMalformed(context), coerce(context),postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -121,11 +121,11 @@ public class ShortFieldMapper extends NumberFieldMapper { private String nullValueAsString; protected ShortFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - Short nullValue, Explicit ignoreMalformed, + Short nullValue, Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, new NamedAnalyzer("_short/" + precisionStep, + super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, new NamedAnalyzer("_short/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), new NamedAnalyzer("_short/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; @@ -294,7 +294,7 @@ public class ShortFieldMapper extends NumberFieldMapper { } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { - objValue = parser.shortValue(); + objValue = parser.shortValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); @@ -309,7 +309,7 @@ public class ShortFieldMapper extends NumberFieldMapper { } value = objValue; } else { - value = parser.shortValue(); + value = parser.shortValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java index 20868c9df64..4544ee917f0 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java @@ -79,7 +79,7 @@ public class TokenCountFieldMapper extends IntegerFieldMapper { public TokenCountFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); TokenCountFieldMapper fieldMapper = new TokenCountFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, nullValue, - ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), + ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), analyzer, multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -115,11 +115,12 @@ public class TokenCountFieldMapper extends IntegerFieldMapper { private NamedAnalyzer analyzer; protected TokenCountFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, Integer nullValue, - Explicit ignoreMalformed, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, + Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, Settings fieldDataSettings, Settings indexSettings, NamedAnalyzer analyzer, MultiFields multiFields) { - super(names, precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed, postingsProvider, docValuesProvider, similarity, - normsLoading, fieldDataSettings, indexSettings, multiFields); + super(names, precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed, coerce, postingsProvider, docValuesProvider, + similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); + this.analyzer = analyzer; } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java index 56f9243b4c8..c142acae2a6 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java @@ -146,6 +146,8 @@ public class TypeParsers { builder.precisionStep(nodeIntegerValue(propNode)); } else if (propName.equals("ignore_malformed")) { builder.ignoreMalformed(nodeBooleanValue(propNode)); + } else if (propName.equals("coerce")) { + builder.coerce(nodeBooleanValue(propNode)); } else if (propName.equals("omit_norms")) { builder.omitNorms(nodeBooleanValue(propNode)); } else if (propName.equals("similarity")) { diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java index 77e12f64f78..c38a3ea113e 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java @@ -125,7 +125,7 @@ public class BoostFieldMapper extends NumberFieldMapper implements Intern protected BoostFieldMapper(String name, String indexName, int precisionStep, float boost, FieldType fieldType, Boolean docValues, Float nullValue, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, @Nullable Settings fieldDataSettings, Settings indexSettings) { - super(new Names(name, indexName, indexName, name), precisionStep, boost, fieldType, docValues, Defaults.IGNORE_MALFORMED, + super(new Names(name, indexName, indexName, name), precisionStep, boost, fieldType, docValues, Defaults.IGNORE_MALFORMED, Defaults.COERCE, NumericFloatAnalyzer.buildNamedAnalyzer(precisionStep), NumericFloatAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE), postingsProvider, docValuesProvider, null, null, fieldDataSettings, indexSettings, MultiFields.empty()); this.nullValue = nullValue; @@ -273,7 +273,7 @@ public class BoostFieldMapper extends NumberFieldMapper implements Intern } value = nullValue; } else { - value = context.parser().floatValue(); + value = context.parser().floatValue(coerce.value()); } return value; } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java index cd619405de4..4333032686b 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java @@ -102,7 +102,7 @@ public class SizeFieldMapper extends IntegerFieldMapper implements RootMapper { public SizeFieldMapper(EnabledAttributeMapper enabled, FieldType fieldType, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, @Nullable Settings fieldDataSettings, Settings indexSettings) { super(new Names(Defaults.NAME), Defaults.PRECISION_STEP, Defaults.BOOST, fieldType, null, Defaults.NULL_VALUE, - Defaults.IGNORE_MALFORMED, postingsProvider, docValuesProvider, null, null, fieldDataSettings, + Defaults.IGNORE_MALFORMED, Defaults.COERCE, postingsProvider, docValuesProvider, null, null, fieldDataSettings, indexSettings, MultiFields.empty()); this.enabledState = enabled; } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index 48be044e91a..d518b885238 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -89,7 +89,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R @Override public TTLFieldMapper build(BuilderContext context) { - return new TTLFieldMapper(fieldType, enabledState, defaultTTL, ignoreMalformed(context), postingsProvider, docValuesProvider, fieldDataSettings, context.indexSettings()); + return new TTLFieldMapper(fieldType, enabledState, defaultTTL, ignoreMalformed(context),coerce(context), postingsProvider, docValuesProvider, fieldDataSettings, context.indexSettings()); } } @@ -119,14 +119,14 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R private long defaultTTL; public TTLFieldMapper() { - this(new FieldType(Defaults.TTL_FIELD_TYPE), Defaults.ENABLED_STATE, Defaults.DEFAULT, Defaults.IGNORE_MALFORMED, null, null, null, ImmutableSettings.EMPTY); + this(new FieldType(Defaults.TTL_FIELD_TYPE), Defaults.ENABLED_STATE, Defaults.DEFAULT, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, ImmutableSettings.EMPTY); } protected TTLFieldMapper(FieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL, Explicit ignoreMalformed, - PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, - @Nullable Settings fieldDataSettings, Settings indexSettings) { + Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, + @Nullable Settings fieldDataSettings, Settings indexSettings) { super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP, - Defaults.BOOST, fieldType, null, Defaults.NULL_VALUE, ignoreMalformed, + Defaults.BOOST, fieldType, null, Defaults.NULL_VALUE, ignoreMalformed, coerce, postingsProvider, docValuesProvider, null, null, fieldDataSettings, indexSettings, MultiFields.empty()); this.enabledState = enabled; this.defaultTTL = defaultTTL; @@ -184,7 +184,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R if (context.parser().currentToken() == XContentParser.Token.VALUE_STRING) { ttl = TimeValue.parseTimeValue(context.parser().text(), null).millis(); } else { - ttl = context.parser().longValue(); + ttl = context.parser().longValue(coerce.value()); } if (ttl <= 0) { throw new MapperParsingException("TTL value must be > 0. Illegal value provided [" + ttl + "]"); diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index 51978b59f58..58fe783f5c5 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -105,7 +105,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap roundCeil = settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL)); } return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, roundCeil, - ignoreMalformed(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings()); + ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings()); } } @@ -137,18 +137,18 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap public TimestampFieldMapper() { this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, - Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, null, null, null, null, ImmutableSettings.EMPTY); + Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, ImmutableSettings.EMPTY); } protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path, FormatDateTimeFormatter dateTimeFormatter, boolean roundCeil, - Explicit ignoreMalformed, PostingsFormatProvider postingsProvider, + Explicit ignoreMalformed,Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings) { super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter, Defaults.PRECISION_STEP, Defaults.BOOST, fieldType, docValues, Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/, - roundCeil, ignoreMalformed, postingsProvider, docValuesProvider, null, normsLoading, fieldDataSettings, + roundCeil, ignoreMalformed, coerce, postingsProvider, docValuesProvider, null, normsLoading, fieldDataSettings, indexSettings, MultiFields.empty()); this.enabledState = enabledState; this.path = path; diff --git a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java index db379e45dc2..f9f31688ec3 100644 --- a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java @@ -122,7 +122,8 @@ public class IpFieldMapper extends NumberFieldMapper { public IpFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); IpFieldMapper fieldMapper = new IpFieldMapper(buildNames(context), - precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, + precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), coerce(context), + postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context)); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -148,12 +149,12 @@ public class IpFieldMapper extends NumberFieldMapper { private String nullValue; protected IpFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, - String nullValue, Explicit ignoreMalformed, + String nullValue, Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) { super(names, precisionStep, boost, fieldType, docValues, - ignoreMalformed, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)), + ignoreMalformed, coerce, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)), new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields); this.nullValue = nullValue; diff --git a/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java b/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java index e7f29d0feb6..541a05d17f7 100644 --- a/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java @@ -145,6 +145,105 @@ public class SimpleNumericTests extends ElasticsearchTestCase { } } + @Test + public void testCoerceOption() throws Exception { + String [] nonFractionNumericFieldTypes={"integer","long","short"}; + //Test co-ercion policies on all non-fraction numerics + for (String nonFractionNumericFieldType : nonFractionNumericFieldTypes) { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("noErrorNoCoerceField").field("type", nonFractionNumericFieldType).field("ignore_malformed", true) + .field("coerce", false).endObject() + .startObject("noErrorCoerceField").field("type", nonFractionNumericFieldType).field("ignore_malformed", true) + .field("coerce", true).endObject() + .startObject("errorDefaultCoerce").field("type", nonFractionNumericFieldType).field("ignore_malformed", false).endObject() + .startObject("errorNoCoerce").field("type", nonFractionNumericFieldType).field("ignore_malformed", false) + .field("coerce", false).endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping); + + //Test numbers passed as strings + String invalidJsonNumberAsString="1"; + ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("noErrorNoCoerceField", invalidJsonNumberAsString) + .field("noErrorCoerceField", invalidJsonNumberAsString) + .field("errorDefaultCoerce", invalidJsonNumberAsString) + .endObject() + .bytes()); + assertThat(doc.rootDoc().getField("noErrorNoCoerceField"), nullValue()); + assertThat(doc.rootDoc().getField("noErrorCoerceField"), notNullValue()); + //Default is ignore_malformed=true and coerce=true + assertThat(doc.rootDoc().getField("errorDefaultCoerce"), notNullValue()); + + //Test valid case of numbers passed as numbers + int validNumber=1; + doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("noErrorNoCoerceField", validNumber) + .field("noErrorCoerceField", validNumber) + .field("errorDefaultCoerce", validNumber) + .endObject() + .bytes()); + assertEquals(validNumber,doc.rootDoc().getField("noErrorNoCoerceField").numericValue().intValue()); + assertEquals(validNumber,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue()); + assertEquals(validNumber,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue()); + + //Test valid case of negative numbers passed as numbers + int validNegativeNumber=-1; + doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("noErrorNoCoerceField", validNegativeNumber) + .field("noErrorCoerceField", validNegativeNumber) + .field("errorDefaultCoerce", validNegativeNumber) + .endObject() + .bytes()); + assertEquals(validNegativeNumber,doc.rootDoc().getField("noErrorNoCoerceField").numericValue().intValue()); + assertEquals(validNegativeNumber,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue()); + assertEquals(validNegativeNumber,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue()); + + + try { + defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("errorNoCoerce", invalidJsonNumberAsString) + .endObject() + .bytes()); + } catch (MapperParsingException e) { + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + } + + + //Test questionable case of floats passed to ints + float invalidJsonForInteger=1.9f; + int coercedFloatValue=1; //This is what the JSON parser will do to a float - truncate not round + doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("noErrorNoCoerceField", invalidJsonForInteger) + .field("noErrorCoerceField", invalidJsonForInteger) + .field("errorDefaultCoerce", invalidJsonForInteger) + .endObject() + .bytes()); + assertThat(doc.rootDoc().getField("noErrorNoCoerceField"), nullValue()); + assertEquals(coercedFloatValue,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue()); + //Default is ignore_malformed=true and coerce=true + assertEquals(coercedFloatValue,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue()); + + try { + defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("errorNoCoerce", invalidJsonForInteger) + .endObject() + .bytes()); + } catch (MapperParsingException e) { + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + } + } + } + + public void testDocValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties")