honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adds a new coerce flag for numeric field mappings which is defaulted to true. 

When set to false a new strict mode of parsing is employed which
a) does not permit numbers to be passed as JSON strings in quotes
b) rejects numbers with fractions that are passed to integer, short or long fields.

Closes #4117
This commit is contained in:
markharwood 2014-01-10 13:50:58 +00:00
parent 2c647b3a82
commit 541059a4d1
21 changed files with 307 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/reference/mapping.asciidoc
View File

@ -51,8 +51,16 @@ index>>.
The `index.mapping.ignore_malformed` global setting can be set on the
index level to allow to ignore malformed content globally across all
mapping types (malformed content example is trying to index a string
mapping types (malformed content example is trying to index a text string
value as a numeric type).
The `index.mapping.coerce` global setting can be set on the
index level to coerce numeric content globally across all
mapping types (The default setting is true and coercions attempted are
to convert strings with numbers into numeric types and also numeric values
with fractions to any integer/short/long values minus the fraction part).
When the permitted conversions fail in their attempts, the value is considered
malformed and the ignore_malformed setting dictates what will happen next.
--
include::mapping/fields.asciidoc[]

2
docs/reference/mapping/types/core-types.asciidoc
View File

@ -230,6 +230,8 @@ defaults to `true` or to the parent `object` type setting.
|`ignore_malformed` |Ignored a malformed number. Defaults to `false`.
|`coerce` |Try convert strings to numbers and truncate fractions for integers. Defaults to `true`.
|=======================================================================
[float]

18
src/main/java/org/elasticsearch/common/Explicit.java
View File

@ -20,12 +20,24 @@
package org.elasticsearch.common;
/**
* Holds a value that is either:
* a) set implicitly e.g. through some default value
* b) set explicitly e.g. from a user selection
*
* When merging conflicting configuration settings such as
* field mapping settings it is preferable to preserve an explicit
* choice rather than a choice made only made implicitly by defaults.
*
*/
public class Explicit<T> {
private final T value;
private final boolean explicit;
/**
* Create a value with an indication if this was an explicit choice
* @param value a setting value
* @param explicit true if the value passed is a conscious decision, false if using some kind of default
*/
public Explicit(T value, boolean explicit) {
this.value = value;
this.explicit = explicit;
@ -35,6 +47,10 @@ public class Explicit<T> {
return this.value;
}
/**
*
* @return true if the value passed is a conscious decision, false if using some kind of default
*/
public boolean explicit() {
return this.explicit;
}

10
src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
View File

@ -157,6 +157,16 @@ public interface XContentParser extends Closeable {
*/
boolean estimatedNumberType();
short shortValue(boolean coerce) throws IOException;
int intValue(boolean coerce) throws IOException;
long longValue(boolean coerce) throws IOException;
float floatValue(boolean coerce) throws IOException;
double doubleValue(boolean coerce) throws IOException;
short shortValue() throws IOException;
int intValue() throws IOException;

77
src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
View File

@ -31,6 +31,39 @@ import java.util.*;
*/
public abstract class AbstractXContentParser implements XContentParser {
//Currently this is not a setting that can be changed and is a policy
// that relates to how parsing of things like "boost" are done across
// the whole of Elasticsearch (eg if String "1.0" is a valid float).
// The idea behind keeping it as a constant is that we can track
// references to this policy decision throughout the codebase and find
// and change any code that needs to apply an alternative policy.
public static final boolean DEFAULT_NUMBER_COEERCE_POLICY = true;
private static void checkCoerceString(boolean coeerce, Class<? extends Number> clazz) {
if (!coeerce) {
//Need to throw type IllegalArgumentException as current catch logic in
//NumberFieldMapper.parseCreateField relies on this for "malformed" value detection
throw new IllegalArgumentException(clazz.getSimpleName() + " value passed as String");
}
}
// The 3rd party parsers we rely on are known to silently truncate fractions: see
// http://fasterxml.github.io/jackson-core/javadoc/2.3.0/com/fasterxml/jackson/core/JsonParser.html#getShortValue()
// If this behaviour is flagged as undesirable and any truncation occurs
// then this method is called to trigger the"malformed" handling logic
void ensureNumberConversion(boolean coerce, long result, Class<? extends Number> clazz) throws IOException {
if (!coerce) {
double fullVal = doDoubleValue();
if (result != fullVal) {
// Need to throw type IllegalArgumentException as current catch
// logic in NumberFieldMapper.parseCreateField relies on this
// for "malformed" value detection
throw new IllegalArgumentException(fullVal + " cannot be converted to " + clazz.getSimpleName() + " without data loss");
}
}
}
@Override
public boolean isBooleanValue() throws IOException {
@ -62,41 +95,72 @@ public abstract class AbstractXContentParser implements XContentParser {
@Override
public short shortValue() throws IOException {
return shortValue(DEFAULT_NUMBER_COEERCE_POLICY);
}
@Override
public short shortValue(boolean coerce) throws IOException {
Token token = currentToken();
if (token == Token.VALUE_STRING) {
checkCoerceString(coerce, Short.class);
return Short.parseShort(text());
}
return doShortValue();
short result = doShortValue();
ensureNumberConversion(coerce, result, Short.class);
return result;
}
protected abstract short doShortValue() throws IOException;
@Override
public int intValue() throws IOException {
return intValue(DEFAULT_NUMBER_COEERCE_POLICY);
}
@Override
public int intValue(boolean coerce) throws IOException {
Token token = currentToken();
if (token == Token.VALUE_STRING) {
checkCoerceString(coerce, Integer.class);
return Integer.parseInt(text());
}
return doIntValue();
int result = doIntValue();
ensureNumberConversion(coerce, result, Integer.class);
return result;
}
protected abstract int doIntValue() throws IOException;
@Override
public long longValue() throws IOException {
return longValue(DEFAULT_NUMBER_COEERCE_POLICY);
}
@Override
public long longValue(boolean coerce) throws IOException {
Token token = currentToken();
if (token == Token.VALUE_STRING) {
checkCoerceString(coerce, Long.class);
return Long.parseLong(text());
}
return doLongValue();
long result = doLongValue();
ensureNumberConversion(coerce, result, Long.class);
return result;
}
protected abstract long doLongValue() throws IOException;
@Override
public float floatValue() throws IOException {
return floatValue(DEFAULT_NUMBER_COEERCE_POLICY);
}
@Override
public float floatValue(boolean coerce) throws IOException {
Token token = currentToken();
if (token == Token.VALUE_STRING) {
checkCoerceString(coerce, Float.class);
return Float.parseFloat(text());
}
return doFloatValue();
@ -104,10 +168,17 @@ public abstract class AbstractXContentParser implements XContentParser {
protected abstract float doFloatValue() throws IOException;
@Override
public double doubleValue() throws IOException {
return doubleValue(DEFAULT_NUMBER_COEERCE_POLICY);
}
@Override
public double doubleValue(boolean coerce) throws IOException {
Token token = currentToken();
if (token == Token.VALUE_STRING) {
checkCoerceString(coerce, Double.class);
return Double.parseDouble(text());
}
return doDoubleValue();

15
src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
View File

@ -92,8 +92,8 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context),
precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context),
postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings,
context.indexSettings(), multiFieldsBuilder.build(this, context));
coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading,
fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -120,11 +120,12 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
private String nullValueAsString;
protected ByteFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues,
Byte nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider postingsProvider,
Byte nullValue, Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
PostingsFormatProvider postingsProvider,
DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity, Loading normsLoading,
@Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) {
super(names, precisionStep, boost, fieldType, docValues,
ignoreMalformed, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
ignoreMalformed, coerce, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), postingsProvider,
docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields);
this.nullValue = nullValue;
@ -293,7 +294,7 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
objValue = (byte) parser.shortValue();
objValue = (byte) parser.shortValue(coerce.value());
}
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
@ -308,7 +309,7 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
}
value = objValue;
} else {
value = (byte) parser.shortValue();
value = (byte) parser.shortValue(coerce.value());
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), parser.text(), boost);
}
@ -383,4 +384,4 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
return Byte.toString(number);
}
}
}
}

11
src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -128,7 +128,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
dateTimeFormatter = new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale);
}
DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter,
precisionStep, boost, fieldType, docValues, nullValue, timeUnit, roundCeil, ignoreMalformed(context),
precisionStep, boost, fieldType, docValues, nullValue, timeUnit, roundCeil, ignoreMalformed(context), coerce(context),
postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(),
multiFieldsBuilder.build(this, context));
fieldMapper.includeInAll(includeInAll);
@ -202,10 +202,11 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
protected final TimeUnit timeUnit;
protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, float boost, FieldType fieldType, Boolean docValues,
String nullValue, TimeUnit timeUnit, boolean roundCeil, Explicit<Boolean> ignoreMalformed,
String nullValue, TimeUnit timeUnit, boolean roundCeil, Explicit<Boolean> ignoreMalformed,Explicit<Boolean> coerce,
PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity,
Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields) {
super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, new NamedAnalyzer("_date/" + precisionStep,
super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, new NamedAnalyzer("_date/" + precisionStep,
new NumericDateAnalyzer(precisionStep, dateTimeFormatter.parser())),
new NamedAnalyzer("_date/max", new NumericDateAnalyzer(Integer.MAX_VALUE, dateTimeFormatter.parser())),
postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields);
@ -393,7 +394,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
if (token == XContentParser.Token.VALUE_NULL) {
dateAsString = nullValue;
} else if (token == XContentParser.Token.VALUE_NUMBER) {
value = parser.longValue();
value = parser.longValue(coerce.value());
} else if (token == XContentParser.Token.START_OBJECT) {
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@ -404,7 +405,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
if (token == XContentParser.Token.VALUE_NULL) {
dateAsString = nullValue;
} else if (token == XContentParser.Token.VALUE_NUMBER) {
value = parser.longValue();
value = parser.longValue(coerce.value());
} else {
dateAsString = parser.text();
}

13
src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
View File

@ -95,8 +95,8 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
public DoubleFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context),
precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), postingsProvider, docValuesProvider,
similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context));
precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), coerce(context), postingsProvider,
docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -124,11 +124,12 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
private String nullValueAsString;
protected DoubleFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues,
Double nullValue, Explicit<Boolean> ignoreMalformed,
Double nullValue, Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider,
SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings,
Settings indexSettings, MultiFields multiFields) {
super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed,
super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce,
NumericDoubleAnalyzer.buildNamedAnalyzer(precisionStep), NumericDoubleAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE),
postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields);
this.nullValue = nullValue;
@ -288,7 +289,7 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
objValue = parser.doubleValue();
objValue = parser.doubleValue(coerce.value());
}
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
@ -303,7 +304,7 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
}
value = objValue;
} else {
value = parser.doubleValue();
value = parser.doubleValue(coerce.value());
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), parser.text(), boost);
}

12
src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
View File

@ -96,8 +96,8 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
public FloatFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context),
precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), postingsProvider, docValuesProvider,
similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context));
precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), coerce(context), postingsProvider,
docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -124,11 +124,11 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
private String nullValueAsString;
protected FloatFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues,
Float nullValue, Explicit<Boolean> ignoreMalformed,
Float nullValue, Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider,
SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings,
Settings indexSettings, MultiFields multiFields) {
super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed,
super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce,
NumericFloatAnalyzer.buildNamedAnalyzer(precisionStep), NumericFloatAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE),
postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields);
this.nullValue = nullValue;
@ -294,7 +294,7 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
objValue = parser.floatValue();
objValue = parser.floatValue(coerce.value());
}
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
@ -309,7 +309,7 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
}
value = objValue;
} else {
value = parser.floatValue();
value = parser.floatValue(coerce.value());
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), parser.text(), boost);
}

12
src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
View File

@ -92,8 +92,8 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
public IntegerFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues,
nullValue, ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, normsLoading,
fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context));
nullValue, ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings,
context.indexSettings(), multiFieldsBuilder.build(this, context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -120,11 +120,11 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
private String nullValueAsString;
protected IntegerFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues,
Integer nullValue, Explicit<Boolean> ignoreMalformed,
Integer nullValue, Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider,
SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings,
Settings indexSettings, MultiFields multiFields) {
super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed,
super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce,
NumericIntegerAnalyzer.buildNamedAnalyzer(precisionStep), NumericIntegerAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE),
postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields);
this.nullValue = nullValue;
@ -289,7 +289,7 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
objValue = parser.intValue();
objValue = parser.intValue(coerce.value());
}
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
@ -304,7 +304,7 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
}
value = objValue;
} else {
value = parser.intValue();
value = parser.intValue(coerce.value());
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), parser.text(), boost);
}

10
src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
View File

@ -92,7 +92,7 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
public LongFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
LongFieldMapper fieldMapper = new LongFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, nullValue,
ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, normsLoading,
ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading,
fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
@ -120,11 +120,11 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
private String nullValueAsString;
protected LongFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues,
Long nullValue, Explicit<Boolean> ignoreMalformed,
Long nullValue, Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider,
SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings,
Settings indexSettings, MultiFields multiFields) {
super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed,
super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce,
NumericLongAnalyzer.buildNamedAnalyzer(precisionStep), NumericLongAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE),
postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields);
this.nullValue = nullValue;
@ -279,7 +279,7 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
objValue = parser.longValue();
objValue = parser.longValue(coerce.value());
}
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
@ -294,7 +294,7 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
}
value = objValue;
} else {
value = parser.longValue();
value = parser.longValue(coerce.value());
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), parser.text(), boost);
}

30
src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
View File

@ -77,6 +77,7 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
}
public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<Boolean>(false, false);
public static final Explicit<Boolean> COERCE = new Explicit<Boolean>(true, false);
}
public abstract static class Builder<T extends Builder, Y extends NumberFieldMapper> extends AbstractFieldMapper.Builder<T, Y> {
@ -85,6 +86,8 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
private Boolean ignoreMalformed;
private Boolean coerce;
public Builder(String name, FieldType fieldType) {
super(name, fieldType);
}
@ -108,6 +111,22 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
}
return Defaults.IGNORE_MALFORMED;
}
public T coerce(boolean coerce) {
this.coerce = coerce;
return builder;
}
protected Explicit<Boolean> coerce(BuilderContext context) {
if (coerce != null) {
return new Explicit<Boolean>(coerce, true);
}
if (context.indexSettings() != null) {
return new Explicit<Boolean>(context.indexSettings().getAsBoolean("index.mapping.coerce", Defaults.COERCE.value()), false);
}
return Defaults.COERCE;
}
}
protected int precisionStep;
@ -116,6 +135,8 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
protected Explicit<Boolean> ignoreMalformed;
protected Explicit<Boolean> coerce;
private ThreadLocal<NumericTokenStream> tokenStream = new ThreadLocal<NumericTokenStream>() {
@Override
protected NumericTokenStream initialValue() {
@ -145,7 +166,7 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
};
protected NumberFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues,
Explicit<Boolean> ignoreMalformed, NamedAnalyzer indexAnalyzer,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, NamedAnalyzer indexAnalyzer,
NamedAnalyzer searchAnalyzer, PostingsFormatProvider postingsProvider,
DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity,
Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings,
@ -159,6 +180,7 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
this.precisionStep = precisionStep;
}
this.ignoreMalformed = ignoreMalformed;
this.coerce = coerce;
}
@Override
@ -334,6 +356,9 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
if (nfmMergeWith.ignoreMalformed.explicit()) {
this.ignoreMalformed = nfmMergeWith.ignoreMalformed;
}
if (nfmMergeWith.coerce.explicit()) {
this.coerce = nfmMergeWith.coerce;
}
}
}
@ -477,6 +502,9 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
if (includeDefaults || ignoreMalformed.explicit()) {
builder.field("ignore_malformed", ignoreMalformed.value());
}
if (includeDefaults || coerce.explicit()) {
builder.field("coerce", coerce.value());
}
}
@Override

10
src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
View File

@ -93,7 +93,7 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
public ShortFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
ShortFieldMapper fieldMapper = new ShortFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, nullValue,
ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings,
ignoreMalformed(context), coerce(context),postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings,
context.indexSettings(), multiFieldsBuilder.build(this, context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
@ -121,11 +121,11 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
private String nullValueAsString;
protected ShortFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues,
Short nullValue, Explicit<Boolean> ignoreMalformed,
Short nullValue, Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider,
SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings,
Settings indexSettings, MultiFields multiFields) {
super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, new NamedAnalyzer("_short/" + precisionStep,
super(names, precisionStep, boost, fieldType, docValues, ignoreMalformed, coerce, new NamedAnalyzer("_short/" + precisionStep,
new NumericIntegerAnalyzer(precisionStep)), new NamedAnalyzer("_short/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)),
postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, indexSettings, multiFields);
this.nullValue = nullValue;
@ -294,7 +294,7 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
objValue = parser.shortValue();
objValue = parser.shortValue(coerce.value());
}
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
@ -309,7 +309,7 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
}
value = objValue;
} else {
value = parser.shortValue();
value = parser.shortValue(coerce.value());
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), parser.text(), boost);
}

9
src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java
View File

@ -79,7 +79,7 @@ public class TokenCountFieldMapper extends IntegerFieldMapper {
public TokenCountFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
TokenCountFieldMapper fieldMapper = new TokenCountFieldMapper(buildNames(context), precisionStep, boost, fieldType, docValues, nullValue,
ignoreMalformed(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(),
ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(),
analyzer, multiFieldsBuilder.build(this, context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
@ -115,11 +115,12 @@ public class TokenCountFieldMapper extends IntegerFieldMapper {
private NamedAnalyzer analyzer;
protected TokenCountFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues, Integer nullValue,
Explicit<Boolean> ignoreMalformed, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider,
SimilarityProvider similarity, Loading normsLoading, Settings fieldDataSettings, Settings indexSettings, NamedAnalyzer analyzer,
MultiFields multiFields) {
super(names, precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed, postingsProvider, docValuesProvider, similarity,
normsLoading, fieldDataSettings, indexSettings, multiFields);
super(names, precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed, coerce, postingsProvider, docValuesProvider,
similarity, normsLoading, fieldDataSettings, indexSettings, multiFields);
this.analyzer = analyzer;
}

2
src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
View File

@ -146,6 +146,8 @@ public class TypeParsers {
builder.precisionStep(nodeIntegerValue(propNode));
} else if (propName.equals("ignore_malformed")) {
builder.ignoreMalformed(nodeBooleanValue(propNode));
} else if (propName.equals("coerce")) {
builder.coerce(nodeBooleanValue(propNode));
} else if (propName.equals("omit_norms")) {
builder.omitNorms(nodeBooleanValue(propNode));
} else if (propName.equals("similarity")) {

4
src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
View File

@ -125,7 +125,7 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements Intern
protected BoostFieldMapper(String name, String indexName, int precisionStep, float boost, FieldType fieldType, Boolean docValues, Float nullValue,
PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, @Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(name, indexName, indexName, name), precisionStep, boost, fieldType, docValues, Defaults.IGNORE_MALFORMED,
super(new Names(name, indexName, indexName, name), precisionStep, boost, fieldType, docValues, Defaults.IGNORE_MALFORMED, Defaults.COERCE,
NumericFloatAnalyzer.buildNamedAnalyzer(precisionStep), NumericFloatAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE),
postingsProvider, docValuesProvider, null, null, fieldDataSettings, indexSettings, MultiFields.empty());
this.nullValue = nullValue;
@ -273,7 +273,7 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements Intern
}
value = nullValue;
} else {
value = context.parser().floatValue();
value = context.parser().floatValue(coerce.value());
}
return value;
}

2
src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java
View File

@ -102,7 +102,7 @@ public class SizeFieldMapper extends IntegerFieldMapper implements RootMapper {
public SizeFieldMapper(EnabledAttributeMapper enabled, FieldType fieldType, PostingsFormatProvider postingsProvider,
DocValuesFormatProvider docValuesProvider, @Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(Defaults.NAME), Defaults.PRECISION_STEP, Defaults.BOOST, fieldType, null, Defaults.NULL_VALUE,
Defaults.IGNORE_MALFORMED, postingsProvider, docValuesProvider, null, null, fieldDataSettings,
Defaults.IGNORE_MALFORMED, Defaults.COERCE, postingsProvider, docValuesProvider, null, null, fieldDataSettings,
indexSettings, MultiFields.empty());
this.enabledState = enabled;
}

12
src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
View File

@ -89,7 +89,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
@Override
public TTLFieldMapper build(BuilderContext context) {
return new TTLFieldMapper(fieldType, enabledState, defaultTTL, ignoreMalformed(context), postingsProvider, docValuesProvider, fieldDataSettings, context.indexSettings());
return new TTLFieldMapper(fieldType, enabledState, defaultTTL, ignoreMalformed(context),coerce(context), postingsProvider, docValuesProvider, fieldDataSettings, context.indexSettings());
}
}
@ -119,14 +119,14 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
private long defaultTTL;
public TTLFieldMapper() {
this(new FieldType(Defaults.TTL_FIELD_TYPE), Defaults.ENABLED_STATE, Defaults.DEFAULT, Defaults.IGNORE_MALFORMED, null, null, null, ImmutableSettings.EMPTY);
this(new FieldType(Defaults.TTL_FIELD_TYPE), Defaults.ENABLED_STATE, Defaults.DEFAULT, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, ImmutableSettings.EMPTY);
}
protected TTLFieldMapper(FieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider,
@Nullable Settings fieldDataSettings, Settings indexSettings) {
Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider,
@Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP,
Defaults.BOOST, fieldType, null, Defaults.NULL_VALUE, ignoreMalformed,
Defaults.BOOST, fieldType, null, Defaults.NULL_VALUE, ignoreMalformed, coerce,
postingsProvider, docValuesProvider, null, null, fieldDataSettings, indexSettings, MultiFields.empty());
this.enabledState = enabled;
this.defaultTTL = defaultTTL;
@ -184,7 +184,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
if (context.parser().currentToken() == XContentParser.Token.VALUE_STRING) {
ttl = TimeValue.parseTimeValue(context.parser().text(), null).millis();
} else {
ttl = context.parser().longValue();
ttl = context.parser().longValue(coerce.value());
}
if (ttl <= 0) {
throw new MapperParsingException("TTL value must be > 0. Illegal value provided [" + ttl + "]");

8
src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View File

@ -105,7 +105,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
roundCeil = settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL));
}
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, roundCeil,
ignoreMalformed(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
}
}
@ -137,18 +137,18 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
public TimestampFieldMapper() {
this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER,
Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, null, null, null, null, ImmutableSettings.EMPTY);
Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, ImmutableSettings.EMPTY);
}
protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path,
FormatDateTimeFormatter dateTimeFormatter, boolean roundCeil,
Explicit<Boolean> ignoreMalformed, PostingsFormatProvider postingsProvider,
Explicit<Boolean> ignoreMalformed,Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
DocValuesFormatProvider docValuesProvider, Loading normsLoading,
@Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
Defaults.PRECISION_STEP, Defaults.BOOST, fieldType, docValues,
Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/,
roundCeil, ignoreMalformed, postingsProvider, docValuesProvider, null, normsLoading, fieldDataSettings,
roundCeil, ignoreMalformed, coerce, postingsProvider, docValuesProvider, null, normsLoading, fieldDataSettings,
indexSettings, MultiFields.empty());
this.enabledState = enabledState;
this.path = path;

7
src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
View File

@ -122,7 +122,8 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
public IpFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
IpFieldMapper fieldMapper = new IpFieldMapper(buildNames(context),
precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), postingsProvider, docValuesProvider, similarity,
precisionStep, boost, fieldType, docValues, nullValue, ignoreMalformed(context), coerce(context),
postingsProvider, docValuesProvider, similarity,
normsLoading, fieldDataSettings, context.indexSettings(), multiFieldsBuilder.build(this, context));
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
@ -148,12 +149,12 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
private String nullValue;
protected IpFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Boolean docValues,
String nullValue, Explicit<Boolean> ignoreMalformed,
String nullValue, Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider,
SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings,
Settings indexSettings, MultiFields multiFields) {
super(names, precisionStep, boost, fieldType, docValues,
ignoreMalformed, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)),
ignoreMalformed, coerce, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)),
new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)), postingsProvider, docValuesProvider,
similarity, normsLoading, fieldDataSettings, indexSettings, multiFields);
this.nullValue = nullValue;

99
src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java
View File

@ -145,6 +145,105 @@ public class SimpleNumericTests extends ElasticsearchTestCase {
}
}
@Test
public void testCoerceOption() throws Exception {
String [] nonFractionNumericFieldTypes={"integer","long","short"};
//Test co-ercion policies on all non-fraction numerics
for (String nonFractionNumericFieldType : nonFractionNumericFieldTypes) {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("noErrorNoCoerceField").field("type", nonFractionNumericFieldType).field("ignore_malformed", true)
.field("coerce", false).endObject()
.startObject("noErrorCoerceField").field("type", nonFractionNumericFieldType).field("ignore_malformed", true)
.field("coerce", true).endObject()
.startObject("errorDefaultCoerce").field("type", nonFractionNumericFieldType).field("ignore_malformed", false).endObject()
.startObject("errorNoCoerce").field("type", nonFractionNumericFieldType).field("ignore_malformed", false)
.field("coerce", false).endObject()
.endObject()
.endObject().endObject().string();
DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping);
//Test numbers passed as strings
String invalidJsonNumberAsString="1";
ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("noErrorNoCoerceField", invalidJsonNumberAsString)
.field("noErrorCoerceField", invalidJsonNumberAsString)
.field("errorDefaultCoerce", invalidJsonNumberAsString)
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("noErrorNoCoerceField"), nullValue());
assertThat(doc.rootDoc().getField("noErrorCoerceField"), notNullValue());
//Default is ignore_malformed=true and coerce=true
assertThat(doc.rootDoc().getField("errorDefaultCoerce"), notNullValue());
//Test valid case of numbers passed as numbers
int validNumber=1;
doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("noErrorNoCoerceField", validNumber)
.field("noErrorCoerceField", validNumber)
.field("errorDefaultCoerce", validNumber)
.endObject()
.bytes());
assertEquals(validNumber,doc.rootDoc().getField("noErrorNoCoerceField").numericValue().intValue());
assertEquals(validNumber,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue());
assertEquals(validNumber,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue());
//Test valid case of negative numbers passed as numbers
int validNegativeNumber=-1;
doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("noErrorNoCoerceField", validNegativeNumber)
.field("noErrorCoerceField", validNegativeNumber)
.field("errorDefaultCoerce", validNegativeNumber)
.endObject()
.bytes());
assertEquals(validNegativeNumber,doc.rootDoc().getField("noErrorNoCoerceField").numericValue().intValue());
assertEquals(validNegativeNumber,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue());
assertEquals(validNegativeNumber,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue());
try {
defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("errorNoCoerce", invalidJsonNumberAsString)
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
}
//Test questionable case of floats passed to ints
float invalidJsonForInteger=1.9f;
int coercedFloatValue=1; //This is what the JSON parser will do to a float - truncate not round
doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("noErrorNoCoerceField", invalidJsonForInteger)
.field("noErrorCoerceField", invalidJsonForInteger)
.field("errorDefaultCoerce", invalidJsonForInteger)
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("noErrorNoCoerceField"), nullValue());
assertEquals(coercedFloatValue,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue());
//Default is ignore_malformed=true and coerce=true
assertEquals(coercedFloatValue,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue());
try {
defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("errorNoCoerce", invalidJsonForInteger)
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
}
}
}
public void testDocValues() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")