diff --git a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 705cf048964..7f196b3433c 100644 --- a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -210,5 +210,11 @@ public interface FieldMapper { */ Filter rangeFilter(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context); + /** + * Null value filter, returns null if there is no null value associated with the field. + */ + @Nullable + Filter nullValueFilter(); + FieldDataType fieldDataType(); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java index 186a32f3cc0..ff2f4346981 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java @@ -415,6 +415,11 @@ public abstract class AbstractFieldMapper implements FieldMapper, Mapper { includeLower, includeUpper); } + @Override + public Filter nullValueFilter() { + return null; + } + @Override public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException { if (!this.getClass().equals(mergeWith.getClass())) { diff --git a/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java index addff856fab..5d1fc3048dd 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java @@ -21,9 +21,11 @@ package org.elasticsearch.index.mapper.core; import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.search.Filter; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.lucene.search.TermFilter; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.Mapper; @@ -160,6 +162,14 @@ public class BooleanFieldMapper extends AbstractFieldMapper { return "F"; } + @Override + public Filter nullValueFilter() { + if (nullValue == null) { + return null; + } + return new TermFilter(names().createIndexNameTerm(nullValue ? "T" : "F")); + } + @Override protected Field parseCreateField(ParseContext context) throws IOException { if (!indexed() && !stored()) { diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java index 1b32632adb2..bd1b206d8e2 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java @@ -197,6 +197,17 @@ public class ByteFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter nullValueFilter() { + if (nullValue == null) { + return null; + } + return NumericRangeFilter.newIntRange(names.indexName(), precisionStep, + nullValue.intValue(), + nullValue.intValue(), + true, true); + } + @Override protected boolean customBoost() { return true; diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 00772fae4d2..8ca00de8742 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -274,6 +274,19 @@ public class DateFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter nullValueFilter() { + if (nullValue == null) { + return null; + } + long value = parseStringValue(nullValue); + return NumericRangeFilter.newLongRange(names.indexName(), precisionStep, + value, + value, + true, true); + } + + @Override protected boolean customBoost() { return true; diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index 3b6be7fb6b9..f24772409ed 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -198,6 +198,17 @@ public class DoubleFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter nullValueFilter() { + if (nullValue == null) { + return null; + } + return NumericRangeFilter.newDoubleRange(names.indexName(), precisionStep, + nullValue, + nullValue, + true, true); + } + @Override protected boolean customBoost() { return true; diff --git a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java index 5a7ea22f573..d10c232c800 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java @@ -193,6 +193,17 @@ public class FloatFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter nullValueFilter() { + if (nullValue == null) { + return null; + } + return NumericRangeFilter.newFloatRange(names.indexName(), precisionStep, + nullValue, + nullValue, + true, true); + } + @Override protected boolean customBoost() { return true; diff --git a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java index 0e3da4131a8..b27886a5f2c 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java @@ -198,6 +198,17 @@ public class IntegerFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter nullValueFilter() { + if (nullValue == null) { + return null; + } + return NumericRangeFilter.newIntRange(names.indexName(), precisionStep, + nullValue, + nullValue, + true, true); + } + @Override protected boolean customBoost() { return true; diff --git a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java index c94e51a0f5c..7f4090420ff 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java @@ -198,6 +198,17 @@ public class LongFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter nullValueFilter() { + if (nullValue == null) { + return null; + } + return NumericRangeFilter.newLongRange(names.indexName(), precisionStep, + nullValue, + nullValue, + true, true); + } + @Override protected boolean customBoost() { return true; diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java index 3c52d1d378c..ce844270cf0 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java @@ -198,6 +198,17 @@ public class ShortFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter nullValueFilter() { + if (nullValue == null) { + return null; + } + return NumericRangeFilter.newIntRange(names.indexName(), precisionStep, + nullValue.intValue(), + nullValue.intValue(), + true, true); + } + @Override protected boolean customBoost() { return true; diff --git a/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java index 8302d0503d0..424d18fb7e7 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper.core; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.search.Filter; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -214,6 +215,14 @@ public class StringFieldMapper extends AbstractFieldMapper implements Al return this.searchQuotedAnalyzer; } + @Override + public Filter nullValueFilter() { + if (nullValue == null) { + return null; + } + return fieldFilter(nullValue, null); + } + @Override protected Field parseCreateField(ParseContext context) throws IOException { String value = nullValue; diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java index fd7751de785..be2f7b668c9 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java @@ -190,6 +190,17 @@ public class BoostFieldMapper extends NumberFieldMapper implements Intern includeLower, includeUpper); } + @Override + public Filter nullValueFilter() { + if (nullValue == null) { + return null; + } + return NumericRangeFilter.newFloatRange(names.indexName(), precisionStep, + nullValue, + nullValue, + true, true); + } + @Override public void preParse(ParseContext context) throws IOException { } diff --git a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java index aee5fb8d675..4a7f13c6491 100644 --- a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java @@ -226,6 +226,18 @@ public class IpFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter nullValueFilter() { + if (nullValue == null) { + return null; + } + final long value = ipToLong(nullValue); + return NumericRangeFilter.newLongRange(names.indexName(), precisionStep, + value, + value, + true, true); + } + @Override protected Fieldable parseCreateField(ParseContext context) throws IOException { String ipAsString; diff --git a/src/main/java/org/elasticsearch/index/query/MissingFilterBuilder.java b/src/main/java/org/elasticsearch/index/query/MissingFilterBuilder.java index 28ccd656b61..f41460b4a69 100644 --- a/src/main/java/org/elasticsearch/index/query/MissingFilterBuilder.java +++ b/src/main/java/org/elasticsearch/index/query/MissingFilterBuilder.java @@ -25,8 +25,6 @@ import java.io.IOException; /** * Constructs a filter that only match on documents that the field has a value in them. - * - * */ public class MissingFilterBuilder extends BaseFilterBuilder { @@ -34,10 +32,32 @@ public class MissingFilterBuilder extends BaseFilterBuilder { private String filterName; + private Boolean nullValue; + + private Boolean existence; + public MissingFilterBuilder(String name) { this.name = name; } + /** + * Should the missing filter automatically include fields with null value configured in the + * mappings. Defaults to false. + */ + public MissingFilterBuilder nullValue(boolean nullValue) { + this.nullValue = nullValue; + return this; + } + + /** + * Should hte missing filter include documents where the field doesn't exists in the docs. + * Defaults to true. + */ + public MissingFilterBuilder existence(boolean existence) { + this.existence = existence; + return this; + } + /** * Sets the filter name for the filter that can be used when searching for matched_filters per hit. */ @@ -46,11 +66,16 @@ public class MissingFilterBuilder extends BaseFilterBuilder { return this; } - @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(MissingFilterParser.NAME); builder.field("field", name); + if (nullValue != null) { + builder.field("null_value", nullValue); + } + if (existence != null) { + builder.field("existence", existence); + } if (filterName != null) { builder.field("_name", filterName); } diff --git a/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java b/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java index 5ec7254d812..109e8d22b3a 100644 --- a/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java @@ -23,6 +23,7 @@ import org.apache.lucene.search.Filter; import org.apache.lucene.search.TermRangeFilter; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.NotFilter; +import org.elasticsearch.common.lucene.search.XBooleanFilter; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MapperService; @@ -52,6 +53,8 @@ public class MissingFilterParser implements FilterParser { String fieldName = null; String filterName = null; + boolean nullValue = false; + boolean existence = true; XContentParser.Token token; String currentFieldName = null; @@ -61,6 +64,10 @@ public class MissingFilterParser implements FilterParser { } else if (token.isValue()) { if ("field".equals(currentFieldName)) { fieldName = parser.text(); + } else if ("null_value".equals(currentFieldName)) { + nullValue = parser.booleanValue(); + } else if ("existence".equals(currentFieldName)) { + existence = parser.booleanValue(); } else if ("_name".equals(currentFieldName)) { filterName = parser.text(); } else { @@ -70,27 +77,66 @@ public class MissingFilterParser implements FilterParser { } if (fieldName == null) { - throw new QueryParsingException(parseContext.index(), "exists must be provided with a [field]"); + throw new QueryParsingException(parseContext.index(), "missing must be provided with a [field]"); } - Filter filter = null; + if (!existence && !nullValue) { + throw new QueryParsingException(parseContext.index(), "missing must have either existence, or null_value, or both set to true"); + } + + Filter existenceFilter = null; + Filter nullFilter = null; + + MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); - if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { - filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true, parseContext); - } - if (filter == null) { - filter = new TermRangeFilter(fieldName, null, null, true, true); + + if (existence) { + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + existenceFilter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true, parseContext); + } + if (existenceFilter == null) { + existenceFilter = new TermRangeFilter(fieldName, null, null, true, true); + } + + // we always cache this one, really does not change... (exists) + existenceFilter = parseContext.cacheFilter(existenceFilter, null); + existenceFilter = new NotFilter(existenceFilter); + // cache the not filter as well, so it will be faster + existenceFilter = parseContext.cacheFilter(existenceFilter, null); } - // we always cache this one, really does not change... (exists) - filter = parseContext.cacheFilter(filter, null); - filter = new NotFilter(filter); - // cache the not filter as well, so it will be faster - filter = parseContext.cacheFilter(filter, null); + if (nullValue) { + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + nullFilter = smartNameFieldMappers.mapper().nullValueFilter(); + if (nullFilter != null) { + // cache the not filter as well, so it will be faster + nullFilter = parseContext.cacheFilter(nullFilter, null); + } + } + } + + Filter filter; + if (nullFilter != null) { + if (existenceFilter != null) { + XBooleanFilter combined = new XBooleanFilter(); + combined.addShould(existenceFilter); + combined.addShould(nullFilter); + // cache the not filter as well, so it will be faster + filter = parseContext.cacheFilter(combined, null); + } else { + filter = nullFilter; + } + } else { + filter = existenceFilter; + } + + if (filter == null) { + return null; + } filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext); if (filterName != null) { - parseContext.addNamedFilter(filterName, filter); + parseContext.addNamedFilter(filterName, existenceFilter); } return filter; }