From 6ae6a078de3b197f8eb576b5385ff2fe7eb245a7 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 22 Sep 2014 19:40:16 +0200 Subject: [PATCH] Search: add `format` support for date range filter and queries When the date format is defined in mapping, you can not use another format when querying using range date query or filter. For example, this won't work: ``` DELETE /test PUT /test/t/1 { "date": "2014-01-01" } GET /test/_search { "query": { "filtered": { "filter": { "range": { "date": { "from": "01/01/2014" } } } } } } ``` It causes: ``` Caused by: org.elasticsearch.ElasticsearchParseException: failed to parse date field [01/01/2014], tried both date format [dateOptionalTime], and timestamp number ``` It could be nice if we can support at query time another date format just like we support `analyzer` at search time on String fields. Something like: ``` GET /test/_search { "query": { "filtered": { "filter": { "range": { "date": { "from": "01/01/2014", "format": "dd/MM/yyyy" } } } } } } ``` Same for queries: ``` GET /test/_search { "query": { "range": { "date": { "from": "01/01/2014", "format": "dd/MM/yyyy" } } } } ``` Closes #7189. --- .../query-dsl/filters/range-filter.asciidoc | 25 ++++ .../query-dsl/queries/range-query.asciidoc | 20 ++++ .../index/mapper/core/DateFieldMapper.java | 36 +++--- .../index/query/RangeFilterParser.java | 8 +- .../index/query/RangeQueryParser.java | 6 +- ...QueryParserFilterDateRangeFormatTests.java | 108 ++++++++++++++++++ .../index/query/date_range_filter_format.json | 13 +++ .../date_range_filter_format_invalid.json | 13 +++ .../index/query/date_range_query_format.json | 9 ++ .../date_range_query_format_invalid.json | 9 ++ 10 files changed, 228 insertions(+), 19 deletions(-) create mode 100644 src/test/java/org/elasticsearch/index/query/IndexQueryParserFilterDateRangeFormatTests.java create mode 100644 src/test/java/org/elasticsearch/index/query/date_range_filter_format.json create mode 100644 src/test/java/org/elasticsearch/index/query/date_range_filter_format_invalid.json create mode 100644 src/test/java/org/elasticsearch/index/query/date_range_query_format.json create mode 100644 src/test/java/org/elasticsearch/index/query/date_range_query_format_invalid.json diff --git a/docs/reference/query-dsl/filters/range-filter.asciidoc b/docs/reference/query-dsl/filters/range-filter.asciidoc index f612aaca73d..83c6ca7e040 100644 --- a/docs/reference/query-dsl/filters/range-filter.asciidoc +++ b/docs/reference/query-dsl/filters/range-filter.asciidoc @@ -30,6 +30,9 @@ The `range` filter accepts the following parameters: `lte`:: Less-than or equal to `lt`:: Less-than +[float] +==== Date options + When applied on `date` fields the `range` filter accepts also a `time_zone` parameter. The `time_zone` parameter will be applied to your input lower and upper bounds and will move them to UTC time based date: @@ -56,6 +59,28 @@ In the above example, `gte` will be actually moved to `2011-12-31T23:00:00` UTC NOTE: if you give a date with a timezone explicitly defined and use the `time_zone` parameter, `time_zone` will be ignored. For example, setting `from` to `2012-01-01T00:00:00+01:00` with `"time_zone":"+10:00"` will still use `+01:00` time zone. +coming[1.5.0,New feature added] + +When applied on `date` fields the `range` filter accepts also a `format` parameter. +The `format` parameter will help support another date format than the one defined in mapping: + +[source,js] +-------------------------------------------------- +{ + "constant_score": { + "filter": { + "range" : { + "born" : { + "gte": "01/01/2012", + "lte": "2013", + "format": "dd/MM/yyyy||yyyy" + } + } + } + } +} +-------------------------------------------------- + [float] ==== Execution diff --git a/docs/reference/query-dsl/queries/range-query.asciidoc b/docs/reference/query-dsl/queries/range-query.asciidoc index dbf07d1f0b6..39025483d31 100644 --- a/docs/reference/query-dsl/queries/range-query.asciidoc +++ b/docs/reference/query-dsl/queries/range-query.asciidoc @@ -29,6 +29,9 @@ The `range` query accepts the following parameters: `lt`:: Less-than `boost`:: Sets the boost value of the query, defaults to `1.0` +[float] +==== Date options + When applied on `date` fields the `range` filter accepts also a `time_zone` parameter. The `time_zone` parameter will be applied to your input lower and upper bounds and will move them to UTC time based date: @@ -51,3 +54,20 @@ In the above example, `gte` will be actually moved to `2011-12-31T23:00:00` UTC NOTE: if you give a date with a timezone explicitly defined and use the `time_zone` parameter, `time_zone` will be ignored. For example, setting `from` to `2012-01-01T00:00:00+01:00` with `"time_zone":"+10:00"` will still use `+01:00` time zone. +coming[1.5.0,New feature added] + +When applied on `date` fields the `range` query accepts also a `format` parameter. +The `format` parameter will help support another date format than the one defined in mapping: + +[source,js] +-------------------------------------------------- +{ + "range" : { + "born" : { + "gte": "01/01/2012", + "lte": "2013", + "format": "dd/MM/yyyy||yyyy" + } + } +} +-------------------------------------------------- diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 57ee70eccd9..93c7c1eb5a7 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -300,19 +300,23 @@ public class DateFieldMapper extends NumberFieldMapper { } public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper) { - return parseToMilliseconds(value, context, includeUpper, null); + return parseToMilliseconds(value, context, includeUpper, null, dateMathParser); } - public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone) { + public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) { if (value instanceof Number) { return ((Number) value).longValue(); } - return parseToMilliseconds(convertToString(value), context, includeUpper, zone); + return parseToMilliseconds(convertToString(value), context, includeUpper, zone, forcedDateParser); } - public long parseToMilliseconds(String value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone) { + public long parseToMilliseconds(String value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) { long now = context == null ? System.currentTimeMillis() : context.nowInMillis(); - long time = includeUpper && roundCeil ? dateMathParser.parseRoundCeil(value, now, zone) : dateMathParser.parse(value, now, zone); + DateMathParser dateParser = dateMathParser; + if (forcedDateParser != null) { + dateParser = forcedDateParser; + } + long time = includeUpper && roundCeil ? dateParser.parseRoundCeil(value, now, zone) : dateParser.parse(value, now, zone); return time; } @@ -325,28 +329,28 @@ public class DateFieldMapper extends NumberFieldMapper { @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) { - return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, null, context); + return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, null, null, context); } - public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable QueryParseContext context) { + public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context) { return NumericRangeQuery.newLongRange(names.indexName(), precisionStep, - lowerTerm == null ? null : parseToMilliseconds(lowerTerm, context, false, timeZone), - upperTerm == null ? null : parseToMilliseconds(upperTerm, context, includeUpper, timeZone), + lowerTerm == null ? null : parseToMilliseconds(lowerTerm, context, false, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser), + upperTerm == null ? null : parseToMilliseconds(upperTerm, context, includeUpper, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser), includeLower, includeUpper); } @Override public Filter rangeFilter(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) { - return rangeFilter(lowerTerm, upperTerm, includeLower, includeUpper, null, context, null); + return rangeFilter(lowerTerm, upperTerm, includeLower, includeUpper, null, null, context, null); } - public Filter rangeFilter(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) { - return rangeFilter(null, lowerTerm, upperTerm, includeLower, includeUpper, timeZone, context, explicitCaching); + public Filter rangeFilter(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) { + return rangeFilter(null, lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser, context, explicitCaching); } @Override public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) { - return rangeFilter(parseContext, lowerTerm, upperTerm, includeLower, includeUpper, null, context, null); + return rangeFilter(parseContext, lowerTerm, upperTerm, includeLower, includeUpper, null, null, context, null); } /* @@ -355,7 +359,7 @@ public class DateFieldMapper extends NumberFieldMapper { * - the object to parse is a String (does not apply to ms since epoch which are UTC based time values) * - the String to parse does not have already a timezone defined (ie. `2014-01-01T00:00:00+03:00`) */ - public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) { + public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) { boolean cache; boolean cacheable = true; Long lowerVal = null; @@ -366,7 +370,7 @@ public class DateFieldMapper extends NumberFieldMapper { } else { String value = convertToString(lowerTerm); cacheable = !hasDateExpressionWithNoRounding(value); - lowerVal = parseToMilliseconds(value, context, false, timeZone); + lowerVal = parseToMilliseconds(value, context, false, timeZone, forcedDateParser); } } if (upperTerm != null) { @@ -375,7 +379,7 @@ public class DateFieldMapper extends NumberFieldMapper { } else { String value = convertToString(upperTerm); cacheable = cacheable && !hasDateExpressionWithNoRounding(value); - upperVal = parseToMilliseconds(value, context, includeUpper, timeZone); + upperVal = parseToMilliseconds(value, context, includeUpper, timeZone, forcedDateParser); } } diff --git a/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java b/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java index 4edd45e5065..a07859d12a7 100644 --- a/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java @@ -23,6 +23,7 @@ import org.apache.lucene.search.Filter; import org.apache.lucene.search.TermRangeFilter; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.joda.DateMathParser; +import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.cache.filter.support.CacheKeyFilter; @@ -64,6 +65,7 @@ public class RangeFilterParser implements FilterParser { boolean includeLower = true; boolean includeUpper = true; DateTimeZone timeZone = null; + DateMathParser forcedDateParser = null; String execution = "index"; String filterName = null; @@ -100,6 +102,8 @@ public class RangeFilterParser implements FilterParser { includeUpper = true; } else if ("time_zone".equals(currentFieldName) || "timeZone".equals(currentFieldName)) { timeZone = DateMathParser.parseZone(parser.text()); + } else if ("format".equals(currentFieldName)) { + forcedDateParser = new DateMathParser(Joda.forPattern(parser.text()), DateFieldMapper.Defaults.TIME_UNIT); } else { throw new QueryParsingException(parseContext.index(), "[range] filter does not support [" + currentFieldName + "]"); } @@ -138,7 +142,7 @@ public class RangeFilterParser implements FilterParser { if ((from instanceof Number || to instanceof Number) && timeZone != null) { throw new QueryParsingException(parseContext.index(), "[range] time_zone when using ms since epoch format as it's UTC based can not be applied to [" + fieldName + "]"); } - filter = ((DateFieldMapper) mapper).rangeFilter(from, to, includeLower, includeUpper, timeZone, parseContext, explicitlyCached); + filter = ((DateFieldMapper) mapper).rangeFilter(from, to, includeLower, includeUpper, timeZone, forcedDateParser, parseContext, explicitlyCached); } else { if (timeZone != null) { throw new QueryParsingException(parseContext.index(), "[range] time_zone can not be applied to non date field [" + fieldName + "]"); @@ -157,7 +161,7 @@ public class RangeFilterParser implements FilterParser { if ((from instanceof Number || to instanceof Number) && timeZone != null) { throw new QueryParsingException(parseContext.index(), "[range] time_zone when using ms since epoch format as it's UTC based can not be applied to [" + fieldName + "]"); } - filter = ((DateFieldMapper) mapper).rangeFilter(parseContext, from, to, includeLower, includeUpper, timeZone, parseContext, explicitlyCached); + filter = ((DateFieldMapper) mapper).rangeFilter(parseContext, from, to, includeLower, includeUpper, timeZone, forcedDateParser, parseContext, explicitlyCached); } else { if (timeZone != null) { throw new QueryParsingException(parseContext.index(), "[range] time_zone can not be applied to non date field [" + fieldName + "]"); diff --git a/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java b/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java index 04054d9715e..83b8e93ffea 100644 --- a/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java @@ -23,6 +23,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermRangeQuery; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.joda.DateMathParser; +import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.FieldMapper; @@ -69,6 +70,7 @@ public class RangeQueryParser implements QueryParser { boolean includeLower = true; boolean includeUpper = true; DateTimeZone timeZone = null; + DateMathParser forcedDateParser = null; float boost = 1.0f; String queryName = null; @@ -103,6 +105,8 @@ public class RangeQueryParser implements QueryParser { timeZone = DateMathParser.parseZone(parser.text()); } else if ("_name".equals(currentFieldName)) { queryName = parser.text(); + } else if ("format".equals(currentFieldName)) { + forcedDateParser = new DateMathParser(Joda.forPattern(parser.text()), DateFieldMapper.Defaults.TIME_UNIT); } else { throw new QueryParsingException(parseContext.index(), "[range] query does not support [" + currentFieldName + "]"); } @@ -124,7 +128,7 @@ public class RangeQueryParser implements QueryParser { if ((from instanceof Number || to instanceof Number) && timeZone != null) { throw new QueryParsingException(parseContext.index(), "[range] time_zone when using ms since epoch format as it's UTC based can not be applied to [" + fieldName + "]"); } - query = ((DateFieldMapper) mapper).rangeQuery(from, to, includeLower, includeUpper, timeZone, parseContext); + query = ((DateFieldMapper) mapper).rangeQuery(from, to, includeLower, includeUpper, timeZone, forcedDateParser, parseContext); } else { if (timeZone != null) { throw new QueryParsingException(parseContext.index(), "[range] time_zone can not be applied to non date field [" + fieldName + "]"); diff --git a/src/test/java/org/elasticsearch/index/query/IndexQueryParserFilterDateRangeFormatTests.java b/src/test/java/org/elasticsearch/index/query/IndexQueryParserFilterDateRangeFormatTests.java new file mode 100644 index 00000000000..c4ffbf87774 --- /dev/null +++ b/src/test/java/org/elasticsearch/index/query/IndexQueryParserFilterDateRangeFormatTests.java @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.query; + + +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.inject.Injector; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.service.IndexService; +import org.elasticsearch.test.ElasticsearchSingleNodeTest; +import org.joda.time.DateTime; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath; +import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath; +import static org.hamcrest.Matchers.*; + +/** + * + */ +public class IndexQueryParserFilterDateRangeFormatTests extends ElasticsearchSingleNodeTest { + + private Injector injector; + private IndexQueryParserService queryParser; + + @Before + public void setup() throws IOException { + IndexService indexService = createIndex("test"); + injector = indexService.injector(); + + MapperService mapperService = indexService.mapperService(); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/query/mapping.json"); + mapperService.merge("person", new CompressedString(mapping), true); + mapperService.documentMapper("person").parse(new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/query/data.json"))); + queryParser = injector.getInstance(IndexQueryParserService.class); + } + + private IndexQueryParserService queryParser() throws IOException { + return this.queryParser; + } + + @Test + public void testDateRangeFilterFormat() throws IOException { + IndexQueryParserService queryParser = queryParser(); + String query = copyToStringFromClasspath("/org/elasticsearch/index/query/date_range_filter_format.json"); + queryParser.parse(query).query(); + // Sadly from NoCacheFilter, we can not access to the delegate filter so we can not check + // it's the one we are expecting + + // Test Invalid format + query = copyToStringFromClasspath("/org/elasticsearch/index/query/date_range_filter_format_invalid.json"); + try { + queryParser.parse(query).query(); + fail("A Range Filter with a specific format but with an unexpected date should raise a QueryParsingException"); + } catch (QueryParsingException e) { + // We expect it + } + } + + @Test + public void testDateRangeQueryFormat() throws IOException { + IndexQueryParserService queryParser = queryParser(); + // We test 01/01/2012 from gte and 2030 for lt + String query = copyToStringFromClasspath("/org/elasticsearch/index/query/date_range_query_format.json"); + Query parsedQuery = queryParser.parse(query).query(); + assertThat(parsedQuery, instanceOf(NumericRangeQuery.class)); + + // Min value was 01/01/2012 (dd/MM/yyyy) + DateTime min = DateTime.parse("2012-01-01T00:00:00.000+00"); + assertThat(((NumericRangeQuery) parsedQuery).getMin().longValue(), is(min.getMillis())); + + // Max value was 2030 (yyyy) + DateTime max = DateTime.parse("2030-01-01T00:00:00.000+00"); + assertThat(((NumericRangeQuery) parsedQuery).getMax().longValue(), is(max.getMillis())); + + // Test Invalid format + query = copyToStringFromClasspath("/org/elasticsearch/index/query/date_range_query_format_invalid.json"); + try { + queryParser.parse(query).query(); + fail("A Range Query with a specific format but with an unexpected date should raise a QueryParsingException"); + } catch (QueryParsingException e) { + // We expect it + } + } +} diff --git a/src/test/java/org/elasticsearch/index/query/date_range_filter_format.json b/src/test/java/org/elasticsearch/index/query/date_range_filter_format.json new file mode 100644 index 00000000000..94596788a23 --- /dev/null +++ b/src/test/java/org/elasticsearch/index/query/date_range_filter_format.json @@ -0,0 +1,13 @@ +{ + "constant_score": { + "filter": { + "range" : { + "born" : { + "gte": "01/01/2012", + "lt": "2030", + "format": "dd/MM/yyyy||yyyy" + } + } + } + } +} diff --git a/src/test/java/org/elasticsearch/index/query/date_range_filter_format_invalid.json b/src/test/java/org/elasticsearch/index/query/date_range_filter_format_invalid.json new file mode 100644 index 00000000000..7b5c2724429 --- /dev/null +++ b/src/test/java/org/elasticsearch/index/query/date_range_filter_format_invalid.json @@ -0,0 +1,13 @@ +{ + "constant_score": { + "filter": { + "range" : { + "born" : { + "gte": "01/01/2012", + "lt": "2030", + "format": "yyyy" + } + } + } + } +} diff --git a/src/test/java/org/elasticsearch/index/query/date_range_query_format.json b/src/test/java/org/elasticsearch/index/query/date_range_query_format.json new file mode 100644 index 00000000000..f679dc9696f --- /dev/null +++ b/src/test/java/org/elasticsearch/index/query/date_range_query_format.json @@ -0,0 +1,9 @@ +{ + "range" : { + "born" : { + "gte": "01/01/2012", + "lt": "2030", + "format": "dd/MM/yyyy||yyyy" + } + } +} diff --git a/src/test/java/org/elasticsearch/index/query/date_range_query_format_invalid.json b/src/test/java/org/elasticsearch/index/query/date_range_query_format_invalid.json new file mode 100644 index 00000000000..307e9775e50 --- /dev/null +++ b/src/test/java/org/elasticsearch/index/query/date_range_query_format_invalid.json @@ -0,0 +1,9 @@ +{ + "range" : { + "born" : { + "gte": "01/01/2012", + "lt": "2030", + "format": "yyyy" + } + } +}