honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Search: add `format` support for date range filter and queries 

When the date format is defined in mapping, you can not use another format when querying using range date query or filter.

For example, this won't work:

```
DELETE /test

PUT /test/t/1
{
  "date": "2014-01-01"
}

GET /test/_search
{
  "query": {
    "filtered": {
      "filter": {
        "range": {
          "date": {
            "from": "01/01/2014"
          }
        }
      }
    }
  }
}
```

It causes:

```
Caused by: org.elasticsearch.ElasticsearchParseException: failed to parse date field [01/01/2014], tried both date format [dateOptionalTime], and timestamp number
```

It could be nice if we can support at query time another date format just like we support `analyzer` at search time on String fields.

Something like:

```
GET /test/_search
{
  "query": {
    "filtered": {
      "filter": {
        "range": {
          "date": {
            "from": "01/01/2014",
            "format": "dd/MM/yyyy"
          }
        }
      }
    }
  }
}
```

Same for queries:

```
GET /test/_search
{
  "query": {
    "range": {
      "date": {
        "from": "01/01/2014",
        "format": "dd/MM/yyyy"
      }
    }
  }
}
```

Closes #7189.
This commit is contained in:
David Pilato 2014-09-22 19:40:16 +02:00
parent 6cf371395a
commit 6ae6a078de
10 changed files with 228 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
docs/reference/query-dsl/filters/range-filter.asciidoc
View File

@ -30,6 +30,9 @@ The `range` filter accepts the following parameters:
`lte`:: Less-than or equal to
`lt`:: Less-than
[float]
==== Date options
When applied on `date` fields the `range` filter accepts also a `time_zone` parameter.
The `time_zone` parameter will be applied to your input lower and upper bounds and will
move them to UTC time based date:
@ -56,6 +59,28 @@ In the above example, `gte` will be actually moved to `2011-12-31T23:00:00` UTC
NOTE: if you give a date with a timezone explicitly defined and use the `time_zone` parameter, `time_zone` will be
ignored. For example, setting `from` to `2012-01-01T00:00:00+01:00` with `"time_zone":"+10:00"` will still use `+01:00` time zone.
coming[1.5.0,New feature added]
When applied on `date` fields the `range` filter accepts also a `format` parameter.
The `format` parameter will help support another date format than the one defined in mapping:
[source,js]
--------------------------------------------------
{
"constant_score": {
"filter": {
"range" : {
"born" : {
"gte": "01/01/2012",
"lte": "2013",
"format": "dd/MM/yyyy||yyyy"
}
}
}
}
}
--------------------------------------------------
[float]
==== Execution

20
docs/reference/query-dsl/queries/range-query.asciidoc
View File

@ -29,6 +29,9 @@ The `range` query accepts the following parameters:
`lt`:: Less-than
`boost`:: Sets the boost value of the query, defaults to `1.0`
[float]
==== Date options
When applied on `date` fields the `range` filter accepts also a `time_zone` parameter.
The `time_zone` parameter will be applied to your input lower and upper bounds and will
move them to UTC time based date:
@ -51,3 +54,20 @@ In the above example, `gte` will be actually moved to `2011-12-31T23:00:00` UTC
NOTE: if you give a date with a timezone explicitly defined and use the `time_zone` parameter, `time_zone` will be
ignored. For example, setting `from` to `2012-01-01T00:00:00+01:00` with `"time_zone":"+10:00"` will still use `+01:00` time zone.
coming[1.5.0,New feature added]
When applied on `date` fields the `range` query accepts also a `format` parameter.
The `format` parameter will help support another date format than the one defined in mapping:
[source,js]
--------------------------------------------------
{
"range" : {
"born" : {
"gte": "01/01/2012",
"lte": "2013",
"format": "dd/MM/yyyy||yyyy"
}
}
}
--------------------------------------------------

36
src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -300,19 +300,23 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
}
public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper) {
return parseToMilliseconds(value, context, includeUpper, null);
return parseToMilliseconds(value, context, includeUpper, null, dateMathParser);
}
public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone) {
public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
if (value instanceof Number) {
return ((Number) value).longValue();
}
return parseToMilliseconds(convertToString(value), context, includeUpper, zone);
return parseToMilliseconds(convertToString(value), context, includeUpper, zone, forcedDateParser);
}
public long parseToMilliseconds(String value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone) {
public long parseToMilliseconds(String value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
long now = context == null ? System.currentTimeMillis() : context.nowInMillis();
long time = includeUpper && roundCeil ? dateMathParser.parseRoundCeil(value, now, zone) : dateMathParser.parse(value, now, zone);
DateMathParser dateParser = dateMathParser;
if (forcedDateParser != null) {
dateParser = forcedDateParser;
}
long time = includeUpper && roundCeil ? dateParser.parseRoundCeil(value, now, zone) : dateParser.parse(value, now, zone);
return time;
}
@ -325,28 +329,28 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
@Override
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, null, context);
return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, null, null, context);
}
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable QueryParseContext context) {
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context) {
return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,
lowerTerm == null ? null : parseToMilliseconds(lowerTerm, context, false, timeZone),
upperTerm == null ? null : parseToMilliseconds(upperTerm, context, includeUpper, timeZone),
lowerTerm == null ? null : parseToMilliseconds(lowerTerm, context, false, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
upperTerm == null ? null : parseToMilliseconds(upperTerm, context, includeUpper, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
includeLower, includeUpper);
}
@Override
public Filter rangeFilter(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
return rangeFilter(lowerTerm, upperTerm, includeLower, includeUpper, null, context, null);
return rangeFilter(lowerTerm, upperTerm, includeLower, includeUpper, null, null, context, null);
}
public Filter rangeFilter(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) {
return rangeFilter(null, lowerTerm, upperTerm, includeLower, includeUpper, timeZone, context, explicitCaching);
public Filter rangeFilter(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) {
return rangeFilter(null, lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser, context, explicitCaching);
}
@Override
public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
return rangeFilter(parseContext, lowerTerm, upperTerm, includeLower, includeUpper, null, context, null);
return rangeFilter(parseContext, lowerTerm, upperTerm, includeLower, includeUpper, null, null, context, null);
}
/*
@ -355,7 +359,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
* - the object to parse is a String (does not apply to ms since epoch which are UTC based time values)
* - the String to parse does not have already a timezone defined (ie. `2014-01-01T00:00:00+03:00`)
*/
public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) {
public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) {
boolean cache;
boolean cacheable = true;
Long lowerVal = null;
@ -366,7 +370,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
} else {
String value = convertToString(lowerTerm);
cacheable = !hasDateExpressionWithNoRounding(value);
lowerVal = parseToMilliseconds(value, context, false, timeZone);
lowerVal = parseToMilliseconds(value, context, false, timeZone, forcedDateParser);
}
}
if (upperTerm != null) {
@ -375,7 +379,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
} else {
String value = convertToString(upperTerm);
cacheable = cacheable && !hasDateExpressionWithNoRounding(value);
upperVal = parseToMilliseconds(value, context, includeUpper, timeZone);
upperVal = parseToMilliseconds(value, context, includeUpper, timeZone, forcedDateParser);
}
}

8
src/main/java/org/elasticsearch/index/query/RangeFilterParser.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.search.Filter;
import org.apache.lucene.search.TermRangeFilter;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.joda.DateMathParser;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
@ -64,6 +65,7 @@ public class RangeFilterParser implements FilterParser {
boolean includeLower = true;
boolean includeUpper = true;
DateTimeZone timeZone = null;
DateMathParser forcedDateParser = null;
String execution = "index";
String filterName = null;
@ -100,6 +102,8 @@ public class RangeFilterParser implements FilterParser {
includeUpper = true;
} else if ("time_zone".equals(currentFieldName) || "timeZone".equals(currentFieldName)) {
timeZone = DateMathParser.parseZone(parser.text());
} else if ("format".equals(currentFieldName)) {
forcedDateParser = new DateMathParser(Joda.forPattern(parser.text()), DateFieldMapper.Defaults.TIME_UNIT);
} else {
throw new QueryParsingException(parseContext.index(), "[range] filter does not support [" + currentFieldName + "]");
}
@ -138,7 +142,7 @@ public class RangeFilterParser implements FilterParser {
if ((from instanceof Number || to instanceof Number) && timeZone != null) {
throw new QueryParsingException(parseContext.index(), "[range] time_zone when using ms since epoch format as it's UTC based can not be applied to [" + fieldName + "]");
}
filter = ((DateFieldMapper) mapper).rangeFilter(from, to, includeLower, includeUpper, timeZone, parseContext, explicitlyCached);
filter = ((DateFieldMapper) mapper).rangeFilter(from, to, includeLower, includeUpper, timeZone, forcedDateParser, parseContext, explicitlyCached);
} else {
if (timeZone != null) {
throw new QueryParsingException(parseContext.index(), "[range] time_zone can not be applied to non date field [" + fieldName + "]");
@ -157,7 +161,7 @@ public class RangeFilterParser implements FilterParser {
if ((from instanceof Number || to instanceof Number) && timeZone != null) {
throw new QueryParsingException(parseContext.index(), "[range] time_zone when using ms since epoch format as it's UTC based can not be applied to [" + fieldName + "]");
}
filter = ((DateFieldMapper) mapper).rangeFilter(parseContext, from, to, includeLower, includeUpper, timeZone, parseContext, explicitlyCached);
filter = ((DateFieldMapper) mapper).rangeFilter(parseContext, from, to, includeLower, includeUpper, timeZone, forcedDateParser, parseContext, explicitlyCached);
} else {
if (timeZone != null) {
throw new QueryParsingException(parseContext.index(), "[range] time_zone can not be applied to non date field [" + fieldName + "]");

6
src/main/java/org/elasticsearch/index/query/RangeQueryParser.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.joda.DateMathParser;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.FieldMapper;
@ -69,6 +70,7 @@ public class RangeQueryParser implements QueryParser {
boolean includeLower = true;
boolean includeUpper = true;
DateTimeZone timeZone = null;
DateMathParser forcedDateParser = null;
float boost = 1.0f;
String queryName = null;
@ -103,6 +105,8 @@ public class RangeQueryParser implements QueryParser {
timeZone = DateMathParser.parseZone(parser.text());
} else if ("_name".equals(currentFieldName)) {
queryName = parser.text();
} else if ("format".equals(currentFieldName)) {
forcedDateParser = new DateMathParser(Joda.forPattern(parser.text()), DateFieldMapper.Defaults.TIME_UNIT);
} else {
throw new QueryParsingException(parseContext.index(), "[range] query does not support [" + currentFieldName + "]");
}
@ -124,7 +128,7 @@ public class RangeQueryParser implements QueryParser {
if ((from instanceof Number || to instanceof Number) && timeZone != null) {
throw new QueryParsingException(parseContext.index(), "[range] time_zone when using ms since epoch format as it's UTC based can not be applied to [" + fieldName + "]");
}
query = ((DateFieldMapper) mapper).rangeQuery(from, to, includeLower, includeUpper, timeZone, parseContext);
query = ((DateFieldMapper) mapper).rangeQuery(from, to, includeLower, includeUpper, timeZone, forcedDateParser, parseContext);
} else {
if (timeZone != null) {
throw new QueryParsingException(parseContext.index(), "[range] time_zone can not be applied to non date field [" + fieldName + "]");

108
src/test/java/org/elasticsearch/index/query/IndexQueryParserFilterDateRangeFormatTests.java Normal file
View File

@ -0,0 +1,108 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.query;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.Query;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.test.ElasticsearchSingleNodeTest;
import org.joda.time.DateTime;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath;
import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
import static org.hamcrest.Matchers.*;
/**
*
*/
public class IndexQueryParserFilterDateRangeFormatTests extends ElasticsearchSingleNodeTest {
private Injector injector;
private IndexQueryParserService queryParser;
@Before
public void setup() throws IOException {
IndexService indexService = createIndex("test");
injector = indexService.injector();
MapperService mapperService = indexService.mapperService();
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/query/mapping.json");
mapperService.merge("person", new CompressedString(mapping), true);
mapperService.documentMapper("person").parse(new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/query/data.json")));
queryParser = injector.getInstance(IndexQueryParserService.class);
}
private IndexQueryParserService queryParser() throws IOException {
return this.queryParser;
}
@Test
public void testDateRangeFilterFormat() throws IOException {
IndexQueryParserService queryParser = queryParser();
String query = copyToStringFromClasspath("/org/elasticsearch/index/query/date_range_filter_format.json");
queryParser.parse(query).query();
// Sadly from NoCacheFilter, we can not access to the delegate filter so we can not check
// it's the one we are expecting
// Test Invalid format
query = copyToStringFromClasspath("/org/elasticsearch/index/query/date_range_filter_format_invalid.json");
try {
queryParser.parse(query).query();
fail("A Range Filter with a specific format but with an unexpected date should raise a QueryParsingException");
} catch (QueryParsingException e) {
// We expect it
}
}
@Test
public void testDateRangeQueryFormat() throws IOException {
IndexQueryParserService queryParser = queryParser();
// We test 01/01/2012 from gte and 2030 for lt
String query = copyToStringFromClasspath("/org/elasticsearch/index/query/date_range_query_format.json");
Query parsedQuery = queryParser.parse(query).query();
assertThat(parsedQuery, instanceOf(NumericRangeQuery.class));
// Min value was 01/01/2012 (dd/MM/yyyy)
DateTime min = DateTime.parse("2012-01-01T00:00:00.000+00");
assertThat(((NumericRangeQuery) parsedQuery).getMin().longValue(), is(min.getMillis()));
// Max value was 2030 (yyyy)
DateTime max = DateTime.parse("2030-01-01T00:00:00.000+00");
assertThat(((NumericRangeQuery) parsedQuery).getMax().longValue(), is(max.getMillis()));
// Test Invalid format
query = copyToStringFromClasspath("/org/elasticsearch/index/query/date_range_query_format_invalid.json");
try {
queryParser.parse(query).query();
fail("A Range Query with a specific format but with an unexpected date should raise a QueryParsingException");
} catch (QueryParsingException e) {
// We expect it
}
}
}

13
src/test/java/org/elasticsearch/index/query/date_range_filter_format.json Normal file
View File

@ -0,0 +1,13 @@
{
"constant_score": {
"filter": {
"range" : {
"born" : {
"gte": "01/01/2012",
"lt": "2030",
"format": "dd/MM/yyyy||yyyy"
}
}
}
}
}

13
src/test/java/org/elasticsearch/index/query/date_range_filter_format_invalid.json Normal file
View File

@ -0,0 +1,13 @@
{
"constant_score": {
"filter": {
"range" : {
"born" : {
"gte": "01/01/2012",
"lt": "2030",
"format": "yyyy"
}
}
}
}
}

9
src/test/java/org/elasticsearch/index/query/date_range_query_format.json Normal file
View File

@ -0,0 +1,9 @@
{
"range" : {
"born" : {
"gte": "01/01/2012",
"lt": "2030",
"format": "dd/MM/yyyy||yyyy"
}
}
}

9
src/test/java/org/elasticsearch/index/query/date_range_query_format_invalid.json Normal file
View File

@ -0,0 +1,9 @@
{
"range" : {
"born" : {
"gte": "01/01/2012",
"lt": "2030",
"format": "yyyy"
}
}
}