Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-6827: DateRangeField support for facet.range, exclusive ranges, DateMath 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1645383 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
David Wayne Smiley 2014-12-14 04:55:26 +00:00
parent 0ddb7567f1
commit 85a43a77a9
7 changed files with 200 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
solr/CHANGES.txt
View File

@ -136,8 +136,9 @@ New Features
modifying solr configuration files. (Erick Erickson)
- SOLR-5539: Admin UI - Remove ability to create/modify files (steffkes)
* SOLR-6103: Added DateRangeField for indexing date ranges, especially
multi-valued ones. Based on LUCENE-5648. (David Smiley)
* SOLR-6103: Added DateRangeField for indexing date ranges, especially multi-valued ones.
Supports facet.range, DateMath, and is mostly interoperable with TrieDateField.
Based on LUCENE-5648. (David Smiley)
* SOLR-6403: TransactionLog replay status logging. (Mark Miller)

40
solr/core/src/java/org/apache/solr/request/SimpleFacets.java
View File

@ -37,10 +37,10 @@ import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
@ -74,6 +74,7 @@ import org.apache.solr.common.util.StrUtils;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.request.IntervalFacets.FacetInterval;
import org.apache.solr.schema.BoolField;
import org.apache.solr.schema.DateRangeField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
@ -1080,6 +1081,8 @@ public class SimpleFacets {
(SolrException.ErrorCode.BAD_REQUEST,
"Unable to range facet on tried field of unexpected type:" + f);
}
} else if (ft instanceof DateRangeField) {
calc = new DateRangeFieldEndpointCalculator(sf, null);
} else {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
@ -1420,6 +1423,7 @@ public class SimpleFacets {
}
private static class DateRangeEndpointCalculator
extends RangeEndpointCalculator<Date> {
private static final String TYPE_ERR_MSG = "SchemaField must use field type extending TrieDateField or DateRangeField";
private final Date now;
public DateRangeEndpointCalculator(final SchemaField f,
final Date now) {
@ -1427,7 +1431,7 @@ public class SimpleFacets {
this.now = now;
if (! (field.getType() instanceof TrieDateField) ) {
throw new IllegalArgumentException
("SchemaField must use field type extending TrieDateField");
(TYPE_ERR_MSG);
}
}
@Override
@ -1449,6 +1453,36 @@ public class SimpleFacets {
return dmp.parseMath(gap);
}
}
private static class DateRangeFieldEndpointCalculator
extends RangeEndpointCalculator<Date> {
private final Date now;
public DateRangeFieldEndpointCalculator(final SchemaField f,
final Date now) {
super(f);
this.now = now;
if (! (field.getType() instanceof DateRangeField) ) {
throw new IllegalArgumentException(DateRangeEndpointCalculator.TYPE_ERR_MSG);
}
}
@Override
public String formatValue(Date val) {
return TrieDateField.formatExternal(val);
}
@Override
protected Date parseVal(String rawval) {
return ((DateRangeField)field.getType()).parseMath(now, rawval);
}
@Override
protected Object parseGap(final String rawval) {
return rawval;
}
@Override
public Date parseAndAddGap(Date value, String gap) throws java.text.ParseException {
final DateMathParser dmp = new DateMathParser();
dmp.setNow(value);
return dmp.parseMath(gap);
}
}
/**
* Returns a <code>NamedList</code> with each entry having the "key" of the interval as name and the count of docs

37
solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
View File

@ -17,6 +17,19 @@ package org.apache.solr.schema;
* limitations under the License.
*/
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
@ -51,19 +64,6 @@ import org.apache.solr.util.SpatialUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
/**
* Abstract base class for Solr FieldTypes based on a Lucene 4 {@link SpatialStrategy}.
*
@ -159,7 +159,7 @@ public abstract class AbstractSpatialFieldType<T extends SpatialStrategy> extend
@Override
public List<StorableField> createFields(SchemaField field, Object val, float boost) {
String shapeStr = null;
Shape shape = null;
Shape shape;
if (val instanceof Shape) {
shape = ((Shape) val);
} else {
@ -178,14 +178,17 @@ public abstract class AbstractSpatialFieldType<T extends SpatialStrategy> extend
}
if (field.stored()) {
if (shapeStr == null)
shapeStr = shapeToString(shape);
result.add(new StoredField(field.getName(), shapeStr));
result.add(new StoredField(field.getName(), getStoredValue(shape, shapeStr)));
}
return result;
}
/** Called by {@link #createFields(SchemaField, Object, float)} to get the stored value. */
protected String getStoredValue(Shape shape, String shapeStr) {
return (shapeStr == null) ? shapeToString(shape) : shapeStr;
}
protected Shape parseShape(String str) {
if (str.length() == 0)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "empty string shape");

113
solr/core/src/java/org/apache/solr/schema/DateRangeField.java
View File

@ -34,9 +34,13 @@ import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialOperation;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SyntaxError;
/**
* A field for indexed dates and date ranges. It's mostly compatible with TrieDateField.
*
* @see NumberRangePrefixTreeStrategy
* @see DateRangePrefixTree
*/
@ -44,7 +48,9 @@ public class DateRangeField extends AbstractSpatialPrefixTreeFieldType<NumberRan
private static final String OP_PARAM = "op";//local-param to resolve SpatialOperation
private final DateRangePrefixTree tree = DateRangePrefixTree.INSTANCE;
private static final DateRangePrefixTree tree = DateRangePrefixTree.INSTANCE;
private static final TrieDateField trieDateField = new TrieDateField();//used for utility methods
@Override
protected void init(IndexSchema schema, Map<String, String> args) {
@ -63,24 +69,74 @@ public class DateRangeField extends AbstractSpatialPrefixTreeFieldType<NumberRan
@Override
public List<StorableField> createFields(SchemaField field, Object val, float boost) {
if (val instanceof Date || val instanceof Calendar)//From URP
if (val instanceof Date || val instanceof Calendar)//From URP?
val = tree.toUnitShape(val);
return super.createFields(field, val, boost);
}
@Override
protected NRShape parseShape(String str) {
try {
return tree.parseShape(str);
} catch (ParseException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Couldn't parse date because: "+ e.getMessage(), e);
protected String getStoredValue(Shape shape, String shapeStr) {
if (shape instanceof UnitNRShape) {
UnitNRShape unitShape = (UnitNRShape) shape;
if (unitShape.getLevel() == tree.getMaxLevels()) {
//fully precise date. We can be fully compatible with TrieDateField.
Date date = tree.toCalendar(unitShape).getTime();
return TrieDateField.formatExternal(date);
}
}
return (shapeStr == null ? shape.toString() : shapeStr);//we don't normalize ranges here; should we?
}
@Override
protected NRShape parseShape(String str) {
if (str.contains(" TO ")) {
//TODO parsing range syntax doesn't support DateMath on either side or exclusive/inclusive
try {
return tree.parseShape(str);
} catch (ParseException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Couldn't parse date because: "+ e.getMessage(), e);
}
} else {
return tree.toShape(parseCalendar(str));
}
}
private Calendar parseCalendar(String str) {
if (str.startsWith("NOW") || str.lastIndexOf('Z') >= 0) {
//use Solr standard date format parsing rules.
//TODO parse a Calendar instead of a Date, rounded according to DateMath syntax.
Date date = trieDateField.parseMath(null, str);
Calendar cal = tree.newCal();
cal.setTime(date);
return cal;
} else {
try {
return tree.parseCalendar(str);
} catch (ParseException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Couldn't parse date because: "+ e.getMessage(), e);
}
}
}
/** For easy compatibility with {@link TrieDateField#parseMath(Date, String)}. */
public Date parseMath(Date now, String rawval) {
return trieDateField.parseMath(now, rawval);
}
@Override
protected String shapeToString(Shape shape) {
return shape.toString();//generally round-trips for DateRangePrefixTree
if (shape instanceof UnitNRShape) {
UnitNRShape unitShape = (UnitNRShape) shape;
if (unitShape.getLevel() == tree.getMaxLevels()) {
//fully precise date. We can be fully compatible with TrieDateField.
Date date = tree.toCalendar(unitShape).getTime();
return TrieDateField.formatExternal(date);
}
}
return shape.toString();//range shape
}
@Override
@ -98,15 +154,38 @@ public class DateRangeField extends AbstractSpatialPrefixTreeFieldType<NumberRan
}
@Override
public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
if (!minInclusive || !maxInclusive)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "exclusive range boundary not supported");
if (part1 == null)
part1 = "*";
if (part2 == null)
part2 = "*";
Shape shape = tree.toRangeShape((UnitNRShape) parseShape(part1), (UnitNRShape) parseShape(part2));
public Query getRangeQuery(QParser parser, SchemaField field, String startStr, String endStr, boolean minInclusive, boolean maxInclusive) {
if (parser == null) {//null when invoked by SimpleFacets. But getQueryFromSpatialArgs expects to get localParams.
final SolrRequestInfo requestInfo = SolrRequestInfo.getRequestInfo();
parser = new QParser("", null, requestInfo.getReq().getParams(), requestInfo.getReq()) {
@Override
public Query parse() throws SyntaxError {
throw new IllegalStateException();
}
};
}
Calendar startCal;
if (startStr == null) {
startCal = tree.newCal();
} else {
startCal = parseCalendar(startStr);
if (!minInclusive) {
startCal.add(Calendar.MILLISECOND, 1);
}
}
Calendar endCal;
if (endStr == null) {
endCal = tree.newCal();
} else {
endCal = parseCalendar(endStr);
if (!maxInclusive) {
endCal.add(Calendar.MILLISECOND, -1);
}
}
Shape shape = tree.toRangeShape(tree.toShape(startCal), tree.toShape(endCal));
SpatialArgs spatialArgs = new SpatialArgs(SpatialOperation.Intersects, shape);
return getQueryFromSpatialArgs(parser, field, spatialArgs);
}
}

1
solr/core/src/test-files/solr/collection1/conf/schema.xml
View File

@ -575,6 +575,7 @@
<dynamicField name="*_tdt" type="tdate" indexed="true" stored="true"/>
<dynamicField name="*_tdt1" type="tdate" indexed="true" stored="true" multiValued="false"/>
<dynamicField name="*_tdtdv" type="tdatedv" indexed="true" stored="true"/>
<dynamicField name="*_drf" type="dateRange" indexed="true" stored="true"/>
<dynamicField name="*_sI" type="string" indexed="true" stored="false"/>
<dynamicField name="*_sS" type="string" indexed="false" stored="true"/>

41
solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java
View File

@ -17,7 +17,12 @@
package org.apache.solr.request;
import org.noggit.ObjectBuilder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
@ -25,10 +30,7 @@ import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.TimeZoneUtils;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Map;
import org.noggit.ObjectBuilder;
public class SimpleFacetsTest extends SolrTestCaseJ4 {
@ -53,6 +55,20 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 {
// committing randomly gives different looking segments each time
static void add_doc(String... fieldsAndValues) {
do {
//do our own copy-field:
List<String> fieldsAndValuesList = new ArrayList<>(Arrays.asList(fieldsAndValues));
int idx = fieldsAndValuesList.indexOf("a_tdt");
if (idx >= 0) {
fieldsAndValuesList.add("a_drf");
fieldsAndValuesList.add(fieldsAndValuesList.get(idx + 1));//copy
}
idx = fieldsAndValuesList.indexOf("bday");
if (idx >= 0) {
fieldsAndValuesList.add("bday_drf");
fieldsAndValuesList.add(fieldsAndValuesList.get(idx + 1));//copy
}
fieldsAndValues = fieldsAndValuesList.toArray(new String[fieldsAndValuesList.size()]);
pendingDocs.add(fieldsAndValues);
} while (random().nextInt(100) <= random_dupe_percent);
@ -690,6 +706,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 {
final String ooo = "00:00:00.000Z";
final String xxx = "15:15:15.155Z";
//note: add_doc duplicates bday to bday_drf and a_tdt to a_drf (date range field)
add_doc(i, "201", f, "1976-07-04T12:08:56.235Z", ff, "1900-01-01T"+ooo);
add_doc(i, "202", f, "1976-07-05T00:00:00.000Z", ff, "1976-07-01T"+ooo);
add_doc(i, "203", f, "1976-07-15T00:07:67.890Z", ff, "1976-07-04T"+ooo);
@ -716,6 +733,11 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 {
helpTestDateFacets("bday", true);
}
@Test
public void testDateRangeFieldFacets() {
helpTestDateFacets("bday_drf", true);
}
private void helpTestDateFacets(final String fieldName,
final boolean rangeMode) {
final String p = rangeMode ? "facet.range" : "facet.date";
@ -913,8 +935,13 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 {
helpTestDateFacetsWithIncludeOption("a_tdt", true);
}
/** similar to helpTestDateFacets, but for differnet fields with test data
exactly on on boundary marks */
@Test
public void testDateRangeFieldDateRangeFacetsWithIncludeOption() {
helpTestDateFacetsWithIncludeOption("a_drf", true);
}
/** Similar to helpTestDateFacets, but for different fields with test data
exactly on boundary marks */
private void helpTestDateFacetsWithIncludeOption(final String fieldName,
final boolean rangeMode) {
final String p = rangeMode ? "facet.range" : "facet.date";

13
solr/core/src/test/org/apache/solr/schema/DateRangeFieldTest.java
View File

@ -3,7 +3,7 @@ package org.apache.solr.schema;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* this work for additional information regarding copyright ownership.NRP
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
@ -33,19 +33,24 @@ public class DateRangeFieldTest extends SolrTestCaseJ4 {
assertU(adoc("id", "0", "dateRange", "[* TO *]"));
assertU(adoc("id", "1", "dateRange", "2014-05-21T12:00:00.000Z"));
assertU(adoc("id", "2", "dateRange", "[2000 TO 2014-05-21]"));
assertU(adoc("id", "3", "dateRange", "2020-05-21T12:00:00.000Z/DAY"));//DateMath syntax
assertU(commit());
//ensure stored value is the same (not toString of Shape)
assertQ(req("q", "id:1", "fl", "dateRange"), "//result/doc/arr[@name='dateRange']/str[.='2014-05-21T12:00:00.000Z']");
//ensure stored value resolves datemath
assertQ(req("q", "id:1", "fl", "dateRange"), "//result/doc/arr[@name='dateRange']/str[.='2014-05-21T12:00:00Z']");//no 000 ms
assertQ(req("q", "id:2", "fl", "dateRange"), "//result/doc/arr[@name='dateRange']/str[.='[2000 TO 2014-05-21]']");//a range; same
assertQ(req("q", "id:3", "fl", "dateRange"), "//result/doc/arr[@name='dateRange']/str[.='2020-05-21T00:00:00Z']");//resolve datemath
String[] commonParams = {"q", "{!field f=dateRange op=$op v=$qq}", "sort", "id asc"};
assertQ(req(commonParams, "qq", "[* TO *]"), xpathMatches(0, 1, 2));
assertQ(req(commonParams, "qq", "[* TO *]"), xpathMatches(0, 1, 2, 3));
assertQ(req(commonParams, "qq", "2012"), xpathMatches(0, 2));
assertQ(req(commonParams, "qq", "2013", "op", "Contains"), xpathMatches(0, 2));
assertQ(req(commonParams, "qq", "2014", "op", "Contains"), xpathMatches(0));
assertQ(req(commonParams, "qq", "[1999 TO 2001]", "op", "IsWithin"), xpathMatches());
assertQ(req(commonParams, "qq", "2014-05", "op", "IsWithin"), xpathMatches(1));
assertQ(req("q", "dateRange:[1998 TO 2000}"), xpathMatches(0));//exclusive end, so we barely miss one doc
//show without local-params
assertQ(req("q", "dateRange:\"2014-05-21T12:00:00.000Z\""), xpathMatches(0, 1, 2));
assertQ(req("q", "dateRange:[1999 TO 2001]"), xpathMatches(0, 2));