Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-1368: add ms() and sub() functions 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@806374 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2009-08-20 22:05:09 +00:00
parent 9099b123d1
commit c49369b6c8
8 changed files with 378 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGES.txt
View File

@ -276,6 +276,10 @@ New Features
71. SOLR-1373: Add Filter query to admin/form.jsp
(Jason Rutherglen via hossman)
72. SOLR-1368: Add ms() function query for getting milliseconds from dates and for
high precision date subtraction, add sub() for subtracting other arguments.
(yonik)
Optimizations
----------------------
1. SOLR-374: Use IndexReader.reopen to save resources by re-using parts of the

85
src/java/org/apache/solr/schema/DateField.java
View File

@ -17,31 +17,26 @@
package org.apache.solr.schema;
import org.apache.solr.common.SolrException;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.search.function.*;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.DateUtil;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.TextResponseWriter;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.function.*;
import org.apache.solr.util.DateMathParser;
import java.util.Map;
import java.io.IOException;
import java.text.*;
import java.util.Date;
import java.util.TimeZone;
import java.util.Locale;
import java.text.DecimalFormatSymbols;
import java.text.SimpleDateFormat;
import java.text.DateFormat;
import java.text.NumberFormat;
import java.text.DecimalFormat;
import java.text.ParsePosition;
import java.text.ParseException;
import java.text.FieldPosition;
import java.util.Map;
import java.util.TimeZone;
// TODO: make a FlexibleDateField that can accept dates in multiple
// formats, better for human entered dates.
@ -262,6 +257,62 @@ public class DateField extends FieldType {
return fmtThreadLocal.get().parse(s);
}
/** Parse a date string in the standard format, or any supported by DateUtil.parseDate */
public Date parseDateLenient(String s, SolrQueryRequest req) throws ParseException {
// request could define timezone in the future
try {
return fmtThreadLocal.get().parse(s);
} catch (Exception e) {
return DateUtil.parseDate(s);
}
}
/**
* Parses a String which may be a date
* followed by an optional math expression.
* @param now an optional fixed date to use as "NOW" in the DateMathParser
* @param val the string to parse
*/
public Date parseMathLenient(Date now, String val, SolrQueryRequest req) {
String math = null;
final DateMathParser p = new DateMathParser(MATH_TZ, MATH_LOCALE);
if (null != now) p.setNow(now);
if (val.startsWith(NOW)) {
math = val.substring(NOW.length());
} else {
final int zz = val.indexOf(Z);
if (0 < zz) {
math = val.substring(zz+1);
try {
// p.setNow(toObject(val.substring(0,zz)));
p.setNow(parseDateLenient(val.substring(0,zz+1), req));
} catch (ParseException e) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
"Invalid Date in Date Math String:'"
+val+'\'',e);
}
} else {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
"Invalid Date String:'" +val+'\'');
}
}
if (null == math || math.equals("")) {
return p.getNow();
}
try {
return p.parseMath(math);
} catch (ParseException e) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
"Invalid Date Math String:'" +val+'\'',e);
}
}
/**
* Thread safe DateFormat that can <b>format</b> in the canonical
* ISO8601 date format, not including the trailing "Z" (since it is

2
src/java/org/apache/solr/schema/TrieDateField.java
View File

@ -42,7 +42,7 @@ import java.io.IOException;
public class TrieDateField extends DateField {
protected int precisionStepArg = TrieField.DEFAULT_PRECISION_STEP; // the one passed in or defaulted
protected int precisionStep; // normalized
protected int precisionStep = precisionStepArg; // normalized
@Override
protected void init(IndexSchema schema, Map<String, String> args) {

38
src/java/org/apache/solr/search/FunctionQParser.java
View File

@ -87,6 +87,44 @@ public class FunctionQParser extends QParser {
return value;
}
public String parseArg() throws ParseException {
sp.eatws();
char ch = sp.peek();
String val = null;
switch (ch) {
case ')': return null;
case '$':
sp.pos++;
String param = sp.getId();
val = getParam(param);
break;
case '\'':
case '"':
val = sp.getQuotedString();
break;
default:
// read unquoted literal ended by whitespace ',' or ')'
// there is no escaping.
int valStart = sp.pos;
for (;;) {
if (sp.pos >= sp.end) {
throw new ParseException("Missing end to unquoted value starting at " + valStart + " str='" + sp.val +"'");
}
char c = sp.val.charAt(sp.pos);
if (c==')' || c==',' || Character.isWhitespace(c)) {
val = sp.val.substring(valStart, sp.pos);
break;
}
sp.pos++;
}
}
sp.eatws();
consumeArgumentDelimiter();
return val;
}
/**
* Parse a list of ValueSource. Must be the final set of arguments
* to a ValueSource.

169
src/java/org/apache/solr/search/ValueSourceParser.java
View File

@ -19,12 +19,20 @@ package org.apache.solr.search;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Date;
import java.io.IOException;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.SolrException;
import org.apache.solr.search.function.*;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import org.apache.solr.schema.TrieDateField;
import org.apache.solr.schema.DateField;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.LegacyDateField;
/**
* A factory that parses user queries to generate ValueSource instances.
@ -231,6 +239,24 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin
public void init(NamedList args) {
}
});
standardValueSourceParsers.put("sub", new ValueSourceParser() {
public ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource a = fp.parseValueSource();
ValueSource b = fp.parseValueSource();
return new DualFloatFunction(a,b) {
protected String name() {
return "sub";
}
protected float func(int doc, DocValues aVals, DocValues bVals) {
return aVals.floatVal(doc) - bVals.floatVal(doc);
}
};
}
public void init(NamedList args) {
}
});
standardValueSourceParsers.put("query", new ValueSourceParser() {
// boost(query($q),rating)
@ -259,6 +285,149 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin
}
});
standardValueSourceParsers.put("ms", new DateValueSourceParser());
}
}
class DateValueSourceParser extends ValueSourceParser {
DateField df = new TrieDateField();
public void init(NamedList args) {}
public Date getDate(FunctionQParser fp, String arg) {
if (arg==null) return null;
if (arg.startsWith("NOW") || (arg.length()>0 && Character.isDigit(arg.charAt(0)))) {
return df.parseMathLenient(null, arg, fp.req);
}
return null;
}
public ValueSource getValueSource(FunctionQParser fp, String arg) {
if (arg==null) return null;
SchemaField f = fp.req.getSchema().getField(arg);
if (f.getType().getClass() == DateField.class || f.getType().getClass() == LegacyDateField.class) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can't use ms() function on non-numeric legacy date field " + arg);
}
return f.getType().getValueSource(f, fp);
}
public ValueSource parse(FunctionQParser fp) throws ParseException {
String first = fp.parseArg();
String second = fp.parseArg();
if (first==null) first="NOW";
Date d1=getDate(fp,first);
ValueSource v1 = d1==null ? getValueSource(fp, first) : null;
Date d2=getDate(fp,second);
ValueSource v2 = d2==null ? getValueSource(fp, second) : null;
// d constant
// v field
// dd constant
// dv subtract field from constant
// vd subtract constant from field
// vv subtract fields
final long ms1 = (d1 == null) ? 0 : d1.getTime();
final long ms2 = (d2 == null) ? 0 : d2.getTime();
// "d,dd" handle both constant cases
if (d1 != null && v2==null) {
return new LongConstValueSource(ms1-ms2);
}
// "v" just the date field
if (v1 != null && v2==null && d2==null) {
return v1;
}
// "dv"
if (d1!=null && v2!=null)
return new DualFloatFunction(new LongConstValueSource(ms1), v2) {
protected String name() {
return "ms";
}
protected float func(int doc, DocValues aVals, DocValues bVals) {
return ms1 - bVals.longVal(doc);
}
};
// "vd"
if (v1!=null && d2!=null)
return new DualFloatFunction(v1, new LongConstValueSource(ms2)) {
protected String name() {
return "ms";
}
protected float func(int doc, DocValues aVals, DocValues bVals) {
return aVals.longVal(doc) - ms2;
}
};
// "vv"
if (v1!=null && v2!=null)
return new DualFloatFunction(v1,v2) {
protected String name() {
return "ms";
}
protected float func(int doc, DocValues aVals, DocValues bVals) {
return aVals.longVal(doc) - bVals.longVal(doc);
}
};
return null; // shouldn't happen
}
}
// Private for now - we need to revisit how to handle typing in function queries
class LongConstValueSource extends ValueSource {
final long constant;
public LongConstValueSource(long constant) {
this.constant = constant;
}
public String description() {
return "const(" + constant + ")";
}
public DocValues getValues(IndexReader reader) throws IOException {
return new DocValues() {
public float floatVal(int doc) {
return constant;
}
public int intVal(int doc) {
return (int)constant;
}
public long longVal(int doc) {
return constant;
}
public double doubleVal(int doc) {
return constant;
}
public String strVal(int doc) {
return Long.toString(constant);
}
public String toString(int doc) {
return description();
}
};
}
public int hashCode() {
return (int)constant + (int)(constant>>>32);
}
public boolean equals(Object o) {
if (LongConstValueSource.class != o.getClass()) return false;
LongConstValueSource other = (LongConstValueSource)o;
return this.constant == other.constant;
}
}

84
src/java/org/apache/solr/search/function/DualFloatFunction.java Executable file
View File

@ -0,0 +1,84 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search.function;
import org.apache.lucene.index.IndexReader;
import java.io.IOException;
public abstract class DualFloatFunction extends ValueSource {
protected final ValueSource a;
protected final ValueSource b;
/**
* @param a the base.
* @param b the exponent.
*/
public DualFloatFunction(ValueSource a, ValueSource b) {
this.a = a;
this.b = b;
}
protected abstract String name();
protected abstract float func(int doc, DocValues aVals, DocValues bVals);
public String description() {
return name() + "(" + a.description() + "," + b.description() + ")";
}
public DocValues getValues(IndexReader reader) throws IOException {
final DocValues aVals = a.getValues(reader);
final DocValues bVals = b.getValues(reader);
return new DocValues() {
public float floatVal(int doc) {
return func(doc, aVals, bVals);
}
public int intVal(int doc) {
return (int)floatVal(doc);
}
public long longVal(int doc) {
return (long)floatVal(doc);
}
public double doubleVal(int doc) {
return floatVal(doc);
}
public String strVal(int doc) {
return Float.toString(floatVal(doc));
}
public String toString(int doc) {
return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
}
};
}
public int hashCode() {
int h = a.hashCode();
h ^= (h << 13) | (h >>> 20);
h += b.hashCode();
h ^= (h << 23) | (h >>> 10);
h += name().hashCode();
return h;
}
public boolean equals(Object o) {
if (this.getClass() != o.getClass()) return false;
DualFloatFunction other = (DualFloatFunction)o;
return this.a.equals(other.a)
&& this.b.equals(other.b);
}
}

61
src/java/org/apache/solr/search/function/PowFloatFunction.java
View File

@ -42,64 +42,3 @@ public class PowFloatFunction extends DualFloatFunction {
}
abstract class DualFloatFunction extends ValueSource {
protected final ValueSource a;
protected final ValueSource b;
/**
* @param a the base.
* @param b the exponent.
*/
public DualFloatFunction(ValueSource a, ValueSource b) {
this.a = a;
this.b = b;
}
protected abstract String name();
protected abstract float func(int doc, DocValues aVals, DocValues bVals);
public String description() {
return name() + "(" + a.description() + "," + b.description() + ")";
}
public DocValues getValues(IndexReader reader) throws IOException {
final DocValues aVals = a.getValues(reader);
final DocValues bVals = b.getValues(reader);
return new DocValues() {
public float floatVal(int doc) {
return func(doc, aVals, bVals);
}
public int intVal(int doc) {
return (int)floatVal(doc);
}
public long longVal(int doc) {
return (long)floatVal(doc);
}
public double doubleVal(int doc) {
return floatVal(doc);
}
public String strVal(int doc) {
return Float.toString(floatVal(doc));
}
public String toString(int doc) {
return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
}
};
}
public int hashCode() {
int h = a.hashCode();
h ^= (h << 13) | (h >>> 20);
h += b.hashCode();
h ^= (h << 23) | (h >>> 10);
h += name().hashCode();
return h;
}
public boolean equals(Object o) {
if (this.getClass() != o.getClass()) return false;
DualFloatFunction other = (DualFloatFunction)o;
return this.a.equals(other.a)
&& this.b.equals(other.b);
}
}

14
src/test/org/apache/solr/search/function/TestFunctionQuery.java
View File

@ -154,6 +154,8 @@ public class TestFunctionQuery extends AbstractSolrTestCase {
singleTest(field,"sum(\0,\0)", 10, 20);
singleTest(field,"sum(\0,\0,5)", 10, 25);
singleTest(field,"sub(\0,1)", 10, 9);
singleTest(field,"product(\0,1)", 10, 10);
singleTest(field,"product(\0,-2,-4)", 10, 80);
@ -276,7 +278,7 @@ public class TestFunctionQuery extends AbstractSolrTestCase {
}
public void testGeneral() {
assertU(adoc("id","1"));
assertU(adoc("id","1", "a_tdt","2009-08-31T12:10:10.123Z", "b_tdt","2009-08-31T12:10:10.124Z"));
assertU(adoc("id","2"));
assertU(commit()); // create more than one segment
assertU(adoc("id","3"));
@ -292,5 +294,15 @@ public class TestFunctionQuery extends AbstractSolrTestCase {
assertQ(req("fl","*,score","q", "{!func}top(ord(id))", "fq","id:6"), "//float[@name='score']='6.0'");
assertQ(req("fl","*,score","q", "{!func}rord(id)", "fq","id:1"),"//float[@name='score']='6.0'");
assertQ(req("fl","*,score","q", "{!func}top(rord(id))", "fq","id:1"),"//float[@name='score']='6.0'");
// test that we can subtract dates to millisecond precision
assertQ(req("fl","*,score","q", "{!func}ms(a_tdt,b_tdt)", "fq","id:1"), "//float[@name='score']='-1.0'");
assertQ(req("fl","*,score","q", "{!func}ms(b_tdt,a_tdt)", "fq","id:1"), "//float[@name='score']='1.0'");
assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z,2009-08-31T12:10:10.124Z)", "fq","id:1"), "//float[@name='score']='1.0'");
assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.124Z,a_tdt)", "fq","id:1"), "//float[@name='score']='1.0'");
assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z,b_tdt)", "fq","id:1"), "//float[@name='score']='1.0'");
assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z/SECOND,2009-08-31T12:10:10.124Z/SECOND)", "fq","id:1"), "//float[@name='score']='0.0'");
}
}