SOLR-1368: add ms() and sub() functions

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@806374 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -276,6 +276,10 @@ New Features
 71. SOLR-1373: Add Filter query to admin/form.jsp
     (Jason Rutherglen via hossman)
 
+72. SOLR-1368: Add ms() function query for getting milliseconds from dates and for 
+    high precision date subtraction, add sub() for subtracting other arguments.
+    (yonik)
+
 Optimizations
 ----------------------
  1. SOLR-374: Use IndexReader.reopen to save resources by re-using parts of the

src/java/org/apache/solr/schema/DateField.java

@@ -17,31 +17,26 @@
 
 package org.apache.solr.schema;
 
-import org.apache.solr.common.SolrException;
-import org.apache.solr.request.XMLWriter;
-import org.apache.solr.request.TextResponseWriter;
 import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.index.IndexReader;
-import org.apache.solr.search.function.*;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermRangeQuery;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.DateUtil;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.TextResponseWriter;
+import org.apache.solr.request.XMLWriter;
 import org.apache.solr.search.QParser;
+import org.apache.solr.search.function.*;
 import org.apache.solr.util.DateMathParser;
-  
+
-import java.util.Map;
 import java.io.IOException;
+import java.text.*;
 import java.util.Date;
-import java.util.TimeZone;
 import java.util.Locale;
-import java.text.DecimalFormatSymbols;
+import java.util.Map;
-import java.text.SimpleDateFormat;
+import java.util.TimeZone;
-import java.text.DateFormat;
-import java.text.NumberFormat;
-import java.text.DecimalFormat;
-import java.text.ParsePosition;
-import java.text.ParseException;
-import java.text.FieldPosition;
 
 // TODO: make a FlexibleDateField that can accept dates in multiple
 // formats, better for human entered dates.
@@ -261,6 +256,62 @@ public class DateField extends FieldType {
    protected Date parseDate(String s) throws ParseException {
      return fmtThreadLocal.get().parse(s);
    }
+
+  /** Parse a date string in the standard format, or any supported by DateUtil.parseDate */
+   public Date parseDateLenient(String s, SolrQueryRequest req) throws ParseException {
+     // request could define timezone in the future
+     try {
+       return fmtThreadLocal.get().parse(s);
+     } catch (Exception e) {
+       return DateUtil.parseDate(s);
+     }
+   }
+
+  /**
+   * Parses a String which may be a date
+   * followed by an optional math expression.
+   * @param now an optional fixed date to use as "NOW" in the DateMathParser
+   * @param val the string to parse
+   */
+  public Date parseMathLenient(Date now, String val, SolrQueryRequest req) {
+    String math = null;
+    final DateMathParser p = new DateMathParser(MATH_TZ, MATH_LOCALE);
+
+    if (null != now) p.setNow(now);
+
+    if (val.startsWith(NOW)) {
+      math = val.substring(NOW.length());
+    } else {
+      final int zz = val.indexOf(Z);
+      if (0 < zz) {
+        math = val.substring(zz+1);
+        try {
+          // p.setNow(toObject(val.substring(0,zz)));
+          p.setNow(parseDateLenient(val.substring(0,zz+1), req));
+        } catch (ParseException e) {
+          throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
+                                   "Invalid Date in Date Math String:'"
+                                   +val+'\'',e);
+        }
+      } else {
+        throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
+                                 "Invalid Date String:'" +val+'\'');
+      }
+    }
+
+    if (null == math || math.equals("")) {
+      return p.getNow();
+    }
+
+    try {
+      return p.parseMath(math);
+    } catch (ParseException e) {
+      throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
+                               "Invalid Date Math String:'" +val+'\'',e);
+    }
+  }
+
+
   
   /**
    * Thread safe DateFormat that can <b>format</b> in the canonical

src/java/org/apache/solr/schema/TrieDateField.java

@@ -42,7 +42,7 @@ import java.io.IOException;
 
 public class TrieDateField extends DateField {
   protected int precisionStepArg = TrieField.DEFAULT_PRECISION_STEP;  // the one passed in or defaulted
-  protected int precisionStep;     // normalized
+  protected int precisionStep = precisionStepArg;     // normalized
 
   @Override
   protected void init(IndexSchema schema, Map<String, String> args) {

src/java/org/apache/solr/search/FunctionQParser.java

@@ -86,6 +86,44 @@ public class FunctionQParser extends QParser {
     consumeArgumentDelimiter();
     return value;
   }
+
+  public String parseArg() throws ParseException {
+    sp.eatws();
+    char ch = sp.peek();
+    String val = null;
+    switch (ch) {
+      case ')': return null;
+      case '$':
+        sp.pos++;
+        String param = sp.getId();
+        val = getParam(param);
+        break;
+      case '\'':
+      case '"':
+        val = sp.getQuotedString();
+        break;
+      default:
+        // read unquoted literal ended by whitespace ',' or ')'
+        // there is no escaping.
+        int valStart = sp.pos;
+        for (;;) {
+          if (sp.pos >= sp.end) {
+            throw new ParseException("Missing end to unquoted value starting at " + valStart + " str='" + sp.val +"'");
+          }
+          char c = sp.val.charAt(sp.pos);
+          if (c==')' || c==',' || Character.isWhitespace(c)) {
+            val = sp.val.substring(valStart, sp.pos);
+            break;
+          }
+          sp.pos++;
+        }
+    }
+
+    sp.eatws();
+    consumeArgumentDelimiter();
+    return val;
+  }
+
   
   /**
    * Parse a list of ValueSource.  Must be the final set of arguments

src/java/org/apache/solr/search/ValueSourceParser.java

@@ -19,12 +19,20 @@ package org.apache.solr.search;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Date;
+import java.io.IOException;
 
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.index.IndexReader;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.SolrException;
 import org.apache.solr.search.function.*;
 import org.apache.solr.util.plugin.NamedListInitializedPlugin;
+import org.apache.solr.schema.TrieDateField;
+import org.apache.solr.schema.DateField;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.schema.LegacyDateField;
 
 /**
  * A factory that parses user queries to generate ValueSource instances.
@@ -231,6 +239,24 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin
       public void init(NamedList args) {
       }
       
+    });
+    standardValueSourceParsers.put("sub", new ValueSourceParser() {
+      public ValueSource parse(FunctionQParser fp) throws ParseException {
+        ValueSource a = fp.parseValueSource();
+        ValueSource b = fp.parseValueSource();
+        return new DualFloatFunction(a,b) {
+          protected String name() {
+            return "sub";
+          }
+          protected float func(int doc, DocValues aVals, DocValues bVals) {
+            return aVals.floatVal(doc) - bVals.floatVal(doc);
+          }
+        };
+      }
+
+      public void init(NamedList args) {
+      }
+
     });
     standardValueSourceParsers.put("query", new ValueSourceParser() {
       // boost(query($q),rating)
@@ -259,6 +285,149 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin
       }
       
     });
+    standardValueSourceParsers.put("ms", new DateValueSourceParser());
   }
 
 }
+
+
+
+
+class DateValueSourceParser extends ValueSourceParser {
+  DateField df = new TrieDateField();
+  public void init(NamedList args) {}
+
+  public Date getDate(FunctionQParser fp, String arg) {
+    if (arg==null) return null;
+    if (arg.startsWith("NOW") || (arg.length()>0 && Character.isDigit(arg.charAt(0)))) {
+      return df.parseMathLenient(null, arg, fp.req);
+    }
+    return null;
+  }
+
+  public ValueSource getValueSource(FunctionQParser fp, String arg) {
+    if (arg==null) return null;
+    SchemaField f = fp.req.getSchema().getField(arg);
+    if (f.getType().getClass() == DateField.class || f.getType().getClass() == LegacyDateField.class) {
+      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can't use ms() function on non-numeric legacy date field " + arg);
+    }
+    return f.getType().getValueSource(f, fp);
+  }
+
+  public ValueSource parse(FunctionQParser fp) throws ParseException {
+    String first = fp.parseArg();
+    String second = fp.parseArg();
+    if (first==null) first="NOW";
+
+    Date d1=getDate(fp,first);
+    ValueSource v1 = d1==null ? getValueSource(fp, first) : null;
+
+    Date d2=getDate(fp,second);
+    ValueSource v2 = d2==null ? getValueSource(fp, second) : null;
+
+    // d     constant
+    // v     field
+    // dd    constant
+    // dv    subtract field from constant
+    // vd    subtract constant from field
+    // vv    subtract fields
+
+    final long ms1 = (d1 == null) ? 0 : d1.getTime();
+    final long ms2 = (d2 == null) ? 0 : d2.getTime(); 
+
+    // "d,dd" handle both constant cases
+
+    if (d1 != null && v2==null) {
+      return new LongConstValueSource(ms1-ms2);
+    }
+
+    // "v" just the date field
+    if (v1 != null && v2==null && d2==null) {
+      return v1;
+    }
+
+
+    // "dv"
+    if (d1!=null && v2!=null)
+      return new DualFloatFunction(new LongConstValueSource(ms1), v2) {
+        protected String name() {
+          return "ms";
+        }
+        protected float func(int doc, DocValues aVals, DocValues bVals) {
+          return ms1 - bVals.longVal(doc);
+        }
+      };
+
+    // "vd"
+    if (v1!=null && d2!=null)
+      return new DualFloatFunction(v1, new LongConstValueSource(ms2)) {
+        protected String name() {
+          return "ms";
+        }
+        protected float func(int doc, DocValues aVals, DocValues bVals) {
+          return aVals.longVal(doc) - ms2;
+        }
+      };
+
+    // "vv"
+    if (v1!=null && v2!=null)
+      return new DualFloatFunction(v1,v2) {
+        protected String name() {
+          return "ms";
+        }
+        protected float func(int doc, DocValues aVals, DocValues bVals) {
+          return aVals.longVal(doc) - bVals.longVal(doc);
+        }
+      };
+
+      return null; // shouldn't happen
+  }
+
+}
+
+
+// Private for now - we need to revisit how to handle typing in function queries
+class LongConstValueSource extends ValueSource {
+  final long constant;
+
+  public LongConstValueSource(long constant) {
+    this.constant = constant;
+  }
+
+  public String description() {
+    return "const(" + constant + ")";
+  }
+
+  public DocValues getValues(IndexReader reader) throws IOException {
+    return new DocValues() {
+      public float floatVal(int doc) {
+        return constant;
+      }
+      public int intVal(int doc) {
+        return (int)constant;
+      }
+      public long longVal(int doc) {
+        return constant;
+      }
+      public double doubleVal(int doc) {
+        return constant;
+      }
+      public String strVal(int doc) {
+        return Long.toString(constant);
+      }
+      public String toString(int doc) {
+        return description();
+      }
+    };
+  }
+
+  public int hashCode() {
+    return (int)constant + (int)(constant>>>32);
+  }
+
+  public boolean equals(Object o) {
+    if (LongConstValueSource.class != o.getClass()) return false;
+    LongConstValueSource other = (LongConstValueSource)o;
+    return this.constant == other.constant;
+  }
+}

src/java/org/apache/solr/search/function/DualFloatFunction.java

@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search.function;
+
+import org.apache.lucene.index.IndexReader;
+
+import java.io.IOException;
+
+public abstract class DualFloatFunction extends ValueSource {
+  protected final ValueSource a;
+  protected final ValueSource b;
+
+ /**
+   * @param   a  the base.
+   * @param   b  the exponent.
+   */
+  public DualFloatFunction(ValueSource a, ValueSource b) {
+    this.a = a;
+    this.b = b;
+  }
+
+  protected abstract String name();
+  protected abstract float func(int doc, DocValues aVals, DocValues bVals);
+
+  public String description() {
+    return name() + "(" + a.description() + "," + b.description() + ")";
+  }
+
+  public DocValues getValues(IndexReader reader) throws IOException {
+    final DocValues aVals =  a.getValues(reader);
+    final DocValues bVals =  b.getValues(reader);
+    return new DocValues() {
+      public float floatVal(int doc) {
+	return func(doc, aVals, bVals);
+      }
+      public int intVal(int doc) {
+        return (int)floatVal(doc);
+      }
+      public long longVal(int doc) {
+        return (long)floatVal(doc);
+      }
+      public double doubleVal(int doc) {
+        return floatVal(doc);
+      }
+      public String strVal(int doc) {
+        return Float.toString(floatVal(doc));
+      }
+      public String toString(int doc) {
+	return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
+      }
+    };
+  }
+
+  public int hashCode() {
+    int h = a.hashCode();
+    h ^= (h << 13) | (h >>> 20);
+    h += b.hashCode();
+    h ^= (h << 23) | (h >>> 10);
+    h += name().hashCode();
+    return h;
+  }
+
+  public boolean equals(Object o) {
+    if (this.getClass() != o.getClass()) return false;
+    DualFloatFunction other = (DualFloatFunction)o;
+    return this.a.equals(other.a)
+        && this.b.equals(other.b);
+  }
+}

src/java/org/apache/solr/search/function/PowFloatFunction.java

@@ -42,64 +42,3 @@ public class PowFloatFunction extends DualFloatFunction {
 }
 
 
-abstract class DualFloatFunction extends ValueSource {
-  protected final ValueSource a;
-  protected final ValueSource b;
-
- /**
-   * @param   a  the base.
-   * @param   b  the exponent.
-   */
-  public DualFloatFunction(ValueSource a, ValueSource b) {
-    this.a = a;
-    this.b = b;
-  }
-
-  protected abstract String name();
-  protected abstract float func(int doc, DocValues aVals, DocValues bVals);
-
-  public String description() {
-    return name() + "(" + a.description() + "," + b.description() + ")";
-  }
-
-  public DocValues getValues(IndexReader reader) throws IOException {
-    final DocValues aVals =  a.getValues(reader);
-    final DocValues bVals =  b.getValues(reader);
-    return new DocValues() {
-      public float floatVal(int doc) {
-	return func(doc, aVals, bVals);
-      }
-      public int intVal(int doc) {
-        return (int)floatVal(doc);
-      }
-      public long longVal(int doc) {
-        return (long)floatVal(doc);
-      }
-      public double doubleVal(int doc) {
-        return floatVal(doc);
-      }
-      public String strVal(int doc) {
-        return Float.toString(floatVal(doc));
-      }
-      public String toString(int doc) {
-	return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
-      }
-    };
-  }
-
-  public int hashCode() {
-    int h = a.hashCode();
-    h ^= (h << 13) | (h >>> 20);
-    h += b.hashCode();
-    h ^= (h << 23) | (h >>> 10);
-    h += name().hashCode();
-    return h;
-  }
-
-  public boolean equals(Object o) {
-    if (this.getClass() != o.getClass()) return false;
-    DualFloatFunction other = (DualFloatFunction)o;
-    return this.a.equals(other.a)
-        && this.b.equals(other.b);
-  }
-}

src/test/org/apache/solr/search/function/TestFunctionQuery.java

@@ -154,6 +154,8 @@ public class TestFunctionQuery extends AbstractSolrTestCase {
     singleTest(field,"sum(\0,\0)", 10, 20);
     singleTest(field,"sum(\0,\0,5)", 10, 25);
 
+    singleTest(field,"sub(\0,1)", 10, 9);
+
     singleTest(field,"product(\0,1)", 10, 10);
     singleTest(field,"product(\0,-2,-4)", 10, 80);
 
@@ -276,7 +278,7 @@ public class TestFunctionQuery extends AbstractSolrTestCase {
   }
 
   public void testGeneral() {
-    assertU(adoc("id","1"));
+    assertU(adoc("id","1", "a_tdt","2009-08-31T12:10:10.123Z", "b_tdt","2009-08-31T12:10:10.124Z"));
     assertU(adoc("id","2"));
     assertU(commit()); // create more than one segment
     assertU(adoc("id","3"));
@@ -292,5 +294,15 @@ public class TestFunctionQuery extends AbstractSolrTestCase {
     assertQ(req("fl","*,score","q", "{!func}top(ord(id))", "fq","id:6"), "//float[@name='score']='6.0'");
     assertQ(req("fl","*,score","q", "{!func}rord(id)", "fq","id:1"),"//float[@name='score']='6.0'");
     assertQ(req("fl","*,score","q", "{!func}top(rord(id))", "fq","id:1"),"//float[@name='score']='6.0'");
+
+
+    // test that we can subtract dates to millisecond precision
+    assertQ(req("fl","*,score","q", "{!func}ms(a_tdt,b_tdt)", "fq","id:1"), "//float[@name='score']='-1.0'");
+    assertQ(req("fl","*,score","q", "{!func}ms(b_tdt,a_tdt)", "fq","id:1"), "//float[@name='score']='1.0'");
+    assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z,2009-08-31T12:10:10.124Z)", "fq","id:1"), "//float[@name='score']='1.0'");
+    assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.124Z,a_tdt)", "fq","id:1"), "//float[@name='score']='1.0'");
+    assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z,b_tdt)", "fq","id:1"), "//float[@name='score']='1.0'");
+
+    assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z/SECOND,2009-08-31T12:10:10.124Z/SECOND)", "fq","id:1"), "//float[@name='score']='0.0'");
   }
 }