SOLR-356: pluggable functions

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@631979 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -208,6 +208,9 @@ New Features
     (Sharad Agarwal, Patrick O'Leary, Sabyasachi Dalal, Stu Hood,
      ryan, yonik)
 
+41. SOLR-356: Pluggable functions (value sources) that allow 
+    registration of new functions via solrconfig.xml
+    (Doug Daniels via yonik)
     
 Changes in runtime behavior
 

src/java/org/apache/solr/core/SolrCore.java

@@ -63,6 +63,7 @@ import org.apache.solr.request.XMLResponseWriter;
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.search.SolrIndexSearcher;
 import org.apache.solr.search.QParserPlugin;
+import org.apache.solr.search.ValueSourceParser;
 import org.apache.solr.update.DirectUpdateHandler;
 import org.apache.solr.update.SolrIndexWriter;
 import org.apache.solr.update.UpdateHandler;
@@ -329,6 +330,7 @@ public final class SolrCore {
       
       initWriters();
       initQParsers();
+      initValueSourceParsers();
       
       this.searchComponents = loadSearchComponents( config );
 
@@ -1038,6 +1040,36 @@ public final class SolrCore {
     if (plugin != null) return plugin;
     throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown query type '"+parserName+"'");
   }
+  
+  private final HashMap<String, ValueSourceParser> valueSourceParsers = new HashMap<String, ValueSourceParser>();
+  
+  /** Configure the ValueSource (function) plugins */
+  private void initValueSourceParsers() {
+    String xpath = "valueSourceParser";
+    NodeList nodes = (NodeList) solrConfig.evaluate(xpath, XPathConstants.NODESET);
+
+    NamedListPluginLoader<ValueSourceParser> loader =
+      new NamedListPluginLoader<ValueSourceParser>( "[solrconfig.xml] "+xpath, valueSourceParsers);
+
+    loader.load( solrConfig.getResourceLoader(), nodes );
+
+    // default value source parsers
+    for (Map.Entry<String, ValueSourceParser> entry : ValueSourceParser.standardValueSourceParsers.entrySet()) {
+      try {
+        String name = entry.getKey();
+        ValueSourceParser valueSourceParser = entry.getValue();
+        valueSourceParsers.put(name, valueSourceParser);
+        valueSourceParser.init(null);
+      } catch (Exception e) {
+        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+      }
+    }
+  }
+  
+  public ValueSourceParser getValueSourceParser(String parserName) {
+    return valueSourceParsers.get(parserName);
+  }
+  
 }
 
 

src/java/org/apache/solr/search/FunctionQParser.java

@@ -26,18 +26,18 @@ import org.apache.solr.search.function.*;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 public class FunctionQParser extends QParser {
+
+  protected QueryParsing.StrParser sp;
+
   public FunctionQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
     super(qstr, localParams, params, req);
   }
 
-  QueryParsing.StrParser sp;
-
   public Query parse() throws ParseException {
     sp = new QueryParsing.StrParser(getString());
-    ValueSource vs = parseValSource();
+    ValueSource vs = parseValueSource();
 
     /***  boost promoted to top-level query type to avoid this hack 
 
@@ -52,208 +52,86 @@ public class FunctionQParser extends QParser {
     return new FunctionQuery(vs);
   }
 
-  private abstract static class VSParser {
+  /**
-    abstract ValueSource parse(FunctionQParser fp) throws ParseException;
+   * Are there more arguments in the argument list being parsed?
+   * 
+   * @return whether more args exist
+   * @throws ParseException
+   */
+  public boolean hasMoreArguments() throws ParseException {
+    int ch = sp.peek();
+    /* determine whether the function is ending with a paren or end of str */
+    return (! (ch == 0 || ch == ')') );
   }
   
-  private static Map<String, VSParser> vsParsers = new HashMap<String, VSParser>();
+  /**
-  static {
+   * TODO: Doc
-    vsParsers.put("ord", new VSParser() {
+   * 
-      ValueSource parse(FunctionQParser fp) throws ParseException {
+   * @return
-        String field = fp.sp.getId();
+   * @throws ParseException
-        return new OrdFieldSource(field);
+   */
-      }
+  public String parseId() throws ParseException {
-    });
+    String value = sp.getId();
-    vsParsers.put("rord", new VSParser() {
+    consumeArgumentDelimiter();
-      ValueSource parse(FunctionQParser fp) throws ParseException {
+    return value;
-        String field = fp.sp.getId();
-        return new ReverseOrdFieldSource(field);
-      }
-    });
-    vsParsers.put("linear", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        ValueSource source = fp.parseValSource();
-        fp.sp.expect(",");
-        float slope = fp.sp.getFloat();
-        fp.sp.expect(",");
-        float intercept = fp.sp.getFloat();
-        return new LinearFloatFunction(source,slope,intercept);
-      }
-    });
-    vsParsers.put("max", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        ValueSource source = fp.parseValSource();
-        fp.sp.expect(",");
-        float val = fp.sp.getFloat();
-        return new MaxFloatFunction(source,val);
-      }
-    });
-    vsParsers.put("recip", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        ValueSource source = fp.parseValSource();
-        fp.sp.expect(",");
-        float m = fp.sp.getFloat();
-        fp.sp.expect(",");
-        float a = fp.sp.getFloat();
-        fp.sp.expect(",");
-        float b = fp.sp.getFloat();
-        return new ReciprocalFloatFunction(source,m,a,b);
-      }
-    });
-    vsParsers.put("scale", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        ValueSource source = fp.parseValSource();
-        fp.sp.expect(",");
-        float min = fp.sp.getFloat();
-        fp.sp.expect(",");
-        float max = fp.sp.getFloat();
-        return new ScaleFloatFunction(source,min,max);
-      }
-    });
-    vsParsers.put("pow", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        ValueSource a = fp.parseValSource();
-        fp.sp.expect(",");
-        ValueSource b = fp.parseValSource();
-        return new PowFloatFunction(a,b);
-      }
-    });
-    vsParsers.put("div", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        ValueSource a = fp.parseValSource();
-        fp.sp.expect(",");
-        ValueSource b = fp.parseValSource();
-        return new DivFloatFunction(a,b);
-      }
-    });
-    vsParsers.put("map", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        ValueSource source = fp.parseValSource();
-        fp.sp.expect(",");
-        float min = fp.sp.getFloat();
-        fp.sp.expect(",");
-        float max = fp.sp.getFloat();
-        fp.sp.expect(",");
-        float target = fp.sp.getFloat();
-        return new RangeMapFloatFunction(source,min,max,target);
-      }
-    });
-    vsParsers.put("sqrt", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        ValueSource source = fp.parseValSource();
-        return new SimpleFloatFunction(source) {
-          protected String name() {
-            return "sqrt";
-          }
-          protected float func(int doc, DocValues vals) {
-            return (float)Math.sqrt(vals.floatVal(doc));
-          }
-        };
-      }
-    });
-    vsParsers.put("log", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        ValueSource source = fp.parseValSource();
-        return new SimpleFloatFunction(source) {
-          protected String name() {
-            return "log";
-          }
-          protected float func(int doc, DocValues vals) {
-            return (float)Math.log10(vals.floatVal(doc));
-          }
-        };
-      }
-    });
-    vsParsers.put("abs", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        ValueSource source = fp.parseValSource();
-        return new SimpleFloatFunction(source) {
-          protected String name() {
-            return "abs";
-          }
-          protected float func(int doc, DocValues vals) {
-            return (float)Math.abs(vals.floatVal(doc));
-          }
-        };
-      }
-    });
-    vsParsers.put("sum", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        List<ValueSource> sources = fp.parseValueSourceList();
-        return new SumFloatFunction(sources.toArray(new ValueSource[sources.size()]));
-      }
-    });
-    vsParsers.put("product", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        List<ValueSource> sources = fp.parseValueSourceList();
-        return new ProductFloatFunction(sources.toArray(new ValueSource[sources.size()]));
-      }
-    });
-    vsParsers.put("query", new VSParser() {
-      // boost(query($q),rating)
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        Query q = fp.getNestedQuery();
-        float defVal = 0.0f;
-        if (fp.sp.opt(",")) {
-          defVal = fp.sp.getFloat();
-        }
-        return new QueryValueSource(q, defVal);
-      }
-    });
-    vsParsers.put("boost", new VSParser() {
-      ValueSource parse(FunctionQParser fp) throws ParseException {
-        Query q = fp.getNestedQuery();
-        fp.sp.expect(",");
-        ValueSource vs = fp.parseValSource();
-        BoostedQuery bq = new BoostedQuery(q, vs);
-System.out.println("Constructed Boostedquery " + bq);
-        return new QueryValueSource(bq, 0.0f);
-      }
-    });
   }
   
-  private List<ValueSource> parseValueSourceList() throws ParseException {
+  /**
+   * Parse a float.
+   * 
+   * @return Float
+   * @throws ParseException
+   */
+  public Float parseFloat() throws ParseException {
+    float value = sp.getFloat();
+    consumeArgumentDelimiter();
+    return value;
+  }
+  
+  /**
+   * Parse a list of ValueSource.  Must be the final set of arguments
+   * to a ValueSource.
+   * 
+   * @return List<ValueSource>
+   * @throws ParseException
+   */
+  public List<ValueSource> parseValueSourceList() throws ParseException {
     List<ValueSource> sources = new ArrayList<ValueSource>(3);
     for (;;) {
-      sources.add(parseValSource());
+      sources.add(parseValueSource(false));
-      char ch = sp.peek();
+      if (! consumeArgumentDelimiter()) break;
-      if (ch==')') break;
-      sp.expect(",");
     }
     return sources;
   }
 
-  private ValueSource parseValSource() throws ParseException {
+  /**
-    int ch = sp.peek();
+   * Parse an individual ValueSource.
-    if (ch>='0' && ch<='9'  || ch=='.' || ch=='+' || ch=='-') {
+   * 
-      return new ConstValueSource(sp.getFloat());
+   * @return
+   * @throws ParseException
+   */
+  public ValueSource parseValueSource() throws ParseException {
+    /* consume the delimiter afterward for an external call to parseValueSource */
+    return parseValueSource(true);
   }
   
-    String id = sp.getId();
+  /**
-    if (sp.opt("(")) {
+   * TODO: Doc
-      // a function... look it up.
+   * 
-      VSParser argParser = vsParsers.get(id);
+   * @return
-      if (argParser==null) {
+   * @throws ParseException
-        throw new ParseException("Unknown function " + id + " in FunctionQuery(" + sp + ")");
+   */
-      }
+  public Query parseNestedQuery() throws ParseException {
-      ValueSource vs = argParser.parse(this);
+    Query nestedQuery;
-      sp.expect(")");
-      return vs;
-    }
     
-    SchemaField f = req.getSchema().getField(id);
-    return f.getType().getValueSource(f, this);
-  }
-
-  private Query getNestedQuery() throws ParseException {
     if (sp.opt("$")) {
       String param = sp.getId();
       sp.pos += param.length();
       String qstr = getParam(param);
       qstr = qstr==null ? "" : qstr;
-      return subQuery(qstr, null).parse();
+      nestedQuery = subQuery(qstr, null).parse();
     }
-
+    else {
       int start = sp.pos;
       int end = sp.pos;
       String v = sp.val; 
@@ -281,7 +159,66 @@ System.out.println("Constructed Boostedquery " + bq);
       }
   
       sp.pos += end-start;  // advance past nested query
-    return sub.getQuery();
+      nestedQuery = sub.getQuery();
+    }
+    consumeArgumentDelimiter();
+    
+    return nestedQuery;
   }
 
+  /**
+   * Parse an individual value source.
+   * 
+   * @param doConsumeDelimiter whether to consume a delimiter following the ValueSource  
+   * @return
+   * @throws ParseException
+   */
+  protected ValueSource parseValueSource(boolean doConsumeDelimiter) throws ParseException {
+    ValueSource valueSource;
+    
+    int ch = sp.peek();
+    if (ch>='0' && ch<='9'  || ch=='.' || ch=='+' || ch=='-') {
+      valueSource = new ConstValueSource(sp.getFloat());
+    }
+    else {
+      String id = sp.getId();
+      if (sp.opt("(")) {
+        // a function... look it up.
+        ValueSourceParser argParser = req.getCore().getValueSourceParser(id);
+        if (argParser==null) {
+          throw new ParseException("Unknown function " + id + " in FunctionQuery(" + sp + ")");
+        }
+        valueSource = argParser.parse(this);
+        sp.expect(")");
+      }
+      else {
+        SchemaField f = req.getSchema().getField(id);
+        valueSource = f.getType().getValueSource(f, this);
+      }
+    }
+    
+    if (doConsumeDelimiter)
+      consumeArgumentDelimiter();
+    
+    return valueSource;
+  }
+
+  /**
+   * Consume an argument delimiter (a comma) from the token stream.
+   * Only consumes if more arguments should exist (no ending parens or end of string).
+   * 
+   * @return whether a delimiter was consumed
+   * @throws ParseException
+   */
+  protected boolean consumeArgumentDelimiter() throws ParseException {
+    /* if a list of args is ending, don't expect the comma */
+    if (hasMoreArguments()) {
+      sp.expect(",");
+      return true;
+    }
+   
+    return false;
+  }
+    
+
 }

src/test/org/apache/solr/search/function/TestFunctionQuery.java

@@ -17,7 +17,17 @@
 
 package org.apache.solr.search.function;
 
+import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.search.Query;
+import org.apache.solr.search.ValueSourceParser;
+import org.apache.solr.search.FunctionQParser;
+import org.apache.solr.search.function.DocValues;
+import org.apache.solr.search.function.QueryValueSource;
+import org.apache.solr.search.function.SimpleFloatFunction;
+import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.util.AbstractSolrTestCase;
+import org.apache.solr.common.util.NamedList;
 import org.apache.solr.core.SolrCore;
 
 import java.util.ArrayList;
@@ -36,7 +46,7 @@ import java.io.FileOutputStream;
 public class TestFunctionQuery extends AbstractSolrTestCase {
 
   public String getSchemaFile() { return "schema11.xml"; }
-  public String getSolrConfigFile() { return "solrconfig.xml"; }
+  public String getSolrConfigFile() { return "solrconfig-functionquery.xml"; }
   public String getCoreName() { return "basic"; }
 
   public void setUp() throws Exception {
@@ -153,6 +163,11 @@ public class TestFunctionQuery extends AbstractSolrTestCase {
     // test that infinity doesn't mess up scale function
     singleTest(field,"scale(log(\0),-1000,1000)",100,1000);
 
+    // test use of an ValueSourceParser plugin: nvl function
+    singleTest(field,"nvl(\0,1)", 0, 1, 100, 100);
+    
+    // compose the ValueSourceParser plugin function with another function
+    singleTest(field, "nvl(sum(0,\0),1)", 0, 1, 100, 100);
   }
 
   public void testFunctions() {
@@ -232,5 +247,4 @@ public class TestFunctionQuery extends AbstractSolrTestCase {
     }
 
   }
-
 }