diff --git a/CHANGES.txt b/CHANGES.txt index 1fd7019174d..acbc344c7a6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -207,7 +207,10 @@ New Features in distributed mode. (Sharad Agarwal, Patrick O'Leary, Sabyasachi Dalal, Stu Hood, ryan, yonik) - + +41. SOLR-356: Pluggable functions (value sources) that allow + registration of new functions via solrconfig.xml + (Doug Daniels via yonik) Changes in runtime behavior diff --git a/src/java/org/apache/solr/core/SolrCore.java b/src/java/org/apache/solr/core/SolrCore.java index b42f1e6497f..027b5df69cf 100644 --- a/src/java/org/apache/solr/core/SolrCore.java +++ b/src/java/org/apache/solr/core/SolrCore.java @@ -63,6 +63,7 @@ import org.apache.solr.request.XMLResponseWriter; import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.QParserPlugin; +import org.apache.solr.search.ValueSourceParser; import org.apache.solr.update.DirectUpdateHandler; import org.apache.solr.update.SolrIndexWriter; import org.apache.solr.update.UpdateHandler; @@ -329,6 +330,7 @@ public final class SolrCore { initWriters(); initQParsers(); + initValueSourceParsers(); this.searchComponents = loadSearchComponents( config ); @@ -1038,6 +1040,36 @@ public final class SolrCore { if (plugin != null) return plugin; throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown query type '"+parserName+"'"); } + + private final HashMap valueSourceParsers = new HashMap(); + + /** Configure the ValueSource (function) plugins */ + private void initValueSourceParsers() { + String xpath = "valueSourceParser"; + NodeList nodes = (NodeList) solrConfig.evaluate(xpath, XPathConstants.NODESET); + + NamedListPluginLoader loader = + new NamedListPluginLoader( "[solrconfig.xml] "+xpath, valueSourceParsers); + + loader.load( solrConfig.getResourceLoader(), nodes ); + + // default value source parsers + for (Map.Entry entry : ValueSourceParser.standardValueSourceParsers.entrySet()) { + try { + String name = entry.getKey(); + ValueSourceParser valueSourceParser = entry.getValue(); + valueSourceParsers.put(name, valueSourceParser); + valueSourceParser.init(null); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + } + } + + public ValueSourceParser getValueSourceParser(String parserName) { + return valueSourceParsers.get(parserName); + } + } diff --git a/src/java/org/apache/solr/search/FunctionQParser.java b/src/java/org/apache/solr/search/FunctionQParser.java index 626ac4773e0..a369744248d 100755 --- a/src/java/org/apache/solr/search/FunctionQParser.java +++ b/src/java/org/apache/solr/search/FunctionQParser.java @@ -26,18 +26,18 @@ import org.apache.solr.search.function.*; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; public class FunctionQParser extends QParser { + + protected QueryParsing.StrParser sp; + public FunctionQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { super(qstr, localParams, params, req); } - QueryParsing.StrParser sp; - public Query parse() throws ParseException { sp = new QueryParsing.StrParser(getString()); - ValueSource vs = parseValSource(); + ValueSource vs = parseValueSource(); /*** boost promoted to top-level query type to avoid this hack @@ -52,236 +52,173 @@ public class FunctionQParser extends QParser { return new FunctionQuery(vs); } - private abstract static class VSParser { - abstract ValueSource parse(FunctionQParser fp) throws ParseException; + /** + * Are there more arguments in the argument list being parsed? + * + * @return whether more args exist + * @throws ParseException + */ + public boolean hasMoreArguments() throws ParseException { + int ch = sp.peek(); + /* determine whether the function is ending with a paren or end of str */ + return (! (ch == 0 || ch == ')') ); } - - private static Map vsParsers = new HashMap(); - static { - vsParsers.put("ord", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - String field = fp.sp.getId(); - return new OrdFieldSource(field); - } - }); - vsParsers.put("rord", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - String field = fp.sp.getId(); - return new ReverseOrdFieldSource(field); - } - }); - vsParsers.put("linear", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - ValueSource source = fp.parseValSource(); - fp.sp.expect(","); - float slope = fp.sp.getFloat(); - fp.sp.expect(","); - float intercept = fp.sp.getFloat(); - return new LinearFloatFunction(source,slope,intercept); - } - }); - vsParsers.put("max", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - ValueSource source = fp.parseValSource(); - fp.sp.expect(","); - float val = fp.sp.getFloat(); - return new MaxFloatFunction(source,val); - } - }); - vsParsers.put("recip", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - ValueSource source = fp.parseValSource(); - fp.sp.expect(","); - float m = fp.sp.getFloat(); - fp.sp.expect(","); - float a = fp.sp.getFloat(); - fp.sp.expect(","); - float b = fp.sp.getFloat(); - return new ReciprocalFloatFunction(source,m,a,b); - } - }); - vsParsers.put("scale", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - ValueSource source = fp.parseValSource(); - fp.sp.expect(","); - float min = fp.sp.getFloat(); - fp.sp.expect(","); - float max = fp.sp.getFloat(); - return new ScaleFloatFunction(source,min,max); - } - }); - vsParsers.put("pow", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - ValueSource a = fp.parseValSource(); - fp.sp.expect(","); - ValueSource b = fp.parseValSource(); - return new PowFloatFunction(a,b); - } - }); - vsParsers.put("div", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - ValueSource a = fp.parseValSource(); - fp.sp.expect(","); - ValueSource b = fp.parseValSource(); - return new DivFloatFunction(a,b); - } - }); - vsParsers.put("map", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - ValueSource source = fp.parseValSource(); - fp.sp.expect(","); - float min = fp.sp.getFloat(); - fp.sp.expect(","); - float max = fp.sp.getFloat(); - fp.sp.expect(","); - float target = fp.sp.getFloat(); - return new RangeMapFloatFunction(source,min,max,target); - } - }); - vsParsers.put("sqrt", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - ValueSource source = fp.parseValSource(); - return new SimpleFloatFunction(source) { - protected String name() { - return "sqrt"; - } - protected float func(int doc, DocValues vals) { - return (float)Math.sqrt(vals.floatVal(doc)); - } - }; - } - }); - vsParsers.put("log", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - ValueSource source = fp.parseValSource(); - return new SimpleFloatFunction(source) { - protected String name() { - return "log"; - } - protected float func(int doc, DocValues vals) { - return (float)Math.log10(vals.floatVal(doc)); - } - }; - } - }); - vsParsers.put("abs", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - ValueSource source = fp.parseValSource(); - return new SimpleFloatFunction(source) { - protected String name() { - return "abs"; - } - protected float func(int doc, DocValues vals) { - return (float)Math.abs(vals.floatVal(doc)); - } - }; - } - }); - vsParsers.put("sum", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - List sources = fp.parseValueSourceList(); - return new SumFloatFunction(sources.toArray(new ValueSource[sources.size()])); - } - }); - vsParsers.put("product", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - List sources = fp.parseValueSourceList(); - return new ProductFloatFunction(sources.toArray(new ValueSource[sources.size()])); - } - }); - vsParsers.put("query", new VSParser() { - // boost(query($q),rating) - ValueSource parse(FunctionQParser fp) throws ParseException { - Query q = fp.getNestedQuery(); - float defVal = 0.0f; - if (fp.sp.opt(",")) { - defVal = fp.sp.getFloat(); - } - return new QueryValueSource(q, defVal); - } - }); - vsParsers.put("boost", new VSParser() { - ValueSource parse(FunctionQParser fp) throws ParseException { - Query q = fp.getNestedQuery(); - fp.sp.expect(","); - ValueSource vs = fp.parseValSource(); - BoostedQuery bq = new BoostedQuery(q, vs); -System.out.println("Constructed Boostedquery " + bq); - return new QueryValueSource(bq, 0.0f); - } - }); + + /** + * TODO: Doc + * + * @return + * @throws ParseException + */ + public String parseId() throws ParseException { + String value = sp.getId(); + consumeArgumentDelimiter(); + return value; } - - private List parseValueSourceList() throws ParseException { + + /** + * Parse a float. + * + * @return Float + * @throws ParseException + */ + public Float parseFloat() throws ParseException { + float value = sp.getFloat(); + consumeArgumentDelimiter(); + return value; + } + + /** + * Parse a list of ValueSource. Must be the final set of arguments + * to a ValueSource. + * + * @return List + * @throws ParseException + */ + public List parseValueSourceList() throws ParseException { List sources = new ArrayList(3); for (;;) { - sources.add(parseValSource()); - char ch = sp.peek(); - if (ch==')') break; - sp.expect(","); + sources.add(parseValueSource(false)); + if (! consumeArgumentDelimiter()) break; } return sources; } - private ValueSource parseValSource() throws ParseException { - int ch = sp.peek(); - if (ch>='0' && ch<='9' || ch=='.' || ch=='+' || ch=='-') { - return new ConstValueSource(sp.getFloat()); - } - - String id = sp.getId(); - if (sp.opt("(")) { - // a function... look it up. - VSParser argParser = vsParsers.get(id); - if (argParser==null) { - throw new ParseException("Unknown function " + id + " in FunctionQuery(" + sp + ")"); - } - ValueSource vs = argParser.parse(this); - sp.expect(")"); - return vs; - } - - SchemaField f = req.getSchema().getField(id); - return f.getType().getValueSource(f, this); + /** + * Parse an individual ValueSource. + * + * @return + * @throws ParseException + */ + public ValueSource parseValueSource() throws ParseException { + /* consume the delimiter afterward for an external call to parseValueSource */ + return parseValueSource(true); } - - private Query getNestedQuery() throws ParseException { + + /** + * TODO: Doc + * + * @return + * @throws ParseException + */ + public Query parseNestedQuery() throws ParseException { + Query nestedQuery; + if (sp.opt("$")) { String param = sp.getId(); sp.pos += param.length(); String qstr = getParam(param); qstr = qstr==null ? "" : qstr; - return subQuery(qstr, null).parse(); + nestedQuery = subQuery(qstr, null).parse(); } - - int start = sp.pos; - int end = sp.pos; - String v = sp.val; - - String qs = v.substring(start); - HashMap nestedLocalParams = new HashMap(); - end = QueryParsing.parseLocalParams(qs, start, nestedLocalParams, getParams()); - - QParser sub; - - if (end>start) { - if (nestedLocalParams.get(QueryParsing.V) != null) { - // value specified directly in local params... so the end of the - // query should be the end of the local params. - sub = subQuery(qs.substring(0, end), null); + else { + int start = sp.pos; + int end = sp.pos; + String v = sp.val; + + String qs = v.substring(start); + HashMap nestedLocalParams = new HashMap(); + end = QueryParsing.parseLocalParams(qs, start, nestedLocalParams, getParams()); + + QParser sub; + + if (end>start) { + if (nestedLocalParams.get(QueryParsing.V) != null) { + // value specified directly in local params... so the end of the + // query should be the end of the local params. + sub = subQuery(qs.substring(0, end), null); + } else { + // value here is *after* the local params... ask the parser. + sub = subQuery(qs, null); + // int subEnd = sub.findEnd(')'); + // TODO.. implement functions to find the end of a nested query + throw new ParseException("Nested local params must have value in v parameter. got '" + qs + "'"); + } } else { - // value here is *after* the local params... ask the parser. - sub = subQuery(qs, null); - // int subEnd = sub.findEnd(')'); - // TODO.. implement functions to find the end of a nested query - throw new ParseException("Nested local params must have value in v parameter. got '" + qs + "'"); + throw new ParseException("Nested function query must use $param or forms. got '" + qs + "'"); } - } else { - throw new ParseException("Nested function query must use $param or forms. got '" + qs + "'"); + + sp.pos += end-start; // advance past nested query + nestedQuery = sub.getQuery(); } - - sp.pos += end-start; // advance past nested query - return sub.getQuery(); + consumeArgumentDelimiter(); + + return nestedQuery; } + /** + * Parse an individual value source. + * + * @param doConsumeDelimiter whether to consume a delimiter following the ValueSource + * @return + * @throws ParseException + */ + protected ValueSource parseValueSource(boolean doConsumeDelimiter) throws ParseException { + ValueSource valueSource; + + int ch = sp.peek(); + if (ch>='0' && ch<='9' || ch=='.' || ch=='+' || ch=='-') { + valueSource = new ConstValueSource(sp.getFloat()); + } + else { + String id = sp.getId(); + if (sp.opt("(")) { + // a function... look it up. + ValueSourceParser argParser = req.getCore().getValueSourceParser(id); + if (argParser==null) { + throw new ParseException("Unknown function " + id + " in FunctionQuery(" + sp + ")"); + } + valueSource = argParser.parse(this); + sp.expect(")"); + } + else { + SchemaField f = req.getSchema().getField(id); + valueSource = f.getType().getValueSource(f, this); + } + } + + if (doConsumeDelimiter) + consumeArgumentDelimiter(); + + return valueSource; + } + + /** + * Consume an argument delimiter (a comma) from the token stream. + * Only consumes if more arguments should exist (no ending parens or end of string). + * + * @return whether a delimiter was consumed + * @throws ParseException + */ + protected boolean consumeArgumentDelimiter() throws ParseException { + /* if a list of args is ending, don't expect the comma */ + if (hasMoreArguments()) { + sp.expect(","); + return true; + } + + return false; + } + + } diff --git a/src/test/org/apache/solr/search/function/TestFunctionQuery.java b/src/test/org/apache/solr/search/function/TestFunctionQuery.java index daeea8b77c2..8926c7d902f 100755 --- a/src/test/org/apache/solr/search/function/TestFunctionQuery.java +++ b/src/test/org/apache/solr/search/function/TestFunctionQuery.java @@ -17,7 +17,17 @@ package org.apache.solr.search.function; +import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.search.Query; +import org.apache.solr.search.ValueSourceParser; +import org.apache.solr.search.FunctionQParser; +import org.apache.solr.search.function.DocValues; +import org.apache.solr.search.function.QueryValueSource; +import org.apache.solr.search.function.SimpleFloatFunction; +import org.apache.solr.search.function.ValueSource; import org.apache.solr.util.AbstractSolrTestCase; +import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import java.util.ArrayList; @@ -36,7 +46,7 @@ import java.io.FileOutputStream; public class TestFunctionQuery extends AbstractSolrTestCase { public String getSchemaFile() { return "schema11.xml"; } - public String getSolrConfigFile() { return "solrconfig.xml"; } + public String getSolrConfigFile() { return "solrconfig-functionquery.xml"; } public String getCoreName() { return "basic"; } public void setUp() throws Exception { @@ -153,6 +163,11 @@ public class TestFunctionQuery extends AbstractSolrTestCase { // test that infinity doesn't mess up scale function singleTest(field,"scale(log(\0),-1000,1000)",100,1000); + // test use of an ValueSourceParser plugin: nvl function + singleTest(field,"nvl(\0,1)", 0, 1, 100, 100); + + // compose the ValueSourceParser plugin function with another function + singleTest(field, "nvl(sum(0,\0),1)", 0, 1, 100, 100); } public void testFunctions() { @@ -232,5 +247,4 @@ public class TestFunctionQuery extends AbstractSolrTestCase { } } - } \ No newline at end of file