SOLR-356: pluggable functions

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@631979 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2008-02-28 14:11:23 +00:00
parent cc61bb647b
commit a73850ef3b
4 changed files with 205 additions and 219 deletions

View File

@ -208,6 +208,9 @@ New Features
(Sharad Agarwal, Patrick O'Leary, Sabyasachi Dalal, Stu Hood, (Sharad Agarwal, Patrick O'Leary, Sabyasachi Dalal, Stu Hood,
ryan, yonik) ryan, yonik)
41. SOLR-356: Pluggable functions (value sources) that allow
registration of new functions via solrconfig.xml
(Doug Daniels via yonik)
Changes in runtime behavior Changes in runtime behavior

View File

@ -63,6 +63,7 @@ import org.apache.solr.request.XMLResponseWriter;
import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.QParserPlugin; import org.apache.solr.search.QParserPlugin;
import org.apache.solr.search.ValueSourceParser;
import org.apache.solr.update.DirectUpdateHandler; import org.apache.solr.update.DirectUpdateHandler;
import org.apache.solr.update.SolrIndexWriter; import org.apache.solr.update.SolrIndexWriter;
import org.apache.solr.update.UpdateHandler; import org.apache.solr.update.UpdateHandler;
@ -329,6 +330,7 @@ public final class SolrCore {
initWriters(); initWriters();
initQParsers(); initQParsers();
initValueSourceParsers();
this.searchComponents = loadSearchComponents( config ); this.searchComponents = loadSearchComponents( config );
@ -1038,6 +1040,36 @@ public final class SolrCore {
if (plugin != null) return plugin; if (plugin != null) return plugin;
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown query type '"+parserName+"'"); throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown query type '"+parserName+"'");
} }
private final HashMap<String, ValueSourceParser> valueSourceParsers = new HashMap<String, ValueSourceParser>();
/** Configure the ValueSource (function) plugins */
private void initValueSourceParsers() {
String xpath = "valueSourceParser";
NodeList nodes = (NodeList) solrConfig.evaluate(xpath, XPathConstants.NODESET);
NamedListPluginLoader<ValueSourceParser> loader =
new NamedListPluginLoader<ValueSourceParser>( "[solrconfig.xml] "+xpath, valueSourceParsers);
loader.load( solrConfig.getResourceLoader(), nodes );
// default value source parsers
for (Map.Entry<String, ValueSourceParser> entry : ValueSourceParser.standardValueSourceParsers.entrySet()) {
try {
String name = entry.getKey();
ValueSourceParser valueSourceParser = entry.getValue();
valueSourceParsers.put(name, valueSourceParser);
valueSourceParser.init(null);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
}
public ValueSourceParser getValueSourceParser(String parserName) {
return valueSourceParsers.get(parserName);
}
} }

View File

@ -26,18 +26,18 @@ import org.apache.solr.search.function.*;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
public class FunctionQParser extends QParser { public class FunctionQParser extends QParser {
protected QueryParsing.StrParser sp;
public FunctionQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { public FunctionQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
super(qstr, localParams, params, req); super(qstr, localParams, params, req);
} }
QueryParsing.StrParser sp;
public Query parse() throws ParseException { public Query parse() throws ParseException {
sp = new QueryParsing.StrParser(getString()); sp = new QueryParsing.StrParser(getString());
ValueSource vs = parseValSource(); ValueSource vs = parseValueSource();
/*** boost promoted to top-level query type to avoid this hack /*** boost promoted to top-level query type to avoid this hack
@ -52,208 +52,86 @@ public class FunctionQParser extends QParser {
return new FunctionQuery(vs); return new FunctionQuery(vs);
} }
private abstract static class VSParser { /**
abstract ValueSource parse(FunctionQParser fp) throws ParseException; * Are there more arguments in the argument list being parsed?
*
* @return whether more args exist
* @throws ParseException
*/
public boolean hasMoreArguments() throws ParseException {
int ch = sp.peek();
/* determine whether the function is ending with a paren or end of str */
return (! (ch == 0 || ch == ')') );
} }
private static Map<String, VSParser> vsParsers = new HashMap<String, VSParser>(); /**
static { * TODO: Doc
vsParsers.put("ord", new VSParser() { *
ValueSource parse(FunctionQParser fp) throws ParseException { * @return
String field = fp.sp.getId(); * @throws ParseException
return new OrdFieldSource(field); */
} public String parseId() throws ParseException {
}); String value = sp.getId();
vsParsers.put("rord", new VSParser() { consumeArgumentDelimiter();
ValueSource parse(FunctionQParser fp) throws ParseException { return value;
String field = fp.sp.getId();
return new ReverseOrdFieldSource(field);
}
});
vsParsers.put("linear", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
fp.sp.expect(",");
float slope = fp.sp.getFloat();
fp.sp.expect(",");
float intercept = fp.sp.getFloat();
return new LinearFloatFunction(source,slope,intercept);
}
});
vsParsers.put("max", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
fp.sp.expect(",");
float val = fp.sp.getFloat();
return new MaxFloatFunction(source,val);
}
});
vsParsers.put("recip", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
fp.sp.expect(",");
float m = fp.sp.getFloat();
fp.sp.expect(",");
float a = fp.sp.getFloat();
fp.sp.expect(",");
float b = fp.sp.getFloat();
return new ReciprocalFloatFunction(source,m,a,b);
}
});
vsParsers.put("scale", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
fp.sp.expect(",");
float min = fp.sp.getFloat();
fp.sp.expect(",");
float max = fp.sp.getFloat();
return new ScaleFloatFunction(source,min,max);
}
});
vsParsers.put("pow", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource a = fp.parseValSource();
fp.sp.expect(",");
ValueSource b = fp.parseValSource();
return new PowFloatFunction(a,b);
}
});
vsParsers.put("div", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource a = fp.parseValSource();
fp.sp.expect(",");
ValueSource b = fp.parseValSource();
return new DivFloatFunction(a,b);
}
});
vsParsers.put("map", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
fp.sp.expect(",");
float min = fp.sp.getFloat();
fp.sp.expect(",");
float max = fp.sp.getFloat();
fp.sp.expect(",");
float target = fp.sp.getFloat();
return new RangeMapFloatFunction(source,min,max,target);
}
});
vsParsers.put("sqrt", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
return new SimpleFloatFunction(source) {
protected String name() {
return "sqrt";
}
protected float func(int doc, DocValues vals) {
return (float)Math.sqrt(vals.floatVal(doc));
}
};
}
});
vsParsers.put("log", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
return new SimpleFloatFunction(source) {
protected String name() {
return "log";
}
protected float func(int doc, DocValues vals) {
return (float)Math.log10(vals.floatVal(doc));
}
};
}
});
vsParsers.put("abs", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
return new SimpleFloatFunction(source) {
protected String name() {
return "abs";
}
protected float func(int doc, DocValues vals) {
return (float)Math.abs(vals.floatVal(doc));
}
};
}
});
vsParsers.put("sum", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
List<ValueSource> sources = fp.parseValueSourceList();
return new SumFloatFunction(sources.toArray(new ValueSource[sources.size()]));
}
});
vsParsers.put("product", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
List<ValueSource> sources = fp.parseValueSourceList();
return new ProductFloatFunction(sources.toArray(new ValueSource[sources.size()]));
}
});
vsParsers.put("query", new VSParser() {
// boost(query($q),rating)
ValueSource parse(FunctionQParser fp) throws ParseException {
Query q = fp.getNestedQuery();
float defVal = 0.0f;
if (fp.sp.opt(",")) {
defVal = fp.sp.getFloat();
}
return new QueryValueSource(q, defVal);
}
});
vsParsers.put("boost", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
Query q = fp.getNestedQuery();
fp.sp.expect(",");
ValueSource vs = fp.parseValSource();
BoostedQuery bq = new BoostedQuery(q, vs);
System.out.println("Constructed Boostedquery " + bq);
return new QueryValueSource(bq, 0.0f);
}
});
} }
private List<ValueSource> parseValueSourceList() throws ParseException { /**
* Parse a float.
*
* @return Float
* @throws ParseException
*/
public Float parseFloat() throws ParseException {
float value = sp.getFloat();
consumeArgumentDelimiter();
return value;
}
/**
* Parse a list of ValueSource. Must be the final set of arguments
* to a ValueSource.
*
* @return List<ValueSource>
* @throws ParseException
*/
public List<ValueSource> parseValueSourceList() throws ParseException {
List<ValueSource> sources = new ArrayList<ValueSource>(3); List<ValueSource> sources = new ArrayList<ValueSource>(3);
for (;;) { for (;;) {
sources.add(parseValSource()); sources.add(parseValueSource(false));
char ch = sp.peek(); if (! consumeArgumentDelimiter()) break;
if (ch==')') break;
sp.expect(",");
} }
return sources; return sources;
} }
private ValueSource parseValSource() throws ParseException { /**
int ch = sp.peek(); * Parse an individual ValueSource.
if (ch>='0' && ch<='9' || ch=='.' || ch=='+' || ch=='-') { *
return new ConstValueSource(sp.getFloat()); * @return
* @throws ParseException
*/
public ValueSource parseValueSource() throws ParseException {
/* consume the delimiter afterward for an external call to parseValueSource */
return parseValueSource(true);
} }
String id = sp.getId(); /**
if (sp.opt("(")) { * TODO: Doc
// a function... look it up. *
VSParser argParser = vsParsers.get(id); * @return
if (argParser==null) { * @throws ParseException
throw new ParseException("Unknown function " + id + " in FunctionQuery(" + sp + ")"); */
} public Query parseNestedQuery() throws ParseException {
ValueSource vs = argParser.parse(this); Query nestedQuery;
sp.expect(")");
return vs;
}
SchemaField f = req.getSchema().getField(id);
return f.getType().getValueSource(f, this);
}
private Query getNestedQuery() throws ParseException {
if (sp.opt("$")) { if (sp.opt("$")) {
String param = sp.getId(); String param = sp.getId();
sp.pos += param.length(); sp.pos += param.length();
String qstr = getParam(param); String qstr = getParam(param);
qstr = qstr==null ? "" : qstr; qstr = qstr==null ? "" : qstr;
return subQuery(qstr, null).parse(); nestedQuery = subQuery(qstr, null).parse();
} }
else {
int start = sp.pos; int start = sp.pos;
int end = sp.pos; int end = sp.pos;
String v = sp.val; String v = sp.val;
@ -281,7 +159,66 @@ System.out.println("Constructed Boostedquery " + bq);
} }
sp.pos += end-start; // advance past nested query sp.pos += end-start; // advance past nested query
return sub.getQuery(); nestedQuery = sub.getQuery();
}
consumeArgumentDelimiter();
return nestedQuery;
} }
/**
* Parse an individual value source.
*
* @param doConsumeDelimiter whether to consume a delimiter following the ValueSource
* @return
* @throws ParseException
*/
protected ValueSource parseValueSource(boolean doConsumeDelimiter) throws ParseException {
ValueSource valueSource;
int ch = sp.peek();
if (ch>='0' && ch<='9' || ch=='.' || ch=='+' || ch=='-') {
valueSource = new ConstValueSource(sp.getFloat());
}
else {
String id = sp.getId();
if (sp.opt("(")) {
// a function... look it up.
ValueSourceParser argParser = req.getCore().getValueSourceParser(id);
if (argParser==null) {
throw new ParseException("Unknown function " + id + " in FunctionQuery(" + sp + ")");
}
valueSource = argParser.parse(this);
sp.expect(")");
}
else {
SchemaField f = req.getSchema().getField(id);
valueSource = f.getType().getValueSource(f, this);
}
}
if (doConsumeDelimiter)
consumeArgumentDelimiter();
return valueSource;
}
/**
* Consume an argument delimiter (a comma) from the token stream.
* Only consumes if more arguments should exist (no ending parens or end of string).
*
* @return whether a delimiter was consumed
* @throws ParseException
*/
protected boolean consumeArgumentDelimiter() throws ParseException {
/* if a list of args is ending, don't expect the comma */
if (hasMoreArguments()) {
sp.expect(",");
return true;
}
return false;
}
} }

View File

@ -17,7 +17,17 @@
package org.apache.solr.search.function; package org.apache.solr.search.function;
import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
import org.apache.solr.search.ValueSourceParser;
import org.apache.solr.search.FunctionQParser;
import org.apache.solr.search.function.DocValues;
import org.apache.solr.search.function.QueryValueSource;
import org.apache.solr.search.function.SimpleFloatFunction;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.util.AbstractSolrTestCase; import org.apache.solr.util.AbstractSolrTestCase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import java.util.ArrayList; import java.util.ArrayList;
@ -36,7 +46,7 @@ import java.io.FileOutputStream;
public class TestFunctionQuery extends AbstractSolrTestCase { public class TestFunctionQuery extends AbstractSolrTestCase {
public String getSchemaFile() { return "schema11.xml"; } public String getSchemaFile() { return "schema11.xml"; }
public String getSolrConfigFile() { return "solrconfig.xml"; } public String getSolrConfigFile() { return "solrconfig-functionquery.xml"; }
public String getCoreName() { return "basic"; } public String getCoreName() { return "basic"; }
public void setUp() throws Exception { public void setUp() throws Exception {
@ -153,6 +163,11 @@ public class TestFunctionQuery extends AbstractSolrTestCase {
// test that infinity doesn't mess up scale function // test that infinity doesn't mess up scale function
singleTest(field,"scale(log(\0),-1000,1000)",100,1000); singleTest(field,"scale(log(\0),-1000,1000)",100,1000);
// test use of an ValueSourceParser plugin: nvl function
singleTest(field,"nvl(\0,1)", 0, 1, 100, 100);
// compose the ValueSourceParser plugin function with another function
singleTest(field, "nvl(sum(0,\0),1)", 0, 1, 100, 100);
} }
public void testFunctions() { public void testFunctions() {
@ -232,5 +247,4 @@ public class TestFunctionQuery extends AbstractSolrTestCase {
} }
} }
} }