mirror of https://github.com/apache/lucene.git
SOLR-356: pluggable functions
git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@631979 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley
parent
cc61bb647b
commit
a73850ef3b
|
|
@ -208,6 +208,9 @@ New Features
|
|||
(Sharad Agarwal, Patrick O'Leary, Sabyasachi Dalal, Stu Hood,
|
||||
ryan, yonik)
|
||||
|
||||
41. SOLR-356: Pluggable functions (value sources) that allow
|
||||
registration of new functions via solrconfig.xml
|
||||
(Doug Daniels via yonik)
|
||||
|
||||
Changes in runtime behavior
|
||||
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ import org.apache.solr.request.XMLResponseWriter;
|
|||
import org.apache.solr.schema.IndexSchema;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.search.QParserPlugin;
|
||||
import org.apache.solr.search.ValueSourceParser;
|
||||
import org.apache.solr.update.DirectUpdateHandler;
|
||||
import org.apache.solr.update.SolrIndexWriter;
|
||||
import org.apache.solr.update.UpdateHandler;
|
||||
|
|
@ -329,6 +330,7 @@ public final class SolrCore {
|
|||
|
||||
initWriters();
|
||||
initQParsers();
|
||||
initValueSourceParsers();
|
||||
|
||||
this.searchComponents = loadSearchComponents( config );
|
||||
|
||||
|
|
@ -1038,6 +1040,36 @@ public final class SolrCore {
|
|||
if (plugin != null) return plugin;
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown query type '"+parserName+"'");
|
||||
}
|
||||
|
||||
private final HashMap<String, ValueSourceParser> valueSourceParsers = new HashMap<String, ValueSourceParser>();
|
||||
|
||||
/** Configure the ValueSource (function) plugins */
|
||||
private void initValueSourceParsers() {
|
||||
String xpath = "valueSourceParser";
|
||||
NodeList nodes = (NodeList) solrConfig.evaluate(xpath, XPathConstants.NODESET);
|
||||
|
||||
NamedListPluginLoader<ValueSourceParser> loader =
|
||||
new NamedListPluginLoader<ValueSourceParser>( "[solrconfig.xml] "+xpath, valueSourceParsers);
|
||||
|
||||
loader.load( solrConfig.getResourceLoader(), nodes );
|
||||
|
||||
// default value source parsers
|
||||
for (Map.Entry<String, ValueSourceParser> entry : ValueSourceParser.standardValueSourceParsers.entrySet()) {
|
||||
try {
|
||||
String name = entry.getKey();
|
||||
ValueSourceParser valueSourceParser = entry.getValue();
|
||||
valueSourceParsers.put(name, valueSourceParser);
|
||||
valueSourceParser.init(null);
|
||||
} catch (Exception e) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ValueSourceParser getValueSourceParser(String parserName) {
|
||||
return valueSourceParsers.get(parserName);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -26,18 +26,18 @@ import org.apache.solr.search.function.*;
|
|||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class FunctionQParser extends QParser {
|
||||
|
||||
protected QueryParsing.StrParser sp;
|
||||
|
||||
public FunctionQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
|
||||
super(qstr, localParams, params, req);
|
||||
}
|
||||
|
||||
QueryParsing.StrParser sp;
|
||||
|
||||
public Query parse() throws ParseException {
|
||||
sp = new QueryParsing.StrParser(getString());
|
||||
ValueSource vs = parseValSource();
|
||||
ValueSource vs = parseValueSource();
|
||||
|
||||
/*** boost promoted to top-level query type to avoid this hack
|
||||
|
||||
|
|
@ -52,236 +52,173 @@ public class FunctionQParser extends QParser {
|
|||
return new FunctionQuery(vs);
|
||||
}
|
||||
|
||||
private abstract static class VSParser {
|
||||
abstract ValueSource parse(FunctionQParser fp) throws ParseException;
|
||||
/**
|
||||
* Are there more arguments in the argument list being parsed?
|
||||
*
|
||||
* @return whether more args exist
|
||||
* @throws ParseException
|
||||
*/
|
||||
public boolean hasMoreArguments() throws ParseException {
|
||||
int ch = sp.peek();
|
||||
/* determine whether the function is ending with a paren or end of str */
|
||||
return (! (ch == 0 || ch == ')') );
|
||||
}
|
||||
|
||||
private static Map<String, VSParser> vsParsers = new HashMap<String, VSParser>();
|
||||
static {
|
||||
vsParsers.put("ord", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
String field = fp.sp.getId();
|
||||
return new OrdFieldSource(field);
|
||||
}
|
||||
});
|
||||
vsParsers.put("rord", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
String field = fp.sp.getId();
|
||||
return new ReverseOrdFieldSource(field);
|
||||
}
|
||||
});
|
||||
vsParsers.put("linear", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
ValueSource source = fp.parseValSource();
|
||||
fp.sp.expect(",");
|
||||
float slope = fp.sp.getFloat();
|
||||
fp.sp.expect(",");
|
||||
float intercept = fp.sp.getFloat();
|
||||
return new LinearFloatFunction(source,slope,intercept);
|
||||
}
|
||||
});
|
||||
vsParsers.put("max", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
ValueSource source = fp.parseValSource();
|
||||
fp.sp.expect(",");
|
||||
float val = fp.sp.getFloat();
|
||||
return new MaxFloatFunction(source,val);
|
||||
}
|
||||
});
|
||||
vsParsers.put("recip", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
ValueSource source = fp.parseValSource();
|
||||
fp.sp.expect(",");
|
||||
float m = fp.sp.getFloat();
|
||||
fp.sp.expect(",");
|
||||
float a = fp.sp.getFloat();
|
||||
fp.sp.expect(",");
|
||||
float b = fp.sp.getFloat();
|
||||
return new ReciprocalFloatFunction(source,m,a,b);
|
||||
}
|
||||
});
|
||||
vsParsers.put("scale", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
ValueSource source = fp.parseValSource();
|
||||
fp.sp.expect(",");
|
||||
float min = fp.sp.getFloat();
|
||||
fp.sp.expect(",");
|
||||
float max = fp.sp.getFloat();
|
||||
return new ScaleFloatFunction(source,min,max);
|
||||
}
|
||||
});
|
||||
vsParsers.put("pow", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
ValueSource a = fp.parseValSource();
|
||||
fp.sp.expect(",");
|
||||
ValueSource b = fp.parseValSource();
|
||||
return new PowFloatFunction(a,b);
|
||||
}
|
||||
});
|
||||
vsParsers.put("div", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
ValueSource a = fp.parseValSource();
|
||||
fp.sp.expect(",");
|
||||
ValueSource b = fp.parseValSource();
|
||||
return new DivFloatFunction(a,b);
|
||||
}
|
||||
});
|
||||
vsParsers.put("map", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
ValueSource source = fp.parseValSource();
|
||||
fp.sp.expect(",");
|
||||
float min = fp.sp.getFloat();
|
||||
fp.sp.expect(",");
|
||||
float max = fp.sp.getFloat();
|
||||
fp.sp.expect(",");
|
||||
float target = fp.sp.getFloat();
|
||||
return new RangeMapFloatFunction(source,min,max,target);
|
||||
}
|
||||
});
|
||||
vsParsers.put("sqrt", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
ValueSource source = fp.parseValSource();
|
||||
return new SimpleFloatFunction(source) {
|
||||
protected String name() {
|
||||
return "sqrt";
|
||||
}
|
||||
protected float func(int doc, DocValues vals) {
|
||||
return (float)Math.sqrt(vals.floatVal(doc));
|
||||
}
|
||||
};
|
||||
}
|
||||
});
|
||||
vsParsers.put("log", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
ValueSource source = fp.parseValSource();
|
||||
return new SimpleFloatFunction(source) {
|
||||
protected String name() {
|
||||
return "log";
|
||||
}
|
||||
protected float func(int doc, DocValues vals) {
|
||||
return (float)Math.log10(vals.floatVal(doc));
|
||||
}
|
||||
};
|
||||
}
|
||||
});
|
||||
vsParsers.put("abs", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
ValueSource source = fp.parseValSource();
|
||||
return new SimpleFloatFunction(source) {
|
||||
protected String name() {
|
||||
return "abs";
|
||||
}
|
||||
protected float func(int doc, DocValues vals) {
|
||||
return (float)Math.abs(vals.floatVal(doc));
|
||||
}
|
||||
};
|
||||
}
|
||||
});
|
||||
vsParsers.put("sum", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
List<ValueSource> sources = fp.parseValueSourceList();
|
||||
return new SumFloatFunction(sources.toArray(new ValueSource[sources.size()]));
|
||||
}
|
||||
});
|
||||
vsParsers.put("product", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
List<ValueSource> sources = fp.parseValueSourceList();
|
||||
return new ProductFloatFunction(sources.toArray(new ValueSource[sources.size()]));
|
||||
}
|
||||
});
|
||||
vsParsers.put("query", new VSParser() {
|
||||
// boost(query($q),rating)
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
Query q = fp.getNestedQuery();
|
||||
float defVal = 0.0f;
|
||||
if (fp.sp.opt(",")) {
|
||||
defVal = fp.sp.getFloat();
|
||||
}
|
||||
return new QueryValueSource(q, defVal);
|
||||
}
|
||||
});
|
||||
vsParsers.put("boost", new VSParser() {
|
||||
ValueSource parse(FunctionQParser fp) throws ParseException {
|
||||
Query q = fp.getNestedQuery();
|
||||
fp.sp.expect(",");
|
||||
ValueSource vs = fp.parseValSource();
|
||||
BoostedQuery bq = new BoostedQuery(q, vs);
|
||||
System.out.println("Constructed Boostedquery " + bq);
|
||||
return new QueryValueSource(bq, 0.0f);
|
||||
}
|
||||
});
|
||||
/**
|
||||
* TODO: Doc
|
||||
*
|
||||
* @return
|
||||
* @throws ParseException
|
||||
*/
|
||||
public String parseId() throws ParseException {
|
||||
String value = sp.getId();
|
||||
consumeArgumentDelimiter();
|
||||
return value;
|
||||
}
|
||||
|
||||
private List<ValueSource> parseValueSourceList() throws ParseException {
|
||||
/**
|
||||
* Parse a float.
|
||||
*
|
||||
* @return Float
|
||||
* @throws ParseException
|
||||
*/
|
||||
public Float parseFloat() throws ParseException {
|
||||
float value = sp.getFloat();
|
||||
consumeArgumentDelimiter();
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a list of ValueSource. Must be the final set of arguments
|
||||
* to a ValueSource.
|
||||
*
|
||||
* @return List<ValueSource>
|
||||
* @throws ParseException
|
||||
*/
|
||||
public List<ValueSource> parseValueSourceList() throws ParseException {
|
||||
List<ValueSource> sources = new ArrayList<ValueSource>(3);
|
||||
for (;;) {
|
||||
sources.add(parseValSource());
|
||||
char ch = sp.peek();
|
||||
if (ch==')') break;
|
||||
sp.expect(",");
|
||||
sources.add(parseValueSource(false));
|
||||
if (! consumeArgumentDelimiter()) break;
|
||||
}
|
||||
return sources;
|
||||
}
|
||||
|
||||
private ValueSource parseValSource() throws ParseException {
|
||||
int ch = sp.peek();
|
||||
if (ch>='0' && ch<='9' || ch=='.' || ch=='+' || ch=='-') {
|
||||
return new ConstValueSource(sp.getFloat());
|
||||
}
|
||||
|
||||
String id = sp.getId();
|
||||
if (sp.opt("(")) {
|
||||
// a function... look it up.
|
||||
VSParser argParser = vsParsers.get(id);
|
||||
if (argParser==null) {
|
||||
throw new ParseException("Unknown function " + id + " in FunctionQuery(" + sp + ")");
|
||||
}
|
||||
ValueSource vs = argParser.parse(this);
|
||||
sp.expect(")");
|
||||
return vs;
|
||||
}
|
||||
|
||||
SchemaField f = req.getSchema().getField(id);
|
||||
return f.getType().getValueSource(f, this);
|
||||
/**
|
||||
* Parse an individual ValueSource.
|
||||
*
|
||||
* @return
|
||||
* @throws ParseException
|
||||
*/
|
||||
public ValueSource parseValueSource() throws ParseException {
|
||||
/* consume the delimiter afterward for an external call to parseValueSource */
|
||||
return parseValueSource(true);
|
||||
}
|
||||
|
||||
private Query getNestedQuery() throws ParseException {
|
||||
/**
|
||||
* TODO: Doc
|
||||
*
|
||||
* @return
|
||||
* @throws ParseException
|
||||
*/
|
||||
public Query parseNestedQuery() throws ParseException {
|
||||
Query nestedQuery;
|
||||
|
||||
if (sp.opt("$")) {
|
||||
String param = sp.getId();
|
||||
sp.pos += param.length();
|
||||
String qstr = getParam(param);
|
||||
qstr = qstr==null ? "" : qstr;
|
||||
return subQuery(qstr, null).parse();
|
||||
nestedQuery = subQuery(qstr, null).parse();
|
||||
}
|
||||
else {
|
||||
int start = sp.pos;
|
||||
int end = sp.pos;
|
||||
String v = sp.val;
|
||||
|
||||
int start = sp.pos;
|
||||
int end = sp.pos;
|
||||
String v = sp.val;
|
||||
String qs = v.substring(start);
|
||||
HashMap nestedLocalParams = new HashMap<String,String>();
|
||||
end = QueryParsing.parseLocalParams(qs, start, nestedLocalParams, getParams());
|
||||
|
||||
String qs = v.substring(start);
|
||||
HashMap nestedLocalParams = new HashMap<String,String>();
|
||||
end = QueryParsing.parseLocalParams(qs, start, nestedLocalParams, getParams());
|
||||
QParser sub;
|
||||
|
||||
QParser sub;
|
||||
|
||||
if (end>start) {
|
||||
if (nestedLocalParams.get(QueryParsing.V) != null) {
|
||||
// value specified directly in local params... so the end of the
|
||||
// query should be the end of the local params.
|
||||
sub = subQuery(qs.substring(0, end), null);
|
||||
if (end>start) {
|
||||
if (nestedLocalParams.get(QueryParsing.V) != null) {
|
||||
// value specified directly in local params... so the end of the
|
||||
// query should be the end of the local params.
|
||||
sub = subQuery(qs.substring(0, end), null);
|
||||
} else {
|
||||
// value here is *after* the local params... ask the parser.
|
||||
sub = subQuery(qs, null);
|
||||
// int subEnd = sub.findEnd(')');
|
||||
// TODO.. implement functions to find the end of a nested query
|
||||
throw new ParseException("Nested local params must have value in v parameter. got '" + qs + "'");
|
||||
}
|
||||
} else {
|
||||
// value here is *after* the local params... ask the parser.
|
||||
sub = subQuery(qs, null);
|
||||
// int subEnd = sub.findEnd(')');
|
||||
// TODO.. implement functions to find the end of a nested query
|
||||
throw new ParseException("Nested local params must have value in v parameter. got '" + qs + "'");
|
||||
throw new ParseException("Nested function query must use $param or <!v=value> forms. got '" + qs + "'");
|
||||
}
|
||||
} else {
|
||||
throw new ParseException("Nested function query must use $param or <!v=value> forms. got '" + qs + "'");
|
||||
}
|
||||
|
||||
sp.pos += end-start; // advance past nested query
|
||||
return sub.getQuery();
|
||||
sp.pos += end-start; // advance past nested query
|
||||
nestedQuery = sub.getQuery();
|
||||
}
|
||||
consumeArgumentDelimiter();
|
||||
|
||||
return nestedQuery;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an individual value source.
|
||||
*
|
||||
* @param doConsumeDelimiter whether to consume a delimiter following the ValueSource
|
||||
* @return
|
||||
* @throws ParseException
|
||||
*/
|
||||
protected ValueSource parseValueSource(boolean doConsumeDelimiter) throws ParseException {
|
||||
ValueSource valueSource;
|
||||
|
||||
int ch = sp.peek();
|
||||
if (ch>='0' && ch<='9' || ch=='.' || ch=='+' || ch=='-') {
|
||||
valueSource = new ConstValueSource(sp.getFloat());
|
||||
}
|
||||
else {
|
||||
String id = sp.getId();
|
||||
if (sp.opt("(")) {
|
||||
// a function... look it up.
|
||||
ValueSourceParser argParser = req.getCore().getValueSourceParser(id);
|
||||
if (argParser==null) {
|
||||
throw new ParseException("Unknown function " + id + " in FunctionQuery(" + sp + ")");
|
||||
}
|
||||
valueSource = argParser.parse(this);
|
||||
sp.expect(")");
|
||||
}
|
||||
else {
|
||||
SchemaField f = req.getSchema().getField(id);
|
||||
valueSource = f.getType().getValueSource(f, this);
|
||||
}
|
||||
}
|
||||
|
||||
if (doConsumeDelimiter)
|
||||
consumeArgumentDelimiter();
|
||||
|
||||
return valueSource;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consume an argument delimiter (a comma) from the token stream.
|
||||
* Only consumes if more arguments should exist (no ending parens or end of string).
|
||||
*
|
||||
* @return whether a delimiter was consumed
|
||||
* @throws ParseException
|
||||
*/
|
||||
protected boolean consumeArgumentDelimiter() throws ParseException {
|
||||
/* if a list of args is ending, don't expect the comma */
|
||||
if (hasMoreArguments()) {
|
||||
sp.expect(",");
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,17 @@
|
|||
|
||||
package org.apache.solr.search.function;
|
||||
|
||||
import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
|
||||
import org.apache.lucene.queryParser.ParseException;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.solr.search.ValueSourceParser;
|
||||
import org.apache.solr.search.FunctionQParser;
|
||||
import org.apache.solr.search.function.DocValues;
|
||||
import org.apache.solr.search.function.QueryValueSource;
|
||||
import org.apache.solr.search.function.SimpleFloatFunction;
|
||||
import org.apache.solr.search.function.ValueSource;
|
||||
import org.apache.solr.util.AbstractSolrTestCase;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
|
@ -36,7 +46,7 @@ import java.io.FileOutputStream;
|
|||
public class TestFunctionQuery extends AbstractSolrTestCase {
|
||||
|
||||
public String getSchemaFile() { return "schema11.xml"; }
|
||||
public String getSolrConfigFile() { return "solrconfig.xml"; }
|
||||
public String getSolrConfigFile() { return "solrconfig-functionquery.xml"; }
|
||||
public String getCoreName() { return "basic"; }
|
||||
|
||||
public void setUp() throws Exception {
|
||||
|
|
@ -153,6 +163,11 @@ public class TestFunctionQuery extends AbstractSolrTestCase {
|
|||
// test that infinity doesn't mess up scale function
|
||||
singleTest(field,"scale(log(\0),-1000,1000)",100,1000);
|
||||
|
||||
// test use of an ValueSourceParser plugin: nvl function
|
||||
singleTest(field,"nvl(\0,1)", 0, 1, 100, 100);
|
||||
|
||||
// compose the ValueSourceParser plugin function with another function
|
||||
singleTest(field, "nvl(sum(0,\0),1)", 0, 1, 100, 100);
|
||||
}
|
||||
|
||||
public void testFunctions() {
|
||||
|
|
@ -232,5 +247,4 @@ public class TestFunctionQuery extends AbstractSolrTestCase {
|
|||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue