Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-334: pluggable query parsers 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@587090 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2007-10-22 13:43:07 +00:00
parent a899f9e24b
commit fc96a325ed
19 changed files with 1738 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGES.txt
View File

@ -136,6 +136,9 @@ New Features
to the detailed field information from the solrj client API.
(Grant Ingersoll via ehatcher)
26. SOLR-334L Pluggable query parsers. Allows specification of query
type and arguments as a prefix on a query string. (yonik)
Changes in runtime behavior
Optimizations

38
src/java/org/apache/solr/core/SolrCore.java
View File

@ -53,6 +53,9 @@ import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.XMLResponseWriter;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.QParserPlugin;
import org.apache.solr.search.LuceneQParserPlugin;
import org.apache.solr.search.OldLuceneQParserPlugin;
import org.apache.solr.update.DirectUpdateHandler;
import org.apache.solr.update.SolrIndexWriter;
import org.apache.solr.update.UpdateHandler;
@ -289,7 +292,8 @@ public final class SolrCore {
initIndex();
initWriters();
initQParsers();
// Processors initialized before the handlers
updateProcessors = loadUpdateProcessors();
reqHandlers = new RequestHandlers(this);
@ -909,6 +913,38 @@ public final class SolrCore {
public final QueryResponseWriter getQueryResponseWriter(SolrQueryRequest request) {
return getQueryResponseWriter(request.getParam("wt"));
}
private final Map<String, QParserPlugin> qParserPlugins = new HashMap<String, QParserPlugin>();
/** Configure the query parsers. */
private void initQParsers() {
String xpath = "queryParser";
NodeList nodes = (NodeList) solrConfig.evaluate(xpath, XPathConstants.NODESET);
NamedListPluginLoader<QParserPlugin> loader =
new NamedListPluginLoader<QParserPlugin>( "[solrconfig.xml] "+xpath, qParserPlugins);
loader.load( solrConfig, nodes );
// default parsers
for (int i=0; i<QParserPlugin.standardPlugins.length; i+=2) {
try {
String name = (String)QParserPlugin.standardPlugins[i];
Class<QParserPlugin> clazz = (Class<QParserPlugin>)QParserPlugin.standardPlugins[i+1];
QParserPlugin plugin = clazz.newInstance();
qParserPlugins.put(name, plugin);
plugin.init(null);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
}
public QParserPlugin getQueryPlugin(String parserName) {
QParserPlugin plugin = qParserPlugins.get(parserName);
if (plugin != null) return plugin;
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown query type '"+parserName+"'");
}
}

49
src/java/org/apache/solr/handler/StandardRequestHandler.java
View File

@ -35,7 +35,6 @@ import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.MoreLikeThisParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrCore;
import org.apache.solr.highlight.SolrHighlighter;
@ -72,58 +71,31 @@ public class StandardRequestHandler extends RequestHandlerBase {
SolrParams p = req.getParams();
String qstr = p.required().get(CommonParams.Q);
String defaultField = p.get(CommonParams.DF);
// find fieldnames to return (fieldlist)
// TODO: make this per-query and add method to QParser to get?
String fl = p.get(CommonParams.FL);
int flags = 0;
if (fl != null) {
flags |= U.setReturnFields(fl, rsp);
}
String sortStr = p.get(CommonParams.SORT);
if( sortStr == null ) {
// TODO? should we disable the ';' syntax with config?
// legacy mode, where sreq is query;sort
List<String> commands = StrUtils.splitSmart(qstr,';');
if( commands.size() == 2 ) {
// TODO? add a deprication warning to the response header
qstr = commands.get( 0 );
sortStr = commands.get( 1 );
}
else if( commands.size() == 1 ) {
// This is need to support the case where someone sends: "q=query;"
qstr = commands.get( 0 );
}
else if( commands.size() > 2 ) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "If you want to use multiple ';' in the query, use the 'sort' param." );
}
}
Sort sort = null;
if( sortStr != null ) {
QueryParsing.SortSpec sortSpec = QueryParsing.parseSort(sortStr, req.getSchema());
if (sortSpec != null) {
sort = sortSpec.getSort();
}
}
QParser parser = QParser.getParser(qstr, OldLuceneQParserPlugin.NAME, req);
Query query = parser.getQuery();
QueryParsing.SortSpec sortSpec = parser.getSort(true);
// parse the query from the 'q' parameter (sort has been striped)
Query query = QueryParsing.parseQuery(qstr, defaultField, p, req.getSchema());
DocListAndSet results = new DocListAndSet();
NamedList facetInfo = null;
List<Query> filters = U.parseFilterQueries(req);
SolrIndexSearcher s = req.getSearcher();
if (p.getBool(FacetParams.FACET,false)) {
results = s.getDocListAndSet(query, filters, sort,
p.getInt(CommonParams.START,0), p.getInt(CommonParams.ROWS,10),
results = s.getDocListAndSet(query, filters, sortSpec.getSort(),
sortSpec.getOffset(), sortSpec.getCount(),
flags);
facetInfo = getFacetInfo(req, rsp, results.docSet);
} else {
results.docList = s.getDocList(query, filters, sort,
p.getInt(CommonParams.START,0), p.getInt(CommonParams.ROWS,10),
results.docList = s.getDocList(query, filters, sortSpec.getSort(),
sortSpec.getOffset(), sortSpec.getCount(),
flags);
}
@ -163,7 +135,10 @@ public class StandardRequestHandler extends RequestHandlerBase {
SolrHighlighter highlighter = req.getCore().getHighlighter();
NamedList sumData = highlighter.doHighlighting(
results.docList, query.rewrite(req.getSearcher().getReader()), req, new String[]{defaultField});
results.docList,
parser.getHighlightQuery().rewrite(req.getSearcher().getReader()),
req,
parser.getDefaultHighlightFields());
if(sumData != null)
rsp.add("highlighting", sumData);
}

15
src/java/org/apache/solr/schema/FieldType.java
View File

@ -27,6 +27,7 @@ import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.OrdFieldSource;
import org.apache.solr.search.Sorting;
import org.apache.solr.search.QParser;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
import org.apache.solr.analysis.SolrAnalyzer;
@ -69,6 +70,14 @@ public abstract class FieldType extends FieldProperties {
protected void init(IndexSchema schema, Map<String,String> args) {
}
protected String getArg(String n, Map<String,String> args) {
String s = args.remove(n);
if (s == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Mising parameter '"+n+"' for FieldType=" + typeName +args);
}
return s;
}
// Handle additional arguments...
void setArgs(IndexSchema schema, Map<String,String> args) {
// default to STORED and INDEXED, and MULTIVALUED depending on schema version
@ -394,7 +403,13 @@ public abstract class FieldType extends FieldProperties {
/** called to get the default value source (normally, from the
* Lucene FieldCache.)
*/
public ValueSource getValueSource(SchemaField field, QParser parser) {
return getValueSource(field);
}
@Deprecated
public ValueSource getValueSource(SchemaField field) {
return new OrdFieldSource(field.name);
}
}

2
src/java/org/apache/solr/schema/IndexSchema.java
View File

@ -317,7 +317,7 @@ public final class IndexSchema {
version = schemaConf.getFloat("/schema/@version", 1.0f);
final IndexSchema schema = this;
AbstractPluginLoader<FieldType> loader = new AbstractPluginLoader<FieldType>( "[schema.xml] fieldType" ) {
AbstractPluginLoader<FieldType> loader = new AbstractPluginLoader<FieldType>( "[schema.xml] fieldType", true, true) {
@Override
protected FieldType create( Config config, String name, String className, Node node ) throws Exception

70
src/java/org/apache/solr/search/BoostQParserPlugin.java Executable file
View File

@ -0,0 +1,70 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.function.BoostedQuery;
import org.apache.solr.search.function.FunctionQuery;
import org.apache.solr.search.function.QueryValueSource;
import org.apache.solr.search.function.ValueSource;
/**
* Create a boosted query from the input value. The main value is the query to be boosted.
* <br>Other parameters: <code>b</code>, the function query to use as the boost.
* <br>Example: <code>&lt;!boost b=log(popularity)&gt;foo</code> creates a query "foo"
* which is boosted (scores are multiplied) by the function query <code>log(popularity</code>.
* The query to be boosted may be of any type.
*/
public class BoostQParserPlugin extends QParserPlugin {
public static String NAME = "boost";
public static String BOOSTFUNC = "b";
public void init(NamedList args) {
}
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new QParser(qstr, localParams, params, req) {
QParser baseParser;
public Query parse() throws ParseException {
String b = localParams.get(BOOSTFUNC);
baseParser = subQuery(localParams.get(QueryParsing.V), null);
Query q = baseParser.parse();
if (b == null) return q;
Query bq = subQuery(b, FunctionQParserPlugin.NAME).parse();
ValueSource vs;
if (bq instanceof FunctionQuery) {
vs = ((FunctionQuery)bq).getValueSource();
} else {
vs = new QueryValueSource(q, 0.0f);
}
return new BoostedQuery(q, vs);
}
public String[] getDefaultHighlightFields() {
return baseParser.getDefaultHighlightFields();
}
};
}
}

231
src/java/org/apache/solr/search/DisMaxQParserPlugin.java Executable file
View File

@ -0,0 +1,231 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.DefaultSolrParams;
import org.apache.solr.common.params.DisMaxParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.util.SolrPluginUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Create a dismax query from the input value.
* <br>Other parameters: all main query related parameters from the {@link org.apache.solr.handler.DisMaxRequestHandler} are supported.
* localParams are checked before global request params.
* <br>Example: <code>&lt;!dismax qf=myfield,mytitle^2&gt;foo</code> creates a dismax query across
* across myfield and mytitle, with a higher weight on mytitle.
*/
public class DisMaxQParserPlugin extends QParserPlugin {
public static String NAME = "dismax";
public void init(NamedList args) {
}
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new DismaxQParser(qstr, localParams, params, req);
}
}
class DismaxQParser extends QParser {
/**
* A field we can't ever find in any schema, so we can safely tell
* DisjunctionMaxQueryParser to use it as our defaultField, and
* map aliases from it to any field in our schema.
*/
private static String IMPOSSIBLE_FIELD_NAME = "\uFFFC\uFFFC\uFFFC";
/** shorten the class references for utilities */
private static class U extends SolrPluginUtils {
/* :NOOP */
}
/** shorten the class references for utilities */
private static interface DMP extends DisMaxParams {
/* :NOOP */
}
public DismaxQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
super(qstr, localParams, params, req);
}
Map<String,Float> queryFields;
Query parsedUserQuery;
public Query parse() throws ParseException {
SolrParams solrParams = localParams == null ? params : new DefaultSolrParams(localParams, params);
IndexSchema schema = req.getSchema();
queryFields = U.parseFieldBoosts(solrParams.getParams(DMP.QF));
Map<String,Float> phraseFields = U.parseFieldBoosts(solrParams.getParams(DMP.PF));
float tiebreaker = solrParams.getFloat(DMP.TIE, 0.0f);
int pslop = solrParams.getInt(DMP.PS, 0);
int qslop = solrParams.getInt(DMP.QS, 0);
/* a generic parser for parsing regular lucene queries */
QueryParser p = schema.getSolrQueryParser(null);
/* a parser for dealing with user input, which will convert
* things to DisjunctionMaxQueries
*/
U.DisjunctionMaxQueryParser up =
new U.DisjunctionMaxQueryParser(schema, IMPOSSIBLE_FIELD_NAME);
up.addAlias(IMPOSSIBLE_FIELD_NAME,
tiebreaker, queryFields);
up.setPhraseSlop(qslop);
/* for parsing sloppy phrases using DisjunctionMaxQueries */
U.DisjunctionMaxQueryParser pp =
new U.DisjunctionMaxQueryParser(schema, IMPOSSIBLE_FIELD_NAME);
pp.addAlias(IMPOSSIBLE_FIELD_NAME,
tiebreaker, phraseFields);
pp.setPhraseSlop(pslop);
/* the main query we will execute. we disable the coord because
* this query is an artificial construct
*/
BooleanQuery query = new BooleanQuery(true);
/* * * Main User Query * * */
parsedUserQuery = null;
String userQuery = getString();
Query altUserQuery = null;
if( userQuery == null || userQuery.trim().length() < 1 ) {
// If no query is specified, we may have an alternate
String altQ = solrParams.get( DMP.ALTQ );
if (altQ != null) {
altUserQuery = p.parse(altQ);
query.add( altUserQuery , BooleanClause.Occur.MUST );
} else {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "missing query string" );
}
}
else {
// There is a valid query string
userQuery = U.partialEscape(U.stripUnbalancedQuotes(userQuery)).toString();
String minShouldMatch = solrParams.get(DMP.MM, "100%");
Query dis = up.parse(userQuery);
parsedUserQuery = dis;
if (dis instanceof BooleanQuery) {
BooleanQuery t = new BooleanQuery();
U.flattenBooleanQuery(t, (BooleanQuery)dis);
U.setMinShouldMatch(t, minShouldMatch);
parsedUserQuery = t;
}
query.add(parsedUserQuery, BooleanClause.Occur.MUST);
/* * * Add on Phrases for the Query * * */
/* build up phrase boosting queries */
/* if the userQuery already has some quotes, stip them out.
* we've already done the phrases they asked for in the main
* part of the query, this is to boost docs that may not have
* matched those phrases but do match looser phrases.
*/
String userPhraseQuery = userQuery.replace("\"","");
Query phrase = pp.parse("\"" + userPhraseQuery + "\"");
if (null != phrase) {
query.add(phrase, BooleanClause.Occur.SHOULD);
}
}
/* * * Boosting Query * * */
String[] boostParams = solrParams.getParams(DMP.BQ);
//List<Query> boostQueries = U.parseQueryStrings(req, boostParams);
List<Query> boostQueries=null;
if (boostParams!=null && boostParams.length>0) {
boostQueries = new ArrayList<Query>();
for (String qs : boostParams) {
Query q = subQuery(qs, null).parse();
boostQueries.add(q);
}
}
if (null != boostQueries) {
if(1 == boostQueries.size() && 1 == boostParams.length) {
/* legacy logic */
Query f = boostQueries.get(0);
if (1.0f == f.getBoost() && f instanceof BooleanQuery) {
/* if the default boost was used, and we've got a BooleanQuery
* extract the subqueries out and use them directly
*/
for (Object c : ((BooleanQuery)f).clauses()) {
query.add((BooleanClause)c);
}
} else {
query.add(f, BooleanClause.Occur.SHOULD);
}
} else {
for(Query f : boostQueries) {
query.add(f, BooleanClause.Occur.SHOULD);
}
}
}
/* * * Boosting Functions * * */
String[] boostFuncs = solrParams.getParams(DMP.BF);
if (null != boostFuncs && 0 != boostFuncs.length) {
for (String boostFunc : boostFuncs) {
if(null == boostFunc || "".equals(boostFunc)) continue;
Map<String,Float> ff = SolrPluginUtils.parseFieldBoosts(boostFunc);
for (String f : ff.keySet()) {
Query fq = subQuery(f, FunctionQParserPlugin.NAME).parse();
Float b = ff.get(f);
if (null != b) {
fq.setBoost(b);
}
query.add(fq, BooleanClause.Occur.SHOULD);
}
}
}
return query;
}
@Override
public String[] getDefaultHighlightFields() {
String[] highFields = queryFields.keySet().toArray(new String[0]);
return highFields;
}
@Override
public Query getHighlightQuery() throws ParseException {
return parsedUserQuery;
}
}

143
src/java/org/apache/solr/search/FieldQParserPlugin.java Normal file
View File

@ -0,0 +1,143 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.*;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.TextField;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
/**
* Create a field query from the input value, applying text analysis and constructing a phrase query if appropriate.
* <br>Other parameters: <code>f</code>, the field
* <br>Example: <code>&lt;!field f=myfield&gt;Foo Bar</code> creates a phrase query with "foo" followed by "bar"
* if the analyzer for myfield is a text field with an analyzer that splits on whitespace and lowercases terms.
* This is generally equivalent to the lucene query parser expression <code>myfield:"Foo Bar"</code>
*/
public class FieldQParserPlugin extends QParserPlugin {
public static String NAME = "field";
public void init(NamedList args) {
}
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new QParser(qstr, localParams, params, req) {
public Query parse() throws ParseException {
String field = localParams.get(QueryParsing.F);
String queryText = localParams.get(QueryParsing.V);
FieldType ft = req.getSchema().getFieldType(field);
if (!(ft instanceof TextField)) {
String internal = ft.toInternal(queryText);
return new TermQuery(new Term(field, internal));
}
int phraseSlop = 0;
Analyzer analyzer = req.getSchema().getQueryAnalyzer();
// most of the following code is taken from the Lucene QueryParser
// Use the analyzer to get all the tokens, and then build a TermQuery,
// PhraseQuery, or nothing based on the term count
TokenStream source = analyzer.tokenStream(field, new StringReader(queryText));
ArrayList<Token> lst = new ArrayList<Token>();
Token t;
int positionCount = 0;
boolean severalTokensAtSamePosition = false;
while (true) {
try {
t = source.next();
}
catch (IOException e) {
t = null;
}
if (t == null)
break;
lst.add(t);
if (t.getPositionIncrement() != 0)
positionCount += t.getPositionIncrement();
else
severalTokensAtSamePosition = true;
}
try {
source.close();
}
catch (IOException e) {
// ignore
}
if (lst.size() == 0)
return null;
else if (lst.size() == 1) {
t = lst.get(0);
return new TermQuery(new Term(field, t.termText()));
} else {
if (severalTokensAtSamePosition) {
if (positionCount == 1) {
// no phrase query:
BooleanQuery q = new BooleanQuery(true);
for (int i = 0; i < lst.size(); i++) {
t = (org.apache.lucene.analysis.Token) lst.get(i);
TermQuery currentQuery = new TermQuery(
new Term(field, t.termText()));
q.add(currentQuery, BooleanClause.Occur.SHOULD);
}
return q;
}
else {
// phrase query:
MultiPhraseQuery mpq = new MultiPhraseQuery();
mpq.setSlop(phraseSlop);
ArrayList multiTerms = new ArrayList();
for (int i = 0; i < lst.size(); i++) {
t = (org.apache.lucene.analysis.Token) lst.get(i);
if (t.getPositionIncrement() == 1 && multiTerms.size() > 0) {
mpq.add((Term[])multiTerms.toArray(new Term[0]));
multiTerms.clear();
}
multiTerms.add(new Term(field, t.termText()));
}
mpq.add((Term[])multiTerms.toArray(new Term[0]));
return mpq;
}
}
else {
PhraseQuery q = new PhraseQuery();
q.setSlop(phraseSlop);
for (int i = 0; i < lst.size(); i++) {
q.add(new Term(field, lst.get(i).termText()));
}
return q;
}
}
}
};
}
}

287
src/java/org/apache/solr/search/FunctionQParser.java Executable file
View File

@ -0,0 +1,287 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.function.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class FunctionQParser extends QParser {
public FunctionQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
super(qstr, localParams, params, req);
}
QueryParsing.StrParser sp;
public Query parse() throws ParseException {
sp = new QueryParsing.StrParser(getString());
ValueSource vs = parseValSource();
/*** boost promoted to top-level query type to avoid this hack
// HACK - if this is a boosted query wrapped in a value-source, return
// that boosted query instead of a FunctionQuery
if (vs instanceof QueryValueSource) {
Query q = ((QueryValueSource)vs).getQuery();
if (q instanceof BoostedQuery) return q;
}
***/
return new FunctionQuery(vs);
}
private abstract static class VSParser {
abstract ValueSource parse(FunctionQParser fp) throws ParseException;
}
private static Map<String, VSParser> vsParsers = new HashMap<String, VSParser>();
static {
vsParsers.put("ord", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
String field = fp.sp.getId();
return new OrdFieldSource(field);
}
});
vsParsers.put("rord", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
String field = fp.sp.getId();
return new ReverseOrdFieldSource(field);
}
});
vsParsers.put("linear", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
fp.sp.expect(",");
float slope = fp.sp.getFloat();
fp.sp.expect(",");
float intercept = fp.sp.getFloat();
return new LinearFloatFunction(source,slope,intercept);
}
});
vsParsers.put("max", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
fp.sp.expect(",");
float val = fp.sp.getFloat();
return new MaxFloatFunction(source,val);
}
});
vsParsers.put("recip", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
fp.sp.expect(",");
float m = fp.sp.getFloat();
fp.sp.expect(",");
float a = fp.sp.getFloat();
fp.sp.expect(",");
float b = fp.sp.getFloat();
return new ReciprocalFloatFunction(source,m,a,b);
}
});
vsParsers.put("scale", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
fp.sp.expect(",");
float min = fp.sp.getFloat();
fp.sp.expect(",");
float max = fp.sp.getFloat();
return new ScaleFloatFunction(source,min,max);
}
});
vsParsers.put("pow", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource a = fp.parseValSource();
fp.sp.expect(",");
ValueSource b = fp.parseValSource();
return new PowFloatFunction(a,b);
}
});
vsParsers.put("div", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource a = fp.parseValSource();
fp.sp.expect(",");
ValueSource b = fp.parseValSource();
return new DivFloatFunction(a,b);
}
});
vsParsers.put("map", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
fp.sp.expect(",");
float min = fp.sp.getFloat();
fp.sp.expect(",");
float max = fp.sp.getFloat();
fp.sp.expect(",");
float target = fp.sp.getFloat();
return new RangeMapFloatFunction(source,min,max,target);
}
});
vsParsers.put("sqrt", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
return new SimpleFloatFunction(source) {
protected String name() {
return "sqrt";
}
protected float func(int doc, DocValues vals) {
return (float)Math.sqrt(vals.floatVal(doc));
}
};
}
});
vsParsers.put("log", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
return new SimpleFloatFunction(source) {
protected String name() {
return "log";
}
protected float func(int doc, DocValues vals) {
return (float)Math.log10(vals.floatVal(doc));
}
};
}
});
vsParsers.put("abs", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
ValueSource source = fp.parseValSource();
return new SimpleFloatFunction(source) {
protected String name() {
return "abs";
}
protected float func(int doc, DocValues vals) {
return (float)Math.abs(vals.floatVal(doc));
}
};
}
});
vsParsers.put("sum", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
List<ValueSource> sources = fp.parseValueSourceList();
return new SumFloatFunction(sources.toArray(new ValueSource[sources.size()]));
}
});
vsParsers.put("product", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
List<ValueSource> sources = fp.parseValueSourceList();
return new ProductFloatFunction(sources.toArray(new ValueSource[sources.size()]));
}
});
vsParsers.put("query", new VSParser() {
// boost(query($q),rating)
ValueSource parse(FunctionQParser fp) throws ParseException {
Query q = fp.getNestedQuery();
float defVal = 0.0f;
if (fp.sp.opt(",")) {
defVal = fp.sp.getFloat();
}
return new QueryValueSource(q, defVal);
}
});
vsParsers.put("boost", new VSParser() {
ValueSource parse(FunctionQParser fp) throws ParseException {
Query q = fp.getNestedQuery();
fp.sp.expect(",");
ValueSource vs = fp.parseValSource();
BoostedQuery bq = new BoostedQuery(q, vs);
System.out.println("Constructed Boostedquery " + bq);
return new QueryValueSource(bq, 0.0f);
}
});
}
private List<ValueSource> parseValueSourceList() throws ParseException {
List<ValueSource> sources = new ArrayList<ValueSource>(3);
for (;;) {
sources.add(parseValSource());
char ch = sp.peek();
if (ch==')') break;
sp.expect(",");
}
return sources;
}
private ValueSource parseValSource() throws ParseException {
int ch = sp.peek();
if (ch>='0' && ch<='9' || ch=='.' || ch=='+' || ch=='-') {
return new ConstValueSource(sp.getFloat());
}
String id = sp.getId();
if (sp.opt("(")) {
// a function... look it up.
VSParser argParser = vsParsers.get(id);
if (argParser==null) {
throw new ParseException("Unknown function " + id + " in FunctionQuery(" + sp + ")");
}
ValueSource vs = argParser.parse(this);
sp.expect(")");
return vs;
}
SchemaField f = req.getSchema().getField(id);
return f.getType().getValueSource(f, this);
}
private Query getNestedQuery() throws ParseException {
if (sp.opt("$")) {
String param = sp.getId();
sp.pos += param.length();
String qstr = getParam(param);
qstr = qstr==null ? "" : qstr;
return subQuery(qstr, null).parse();
}
int start = sp.pos;
int end = sp.pos;
String v = sp.val;
String qs = v.substring(start);
HashMap nestedLocalParams = new HashMap<String,String>();
end = QueryParsing.parseLocalParams(qs, start, nestedLocalParams, getParams());
QParser sub;
if (end>start) {
if (nestedLocalParams.get(QueryParsing.V) != null) {
// value specified directly in local params... so the end of the
// query should be the end of the local params.
sub = subQuery(qs.substring(0, end), null);
} else {
// value here is *after* the local params... ask the parser.
sub = subQuery(qs, null);
// int subEnd = sub.findEnd(')');
// TODO.. implement functions to find the end of a nested query
throw new ParseException("Nested local params must have value in v parameter. got '" + qs + "'");
}
} else {
throw new ParseException("Nested function query must use $param or <!v=value> forms. got '" + qs + "'");
}
sp.pos += end-start; // advance past nested query
return sub.getQuery();
}
}

37
src/java/org/apache/solr/search/FunctionQParserPlugin.java Normal file
View File

@ -0,0 +1,37 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
/**
* Create a function query from the input value.
* <br>Other parameters: none
* <br>Example: <code>&lt;!func&gt;log(foo)</code>
*/
public class FunctionQParserPlugin extends QParserPlugin {
public static String NAME = "func";
public void init(NamedList args) {
}
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new FunctionQParser(qstr, localParams, params, req);
}
}

128
src/java/org/apache/solr/search/LuceneQParserPlugin.java Executable file
View File

@ -0,0 +1,128 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.Query;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.request.SolrQueryRequest;
import java.util.List;
/**
* Parse Solr's variant on the Lucene QueryParser syntax.
* <br>Other parameters:<ul>
* <li>q.op - the default operator "OR" or "AND"</li>
* <li>df - the default field name</li>
* <li>df - the default field name</li>
* </ul>
* <br>Example: <code>&lt;!lucene q.op=AND df=text sort='price asc'&gt;myfield:foo +bar -baz</code>
*/
public class LuceneQParserPlugin extends QParserPlugin {
public static String NAME = "lucene";
public void init(NamedList args) {
}
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new LuceneQParser(qstr, localParams, params, req);
}
}
class LuceneQParser extends QParser {
String sortStr;
SolrQueryParser lparser;
public LuceneQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
super(qstr, localParams, params, req);
}
public Query parse() throws ParseException {
String qstr = getString();
String defaultField = getParam(CommonParams.DF);
if (defaultField==null) {
defaultField = getReq().getSchema().getDefaultSearchFieldName();
}
lparser = new SolrQueryParser(this, defaultField);
// these could either be checked & set here, or in the SolrQueryParser constructor
String opParam = getParam(QueryParsing.OP);
if (opParam != null) {
lparser.setDefaultOperator("AND".equals(opParam) ? QueryParser.Operator.AND : QueryParser.Operator.OR);
}
return lparser.parse(qstr);
}
public String[] getDefaultHighlightFields() {
return new String[]{lparser.getField()};
}
}
class OldLuceneQParser extends LuceneQParser {
String sortStr;
public OldLuceneQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
super(qstr, localParams, params, req);
}
public Query parse() throws ParseException {
// handle legacy "query;sort" syntax
if (getLocalParams() == null) {
String qstr = getString();
sortStr = getParams().get(CommonParams.SORT);
if (sortStr == null) {
// sort may be legacy form, included in the query string
List<String> commands = StrUtils.splitSmart(qstr,';');
if (commands.size() == 2) {
qstr = commands.get(0);
sortStr = commands.get(1);
} else if (commands.size() == 1) {
// This is need to support the case where someone sends: "q=query;"
qstr = commands.get(0);
}
else if (commands.size() > 2) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "If you want to use multiple ';' in the query, use the 'sort' param.");
}
}
setString(qstr);
}
return super.parse();
}
@Override
public QueryParsing.SortSpec getSort(boolean useGlobal) throws ParseException {
QueryParsing.SortSpec sort = super.getSort(useGlobal);
if (sortStr != null && sortStr.length()>0 && sort.getSort()==null) {
QueryParsing.SortSpec oldSort = QueryParsing.parseSort(sortStr, getReq().getSchema());
sort.sort = oldSort.sort;
}
return sort;
}
}

37
src/java/org/apache/solr/search/OldLuceneQParserPlugin.java Executable file
View File

@ -0,0 +1,37 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
/**
* Parse Solr's variant of Lucene QueryParser syntax, including the
* deprecated sort specification after the query.
* <br>Example: <code>&lt;!lucenePlusSort&gt;myfield:foo +bar -baz;price asc</code>
*/
public class OldLuceneQParserPlugin extends QParserPlugin {
public static String NAME = "lucenePlusSort";
public void init(NamedList args) {
}
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new OldLuceneQParser(qstr, localParams, params, req);
}
}

47
src/java/org/apache/solr/search/PrefixQParserPlugin.java Executable file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
/**
* Create a prefix query from the input value. Currently no analysis or
* value transformation is done to create this prefix query (subject to change).
* <br>Other parameters: <code>f</code>, the field
* <br>Example: <code>&lt;!prefix f=myfield&gt;foo</code> is generally equivalent
* to the lucene query parser expression <code>myfield:foo*</code>
*/
public class PrefixQParserPlugin extends QParserPlugin {
public static String NAME = "prefix";
public void init(NamedList args) {
}
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new QParser(qstr, localParams, params, req) {
public Query parse() throws ParseException {
return new PrefixQuery(new Term(localParams.get(QueryParsing.F), localParams.get(QueryParsing.V)));
}
};
}
}

202
src/java/org/apache/solr/search/QParser.java Executable file
View File

@ -0,0 +1,202 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
public abstract class QParser {
String qstr;
SolrParams params;
SolrParams localParams;
SolrQueryRequest req;
int recurseCount;
Query query;
public QParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
this.qstr = qstr;
this.localParams = localParams;
this.params = params;
this.req = req;
}
/** create and return the <code>Query</code> object represented by <code>qstr</code> */
protected abstract Query parse() throws ParseException;
public SolrParams getLocalParams() {
return localParams;
}
public void setLocalParams(SolrParams localParams) {
this.localParams = localParams;
}
public SolrParams getParams() {
return params;
}
public void setParams(SolrParams params) {
this.params = params;
}
public SolrQueryRequest getReq() {
return req;
}
public void setReq(SolrQueryRequest req) {
this.req = req;
}
public String getString() {
return qstr;
}
public void setString(String s) {
this.qstr = s;
}
public Query getQuery() throws ParseException {
if (query==null) {
query=parse();
}
return query;
}
private void checkRecurse() throws ParseException {
if (recurseCount++ >= 100) {
throw new ParseException("Infinite Recursion detected parsing query '" + qstr + "'");
}
}
// TODO: replace with a SolrParams that defaults to checking localParams first?
// ideas..
// create params that satisfy field-specific overrides
// overrideable syntax $x=foo (set global for limited scope) (invariants & security?)
// $x+=foo (append to global for limited scope)
/** check both local and global params */
protected String getParam(String name) {
String val;
if (localParams != null) {
val = localParams.get(name);
if (val != null) return val;
}
return params.get(name);
}
/** Create a new QParser for parsing an embedded sub-query */
public QParser subQuery(String q, String defaultType) throws ParseException {
checkRecurse();
if (defaultType == null && localParams != null) {
// if not passed, try and get the defaultType from local params
defaultType = localParams.get(QueryParsing.DEFTYPE);
}
QParser nestedParser = getParser(q, defaultType, getReq());
nestedParser.recurseCount = recurseCount;
return nestedParser;
}
/**
* @param useGlobalParams look up sort, start, rows in global params if not in local params
* @return the sort specification
*/
public QueryParsing.SortSpec getSort(boolean useGlobalParams) throws ParseException {
getQuery(); // ensure query is parsed first
String sortStr = null;
String startS = null;
String rowsS = null;
if (localParams != null) {
sortStr = localParams.get(CommonParams.SORT);
startS = localParams.get(CommonParams.START);
rowsS = localParams.get(CommonParams.ROWS);
// if any of these parameters are present, don't go back to the global params
if (sortStr != null || startS != null || rowsS != null) {
useGlobalParams = false;
}
}
if (useGlobalParams) {
if (sortStr ==null) {
sortStr = params.get(CommonParams.SORT);
}
if (startS==null) {
startS = params.get(CommonParams.START);
}
if (rowsS==null) {
rowsS = params.get(CommonParams.ROWS);
}
}
int start = startS != null ? Integer.parseInt(startS) : 0;
int rows = rowsS != null ? Integer.parseInt(rowsS) : 10;
QueryParsing.SortSpec sort;
if (sortStr != null) {
sort = QueryParsing.parseSort(sortStr, req.getSchema());
sort.offset = start;
sort.num = rows;
} else {
sort = new QueryParsing.SortSpec(null, start, rows);
}
return sort;
}
public String[] getDefaultHighlightFields() {
return new String[]{};
}
public Query getHighlightQuery() throws ParseException {
return getQuery();
}
/** Create a <code>QParser</code> to parse <code>qstr</code>,
* assuming that the default query type is <code>defaultType</code>.
* The query type may be overridden by local parameters in the query
* string itself. For example if defaultType=<code>"dismax"</code>
* and qstr=<code>foo</code>, then the dismax query parser will be used
* to parse and construct the query object. However
* if qstr=<code>&lt;!prefix f=myfield&gt;foo</code>
* then the prefix query parser will be used.
*/
public static QParser getParser(String qstr, String defaultType, SolrQueryRequest req) throws ParseException {
SolrParams localParams = QueryParsing.getLocalParams(qstr, req.getParams());
String type;
if (localParams == null) {
type = defaultType;
} else {
type = localParams.get(QueryParsing.TYPE);
qstr = localParams.get("v");
}
type = type==null ? QParserPlugin.DEFAULT_QTYPE : type;
QParserPlugin qplug = req.getCore().getQueryPlugin(type);
return qplug.createParser(qstr, localParams, req.getParams(), req);
}
}

43
src/java/org/apache/solr/search/QParserPlugin.java Executable file
View File

@ -0,0 +1,43 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
public abstract class QParserPlugin implements NamedListInitializedPlugin {
/** internal use - name of the default parser */
public static String DEFAULT_QTYPE="lucene";
/** internal use - name to class mappings of builtin parsers */
public static final Object[] standardPlugins = {
DEFAULT_QTYPE, LuceneQParserPlugin.class,
OldLuceneQParserPlugin.NAME, OldLuceneQParserPlugin.class,
FunctionQParserPlugin.NAME, FunctionQParserPlugin.class,
PrefixQParserPlugin.NAME, PrefixQParserPlugin.class,
BoostQParserPlugin.NAME, BoostQParserPlugin.class,
DisMaxQParserPlugin.NAME, DisMaxQParserPlugin.class,
FieldQParserPlugin.NAME, FieldQParserPlugin.class,
RawQParserPlugin.NAME, RawQParserPlugin.class,
};
/** return a {@link QParser} */
public abstract QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req);
}

382
src/java/org/apache/solr/search/QueryParsing.java
View File

@ -17,26 +17,28 @@
package org.apache.solr.search;
import org.apache.lucene.search.*;
import org.apache.solr.search.function.*;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.*;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.search.function.FunctionQuery;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.regex.Pattern;
import java.util.logging.Level;
import java.io.IOException;
import java.util.regex.Pattern;
/**
* Collection of static utilities usefull for query parsing.
@ -46,6 +48,10 @@ import java.io.IOException;
public class QueryParsing {
/** the SolrParam used to override the QueryParser "default operator" */
public static final String OP = "q.op";
public static final String V = "v"; // value of this parameter
public static final String F = "f"; // field that a query or command pertains to
public static final String TYPE = "type";// type of this query or command
public static final String DEFTYPE = "defType"; // default type for any direct subqueries
/**
* Helper utility for parsing a query using the Lucene QueryParser syntax.
@ -106,16 +112,114 @@ public class QueryParsing {
}
}
// note to self: something needs to detect infinite recursion when parsing queries
static int parseLocalParams(String txt, int start, Map<String,String> target, SolrParams params) throws ParseException {
int off=start;
if (!txt.startsWith("<!",off)) return start;
StrParser p = new StrParser(txt,start,txt.length());
p.pos+=2; // skip over "<!"
for(;;) {
/*
if (p.pos>=txt.length()) {
throw new ParseException("Missing '>' parsing local params '" + txt + '"');
}
*/
char ch = p.peek();
if (ch=='>') {
return p.pos+1;
}
String id = p.getId();
if (id.length()==0) {
throw new ParseException("Expected identifier '>' parsing local params '" + txt + '"');
}
String val=null;
ch = p.peek();
if (ch!='=') {
// single word... treat <!func> as ""=func for easy lookup
val = id;
id = TYPE;
} else {
// saw equals, so read value
p.pos++;
ch = p.peek();
if (ch=='\"' || ch=='\'') {
val = p.getQuotedString();
} else if (ch=='$') {
p.pos++;
// dereference parameter
String pname = p.getId();
if (params!=null) {
val = params.get(pname);
}
} else {
// read unquoted literal ended by whitespace or '>'
// there is no escaping.
int valStart = p.pos;
for (;;) {
if (p.pos >= p.end) {
throw new ParseException("Missing end to unquoted value starting at " + valStart + " str='" + txt +"'");
}
char c = p.val.charAt(p.pos);
if (c=='>' || Character.isWhitespace(c)) {
val = p.val.substring(valStart, p.pos);
break;
}
p.pos++;
}
}
}
if (target != null) target.put(id,val);
}
}
/**
* "foo" returns null
* "<!prefix f=myfield>yes" returns type="prefix",f="myfield",v="yes"
* "<!prefix f=myfield v=$p>" returns type="prefix",f="myfield",v=params.get("p")
*/
public static SolrParams getLocalParams(String txt, SolrParams params) throws ParseException {
if (!txt.startsWith("<!")) {
return null;
}
Map<String,String> localParams = new HashMap<String,String>();
int start = QueryParsing.parseLocalParams(txt, 0, localParams, params);
String val;
if (start >= txt.length()) {
// if the rest of the string is empty, check for "v" to provide the value
val = localParams.get(V);
val = val==null ? "" : val;
} else {
val = txt.substring(start);
}
localParams.put(V,val);
return new MapSolrParams(localParams);
}
/***
* SortSpec encapsulates a Lucene Sort and a count of the number of documents
* to return.
*/
public static class SortSpec {
private final Sort sort;
private final int num;
Sort sort;
int num;
int offset;
SortSpec(Sort sort, int num) {
this(sort,0,num);
}
SortSpec(Sort sort, int offset, int num) {
this.sort=sort;
this.offset=offset;
this.num=num;
}
@ -125,12 +229,22 @@ public class QueryParsing {
*/
public Sort getSort() { return sort; }
/**
* Offset into the list of results.
*/
public int getOffset() { return offset; }
/**
* Gets the number of documens to return after sorting.
*
* @return number of docs to return, or -1 for no cut off (just sort)
*/
public int getCount() { return num; }
public String toString() {
return "start="+offset+"&rows="+num
+ (sort==null ? "" : "sort="+sort);
}
}
@ -407,17 +521,29 @@ public class QueryParsing {
// simple class to help with parsing a string
private static class StrParser {
static class StrParser {
String val;
int pos;
int end;
StrParser(String val) {this.val = val; end=val.length(); }
StrParser(String val) {
this(val,0,val.length());
}
StrParser(String val, int start, int end) {
this.val = val;
this.pos = start;
this.end = end;
}
void eatws() {
while (pos<end && Character.isWhitespace(val.charAt(pos))) pos++;
}
void skip(int nChars) {
pos = Math.max(pos+nChars, end);
}
boolean opt(String s) {
eatws();
int slen=s.length();
@ -428,6 +554,16 @@ public class QueryParsing {
return false;
}
boolean opt(char ch) {
eatws();
if (val.charAt(pos) == ch) {
pos++;
return true;
}
return false;
}
void expect(String s) throws ParseException {
eatws();
int slen=s.length();
@ -461,15 +597,56 @@ public class QueryParsing {
String getId() throws ParseException {
eatws();
int id_start=pos;
while (pos<end && Character.isJavaIdentifierPart(val.charAt(pos))) pos++;
return val.substring(id_start, pos);
if (pos<end && Character.isJavaIdentifierStart(val.charAt(pos))) {
pos++;
while (pos<end) {
char ch = val.charAt(pos);
if (!Character.isJavaIdentifierPart(ch) && ch!='.') {
break;
}
pos++;
}
return val.substring(id_start, pos);
}
throw new ParseException("Expected identifier at pos " + pos + " str='" + val + "'");
}
// return null if not a string
String getQuotedString() throws ParseException {
eatws();
char delim = peekChar();
if (!(delim=='\"' || delim=='\'')) {
return null;
}
int val_start = ++pos;
StringBuilder sb = new StringBuilder(); // needed for escaping
for(;;) {
if (pos>=end) {
throw new ParseException("Missing end quote for string at pos " + (val_start-1) + " str='"+val+"'");
}
char ch = val.charAt(pos);
if (ch=='\\') {
ch = pos<end ? val.charAt(pos++) : 0;
} else if (ch==delim) {
pos++;
return sb.toString();
}
sb.append(ch);
pos++;
}
}
// next non-whitespace char
char peek() {
eatws();
return pos<end ? val.charAt(pos) : 0;
}
// next char
char peekChar() {
return pos<end ? val.charAt(pos) : 0;
}
public String toString() {
return "'" + val + "'" + ", pos=" + pos;
}
@ -487,177 +664,6 @@ public class QueryParsing {
return out;
}
private abstract static class VSParser {
abstract ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException;
}
private static Map<String, VSParser> vsParsers = new HashMap<String, VSParser>();
static {
vsParsers.put("ord", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
String field = sp.getId();
return new OrdFieldSource(field);
}
});
vsParsers.put("rord", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
String field = sp.getId();
return new ReverseOrdFieldSource(field);
}
});
vsParsers.put("linear", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
ValueSource source = parseValSource(sp, schema);
sp.expect(",");
float slope = sp.getFloat();
sp.expect(",");
float intercept = sp.getFloat();
return new LinearFloatFunction(source,slope,intercept);
}
});
vsParsers.put("max", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
ValueSource source = parseValSource(sp, schema);
sp.expect(",");
float val = sp.getFloat();
return new MaxFloatFunction(source,val);
}
});
vsParsers.put("recip", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
ValueSource source = parseValSource(sp,schema);
sp.expect(",");
float m = sp.getFloat();
sp.expect(",");
float a = sp.getFloat();
sp.expect(",");
float b = sp.getFloat();
return new ReciprocalFloatFunction(source,m,a,b);
}
});
vsParsers.put("scale", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
ValueSource source = parseValSource(sp,schema);
sp.expect(",");
float min = sp.getFloat();
sp.expect(",");
float max = sp.getFloat();
return new ScaleFloatFunction(source,min,max);
}
});
vsParsers.put("pow", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
ValueSource a = parseValSource(sp,schema);
sp.expect(",");
ValueSource b = parseValSource(sp,schema);
return new PowFloatFunction(a,b);
}
});
vsParsers.put("div", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
ValueSource a = parseValSource(sp,schema);
sp.expect(",");
ValueSource b = parseValSource(sp,schema);
return new DivFloatFunction(a,b);
}
});
vsParsers.put("map", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
ValueSource source = parseValSource(sp,schema);
sp.expect(",");
float min = sp.getFloat();
sp.expect(",");
float max = sp.getFloat();
sp.expect(",");
float target = sp.getFloat();
return new RangeMapFloatFunction(source,min,max,target);
}
});
vsParsers.put("sqrt", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
ValueSource source = parseValSource(sp,schema);
return new SimpleFloatFunction(source) {
protected String name() {
return "sqrt";
}
protected float func(int doc, DocValues vals) {
return (float)Math.sqrt(vals.floatVal(doc));
}
};
}
});
vsParsers.put("log", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
ValueSource source = parseValSource(sp,schema);
return new SimpleFloatFunction(source) {
protected String name() {
return "log";
}
protected float func(int doc, DocValues vals) {
return (float)Math.log10(vals.floatVal(doc));
}
};
}
});
vsParsers.put("abs", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
ValueSource source = parseValSource(sp,schema);
return new SimpleFloatFunction(source) {
protected String name() {
return "abs";
}
protected float func(int doc, DocValues vals) {
return (float)Math.abs(vals.floatVal(doc));
}
};
}
});
vsParsers.put("sum", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
List<ValueSource> sources = parseValueSourceList(sp,schema);
return new SumFloatFunction(sources.toArray(new ValueSource[sources.size()]));
}
});
vsParsers.put("product", new VSParser() {
ValueSource parse(StrParser sp, IndexSchema schema) throws ParseException {
List<ValueSource> sources = parseValueSourceList(sp,schema);
return new ProductFloatFunction(sources.toArray(new ValueSource[sources.size()]));
}
});
}
private static List<ValueSource> parseValueSourceList(StrParser sp, IndexSchema schema) throws ParseException {
List<ValueSource> sources = new ArrayList<ValueSource>(3);
for (;;) {
sources.add(parseValSource(sp,schema));
char ch = sp.peek();
if (ch==')') break;
sp.expect(",");
}
return sources;
}
private static ValueSource parseValSource(StrParser sp, IndexSchema schema) throws ParseException {
int ch = sp.peek();
if (ch>='0' && ch<='9' || ch=='.' || ch=='+' || ch=='-') {
return new ConstValueSource(sp.getFloat());
}
String id = sp.getId();
if (sp.opt("(")) {
// a function... look it up.
VSParser argParser = vsParsers.get(id);
if (argParser==null) {
throw new ParseException("Unknown function " + id + " in FunctionQuery(" + sp + ")");
}
ValueSource vs = argParser.parse(sp, schema);
sp.expect(")");
return vs;
}
SchemaField f = schema.getField(id);
return f.getType().getValueSource(f);
}
/**
* Parse a function, returning a FunctionQuery
*
@ -668,7 +674,7 @@ public class QueryParsing {
* <pre>
* // Numeric fields default to correct type
* // (ie: IntFieldSource or FloatFieldSource)
* // Others use implicit ord(...) to generate numeric field value
* // Others use explicit ord(...) to generate numeric field value
* myfield
*
* // OrdFieldSource
@ -694,7 +700,9 @@ public class QueryParsing {
* </pre>
*/
public static FunctionQuery parseFunction(String func, IndexSchema schema) throws ParseException {
return new FunctionQuery(parseValSource(new StrParser(func), schema));
SolrCore core = SolrCore.getSolrCore();
return (FunctionQuery)(QParser.getParser(func,"func",new LocalSolrQueryRequest(core,new HashMap())).parse());
// return new FunctionQuery(parseValSource(new StrParser(func), schema));
}
}

45
src/java/org/apache/solr/search/RawQParserPlugin.java Normal file
View File

@ -0,0 +1,45 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
/**
* Create a term query from the input value without any text analysis or transformation whatsoever.
* <br>Other parameters: <code>f</code>, the field
* <br>Example: <code>&lt;!raw f=myfield&gt;Foo Bar</code> creates <code>TermQuery(Term("myfield","Foo Bar"))</code>
*/
public class RawQParserPlugin extends QParserPlugin {
public static String NAME = "raw";
public void init(NamedList args) {
}
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new QParser(qstr, localParams, params, req) {
public Query parse() throws ParseException {
return new TermQuery(new Term(localParams.get(QueryParsing.F), localParams.get(QueryParsing.V)));
}
};
}
}

32
src/java/org/apache/solr/search/SolrQueryParser.java
View File

@ -17,12 +17,13 @@
package org.apache.solr.search;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.*;
import org.apache.lucene.index.Term;
import org.apache.solr.schema.IndexSchema;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.ConstantScoreRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
// TODO: implement the analysis of simple fields with
// FieldType.toInternal() instead of going through the
@ -49,6 +50,7 @@ import org.apache.solr.schema.FieldType;
*/
public class SolrQueryParser extends QueryParser {
protected final IndexSchema schema;
protected final QParser parser;
/**
* Constructs a SolrQueryParser using the schema to understand the
@ -63,14 +65,32 @@ public class SolrQueryParser extends QueryParser {
public SolrQueryParser(IndexSchema schema, String defaultField) {
super(defaultField == null ? schema.getDefaultSearchFieldName() : defaultField, schema.getQueryAnalyzer());
this.schema = schema;
this.parser = null;
setLowercaseExpandedTerms(false);
}
public SolrQueryParser(QParser parser, String defaultField) {
super(defaultField, parser.getReq().getSchema().getQueryAnalyzer());
this.schema = parser.getReq().getSchema();
this.parser = parser;
setLowercaseExpandedTerms(false);
}
protected Query getFieldQuery(String field, String queryText) throws ParseException {
// intercept magic field name of "_" to use as a hook for our
// own functions.
if (field.equals("_val_")) {
return QueryParsing.parseFunction(queryText, schema);
if (field.charAt(0) == '_') {
if ("_val_".equals(field)) {
if (parser==null) {
return QueryParsing.parseFunction(queryText, schema);
} else {
QParser nested = parser.subQuery(queryText, "func");
return nested.getQuery();
}
} else if ("_query_".equals(field) && parser != null) {
return parser.subQuery(queryText, null).getQuery();
}
}
// default to a normal field query

179
src/test/org/apache/solr/search/TestQueryTypes.java Executable file
View File

@ -0,0 +1,179 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.solr.util.AbstractSolrTestCase;
public class TestQueryTypes extends AbstractSolrTestCase {
public String getSchemaFile() { return "schema11.xml"; }
public String getSolrConfigFile() { return "solrconfig.xml"; }
public String getCoreName() { return "basic"; }
public void setUp() throws Exception {
// if you override setUp or tearDown, you better call
// the super classes version
super.setUp();
}
public void tearDown() throws Exception {
// if you override setUp or tearDown, you better call
// the super classes version
super.tearDown();
}
public void testQueryTypes() {
assertU(adoc("id","1", "v_t","Hello Dude"));
assertU(adoc("id","2", "v_t","Hello Yonik"));
assertU(adoc("id","3", "v_s","<!literal>"));
assertU(adoc("id","4", "v_s","other stuff"));
assertU(adoc("id","5", "v_f","3.14159"));
assertU(adoc("id","6", "v_f","8983"));
assertU(adoc("id","7", "v_f","1.5"));
assertU(optimize());
// Some basic tests to ensure that parsing local params is working
assertQ("test prefix query",
req("q","<!prefix f=v_t>hel")
,"//result[@numFound='2']"
);
assertQ("test raw query",
req("q","<!raw f=v_t>hello")
,"//result[@numFound='2']"
);
assertQ("test raw query",
req("q","<!raw f=v_t>Hello")
,"//result[@numFound='0']"
);
assertQ("test raw query",
req("q","<!raw f=v_f>1.5")
,"//result[@numFound='0']"
);
assertQ("test single term field query on text type",
req("q","<!field f=v_t>HELLO")
,"//result[@numFound='2']"
);
assertQ("test single term field query on type with diff internal rep",
req("q","<!field f=v_f>1.5")
,"//result[@numFound='1']"
);
assertQ("test multi term field query on text type",
req("q","<!field f=v_t>Hello DUDE")
,"//result[@numFound='1']"
);
assertQ("test prefix query with value in local params",
req("q","<!prefix f=v_t v=hel>")
,"//result[@numFound='2']"
);
assertQ("test optional quotes",
req("q","<!prefix f='v_t' v=\"hel\">")
,"//result[@numFound='2']"
);
assertQ("test extra whitespace",
req("q","<!prefix f=v_t v=hel >")
,"//result[@numFound='2']"
);
assertQ("test literal with <! in it",
req("q","<!prefix f=v_s><!lit")
,"//result[@numFound='1']"
);
assertQ("test param subst",
req("q","<!prefix f=$myf v=$my.v>"
,"myf","v_t", "my.v", "hel"
)
,"//result[@numFound='2']"
);
assertQ("test param subst with literal",
req("q","<!prefix f=$myf v=$my.v>"
,"myf","v_s", "my.v", "<!lit"
)
,"//result[@numFound='1']"
);
// lucene queries
assertQ("test lucene query",
req("q","<!lucene>v_t:hel*")
,"//result[@numFound='2']"
);
// lucene queries
assertQ("test lucene default field",
req("q","<!df=v_t>hel*")
,"//result[@numFound='2']"
);
// lucene operator
assertQ("test lucene operator",
req("q","<!q.op=OR df=v_t>Hello Yonik")
,"//result[@numFound='2']"
);
assertQ("test lucene operator",
req("q","<!q.op=AND df=v_t>Hello Yonik")
,"//result[@numFound='1']"
);
// test boost queries
assertQ("test boost",
req("q","<!boost b=sum(v_f,1)>id:[5 TO 6]"
,"fl","*,score"
)
,"//result[@numFound='2']"
,"//doc[./float[@name='v_f']='3.14159' and ./float[@name='score']='4.14159']"
);
assertQ("test boost and default type of func",
req("q","<!boost v=$q1 b=$q2>"
,"q1", "<!func>v_f", "q2","v_f"
,"fl","*,score"
)
,"//doc[./float[@name='v_f']='1.5' and ./float[@name='score']='2.25']"
);
// dismax query from std request handler
assertQ("test dismax query",
req("q","<!dismax>hello"
,"qf","v_t"
,"bf","sqrt(v_f)^100 log(sum(v_f,1))^50"
,"bq","<!prefix f=v_t>he"
,"debugQuery","on"
)
,"//result[@numFound='2']"
);
// dismax query from std request handler, using local params
assertQ("test dismax query w/ local params",
req("q","<!dismax qf=v_t>hello"
,"qf","v_f"
)
,"//result[@numFound='2']"
);
}
}