diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 002521d44bf..d12f63fbc42 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -558,6 +558,9 @@ Other Changes * SOLR-4254: Harden the 'leader requests replica to recover' code path. (Mark Miller, yonik) +* SOLR-4226: Extract fl parsing code out of ReturnFields constructor. + (Ryan Ernst via Robert Muir) + ================== 4.0.0 ================== Versions of Major Components diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index 3bb55d9ead2..763f4267e58 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -76,7 +76,7 @@ public class MoreLikeThisHandler extends RequestHandlerBase SolrParams params = req.getParams(); // Set field flags - ReturnFields returnFields = new ReturnFields( req ); + ReturnFields returnFields = new SolrReturnFields( req ); rsp.setReturnFields( returnFields ); int flags = 0; if (returnFields.wantsScore()) { diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index e9716022d39..af9c09d82c9 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -93,7 +93,7 @@ public class QueryComponent extends SearchComponent SolrQueryResponse rsp = rb.rsp; // Set field flags - ReturnFields returnFields = new ReturnFields( req ); + ReturnFields returnFields = new SolrReturnFields( req ); rsp.setReturnFields( returnFields ); int flags = 0; if (returnFields.wantsScore()) { diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java index bba3361b3f5..389fdedb57e 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java @@ -56,6 +56,7 @@ import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.ReturnFields; import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SolrReturnFields; import org.apache.solr.update.DocumentBuilder; import org.apache.solr.update.PeerSync; import org.apache.solr.update.UpdateLog; @@ -72,7 +73,7 @@ public class RealTimeGetComponent extends SearchComponent @Override public void prepare(ResponseBuilder rb) throws IOException { // Set field flags - ReturnFields returnFields = new ReturnFields( rb.req ); + ReturnFields returnFields = new SolrReturnFields( rb.req ); rb.rsp.setReturnFields( returnFields ); } diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java index 1bd36a9efa0..33788471c76 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java @@ -36,6 +36,7 @@ import org.apache.solr.search.ReturnFields; import org.apache.solr.search.DocList; import org.apache.solr.search.DocListAndSet; import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SolrReturnFields; import org.apache.solr.util.SolrPluginUtils; import org.apache.solr.util.plugin.SolrCoreAware; @@ -111,7 +112,7 @@ public class TermVectorComponent extends SearchComponent implements SolrCoreAwar (1 == fldLst.length && 0 == fldLst[0].length())) { // no tv.fl, parse the main fl - ReturnFields rf = new ReturnFields + ReturnFields rf = new SolrReturnFields (params.getParams(CommonParams.FL), rb.req); if (rf.wantsAllFields()) { diff --git a/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java b/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java index c3c75e8bf7c..64c134b1aa8 100644 --- a/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java +++ b/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java @@ -20,6 +20,7 @@ package org.apache.solr.response; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.search.ReturnFields; +import org.apache.solr.search.SolrReturnFields; import java.util.*; @@ -123,7 +124,7 @@ public class SolrQueryResponse { */ public ReturnFields getReturnFields() { if( returnFields == null ) { - returnFields = new ReturnFields(); // by default return everything + returnFields = new SolrReturnFields(); // by default return everything } return returnFields; } diff --git a/solr/core/src/java/org/apache/solr/search/ReturnFields.java b/solr/core/src/java/org/apache/solr/search/ReturnFields.java index 3b33faf08f3..bcc99791774 100644 --- a/solr/core/src/java/org/apache/solr/search/ReturnFields.java +++ b/solr/core/src/java/org/apache/solr/search/ReturnFields.java @@ -18,25 +18,7 @@ package org.apache.solr.search; import java.util.*; -import org.apache.commons.io.FilenameUtils; -import org.apache.lucene.queries.function.FunctionQuery; -import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.queries.function.valuesource.QueryValueSource; -import org.apache.lucene.search.Query; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.MapSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.transform.DocTransformer; -import org.apache.solr.response.transform.DocTransformers; -import org.apache.solr.response.transform.RenameFieldTransformer; -import org.apache.solr.response.transform.ScoreAugmenter; -import org.apache.solr.response.transform.TransformerFactory; -import org.apache.solr.response.transform.ValueSourceAugmenter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * A class representing the return fields @@ -44,376 +26,24 @@ import org.slf4j.LoggerFactory; * * @since solr 4.0 */ -public class ReturnFields -{ - static final Logger log = LoggerFactory.getLogger( ReturnFields.class ); +public abstract class ReturnFields { + /** + * Set of field names with their exact names from the lucene index. + *

+ * Class such as ResponseWriters pass this to {@link SolrIndexSearcher#doc(int, Set)}. + * @return Set of field names or null (all fields). + */ + public abstract Set getLuceneFieldNames(); - // Special Field Keys - public static final String SCORE = "score"; + /** Returns true if the specified field should be returned. */ + public abstract boolean wantsField(String name); - private final List globs = new ArrayList(1); - - // The lucene field names to request from the SolrIndexSearcher - // Order is important for CSVResponseWriter - private final Set fields = new LinkedHashSet(); - - // Field names that are OK to include in the response. - // This will include pseudo fields, lucene fields, and matching globs - private Set okFieldNames = new HashSet(); + /** Returns true if all fields should be returned. */ + public abstract boolean wantsAllFields(); - // The list of explicitly requested fields - private Set reqFieldNames = null; - - private DocTransformer transformer; - private boolean _wantsScore = false; - private boolean _wantsAllFields = false; + /** Returns true if the score should be returned. */ + public abstract boolean wantsScore(); - public ReturnFields() { - _wantsAllFields = true; - } - - public ReturnFields(SolrQueryRequest req) { - this( req.getParams().getParams(CommonParams.FL), req ); - } - - public ReturnFields(String fl, SolrQueryRequest req) { -// this( (fl==null)?null:SolrPluginUtils.split(fl), req ); - if( fl == null ) { - parseFieldList((String[])null, req); - } - else { - if( fl.trim().length() == 0 ) { - // legacy thing to support fl=' ' => fl=*,score! - // maybe time to drop support for this? - // See ConvertedLegacyTest - _wantsScore = true; - _wantsAllFields = true; - transformer = new ScoreAugmenter(SCORE); - } - else { - parseFieldList( new String[]{fl}, req); - } - } - } - - public ReturnFields(String[] fl, SolrQueryRequest req) { - parseFieldList(fl, req); - } - - private void parseFieldList(String[] fl, SolrQueryRequest req) { - _wantsScore = false; - _wantsAllFields = false; - if (fl == null || fl.length == 0 || fl.length == 1 && fl[0].length()==0) { - _wantsAllFields = true; - return; - } - - NamedList rename = new NamedList(); - DocTransformers augmenters = new DocTransformers(); - for (String fieldList : fl) { - add(fieldList,rename,augmenters,req); - } - for( int i=0; i(); - } - reqFieldNames.add(to); // don't rename our current target - } - } - } - augmenters.addTransformer( new RenameFieldTransformer( from, to, copy ) ); - } - - if( !_wantsAllFields ) { - if( !globs.isEmpty() ) { - // TODO??? need to fill up the fields with matching field names in the index - // and add them to okFieldNames? - // maybe just get all fields? - // this would disable field selection optimization... i think thatis OK - fields.clear(); // this will get all fields, and use wantsField to limit - } - okFieldNames.addAll( fields ); - } - - if( augmenters.size() == 1 ) { - transformer = augmenters.getTransformer(0); - } - else if( augmenters.size() > 1 ) { - transformer = augmenters; - } - } - - - // like getId, but also accepts dashes for legacy fields - String getFieldName(QueryParsing.StrParser sp) { - sp.eatws(); - int id_start = sp.pos; - char ch; - if (sp.pos < sp.end && (ch = sp.val.charAt(sp.pos)) != '$' && Character.isJavaIdentifierStart(ch)) { - sp.pos++; - while (sp.pos < sp.end) { - ch = sp.val.charAt(sp.pos); - if (!Character.isJavaIdentifierPart(ch) && ch != '.' && ch != '-') { - break; - } - sp.pos++; - } - return sp.val.substring(id_start, sp.pos); - } - - return null; - } - - private void add(String fl, NamedList rename, DocTransformers augmenters, SolrQueryRequest req) { - if( fl == null ) { - return; - } - try { - QueryParsing.StrParser sp = new QueryParsing.StrParser(fl); - - for(;;) { - sp.opt(','); - sp.eatws(); - if (sp.pos >= sp.end) break; - - int start = sp.pos; - - // short circuit test for a really simple field name - String key = null; - String field = getFieldName(sp); - char ch = sp.ch(); - - if (field != null) { - if (sp.opt(':')) { - // this was a key, not a field name - key = field; - field = null; - sp.eatws(); - start = sp.pos; - } else { - if (Character.isWhitespace(ch) || ch == ',' || ch==0) { - addField( field, key, augmenters, req ); - continue; - } - // an invalid field name... reset the position pointer to retry - sp.pos = start; - field = null; - } - } - - if (key != null) { - // we read "key : " - field = sp.getId(null); - ch = sp.ch(); - if (field != null && (Character.isWhitespace(ch) || ch == ',' || ch==0)) { - rename.add(field, key); - addField( field, key, augmenters, req ); - continue; - } - // an invalid field name... reset the position pointer to retry - sp.pos = start; - field = null; - } - - if (field == null) { - // We didn't find a simple name, so let's see if it's a globbed field name. - // Globbing only works with field names of the recommended form (roughly like java identifiers) - - field = sp.getGlobbedId(null); - ch = sp.ch(); - if (field != null && (Character.isWhitespace(ch) || ch == ',' || ch==0)) { - // "*" looks and acts like a glob, but we give it special treatment - if ("*".equals(field)) { - _wantsAllFields = true; - } else { - globs.add(field); - } - continue; - } - - // an invalid glob - sp.pos = start; - } - - String funcStr = sp.val.substring(start); - - // Is it an augmenter of the form [augmenter_name foo=1 bar=myfield]? - // This is identical to localParams syntax except it uses [] instead of {!} - - if (funcStr.startsWith("[")) { - Map augmenterArgs = new HashMap(); - int end = QueryParsing.parseLocalParams(funcStr, 0, augmenterArgs, req.getParams(), "[", ']'); - sp.pos += end; - - // [foo] is short for [type=foo] in localParams syntax - String augmenterName = augmenterArgs.remove("type"); - String disp = key; - if( disp == null ) { - disp = '['+augmenterName+']'; - } - - TransformerFactory factory = req.getCore().getTransformerFactory( augmenterName ); - if( factory != null ) { - MapSolrParams augmenterParams = new MapSolrParams( augmenterArgs ); - augmenters.addTransformer( factory.create(disp, augmenterParams, req) ); - } - else { - // unknown transformer? - } - addField(field, disp, augmenters, req); - continue; - } - - - // let's try it as a function instead - QParser parser = QParser.getParser(funcStr, FunctionQParserPlugin.NAME, req); - Query q = null; - ValueSource vs = null; - - try { - if (parser instanceof FunctionQParser) { - FunctionQParser fparser = (FunctionQParser)parser; - fparser.setParseMultipleSources(false); - fparser.setParseToEnd(false); - - q = fparser.getQuery(); - - if (fparser.localParams != null) { - if (fparser.valFollowedParams) { - // need to find the end of the function query via the string parser - int leftOver = fparser.sp.end - fparser.sp.pos; - sp.pos = sp.end - leftOver; // reset our parser to the same amount of leftover - } else { - // the value was via the "v" param in localParams, so we need to find - // the end of the local params themselves to pick up where we left off - sp.pos = start + fparser.localParamsEnd; - } - } else { - // need to find the end of the function query via the string parser - int leftOver = fparser.sp.end - fparser.sp.pos; - sp.pos = sp.end - leftOver; // reset our parser to the same amount of leftover - } - } else { - // A QParser that's not for function queries. - // It must have been specified via local params. - q = parser.getQuery(); - - assert parser.getLocalParams() != null; - sp.pos = start + parser.localParamsEnd; - } - - - if (q instanceof FunctionQuery) { - vs = ((FunctionQuery)q).getValueSource(); - } else { - vs = new QueryValueSource(q, 0.0f); - } - - if (key==null) { - SolrParams localParams = parser.getLocalParams(); - if (localParams != null) { - key = localParams.get("key"); - } - if (key == null) { - // use the function name itself as the field name - key = sp.val.substring(start, sp.pos); - } - } - - - if (key==null) { - key = funcStr; - } - okFieldNames.add( key ); - okFieldNames.add( funcStr ); - augmenters.addTransformer( new ValueSourceAugmenter( key, parser, vs ) ); - } - catch (SyntaxError e) { - // try again, simple rules for a field name with no whitespace - sp.pos = start; - field = sp.getSimpleString(); - - if (req.getSchema().getFieldOrNull(field) != null) { - // OK, it was an oddly named field - fields.add(field); - if( key != null ) { - rename.add(field, key); - } - } else { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing fieldname: " + e.getMessage(), e); - } - } - - // end try as function - - } // end for(;;) - } catch (SyntaxError e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing fieldname", e); - } - } - - private void addField( String field, String key, DocTransformers augmenters, SolrQueryRequest req ) - { - if(key==null) { - if(reqFieldNames==null) { - reqFieldNames = new HashSet(); - } - reqFieldNames.add(field); - } - - fields.add(field); // need to put in the map to maintain order for things like CSVResponseWriter - okFieldNames.add( field ); - okFieldNames.add( key ); - // a valid field name - if(SCORE.equals(field)) { - _wantsScore = true; - - String disp = (key==null) ? field : key; - augmenters.addTransformer( new ScoreAugmenter( disp ) ); - } - } - - public Set getLuceneFieldNames() - { - return (_wantsAllFields || fields.isEmpty()) ? null : fields; - } - - public boolean wantsAllFields() - { - return _wantsAllFields; - } - - public boolean wantsScore() - { - return _wantsScore; - } - - public boolean wantsField( String name ) - { - if( _wantsAllFields || okFieldNames.contains( name ) ){ - return true; - } - for( String s : globs ) { - // TODO something better? - if( FilenameUtils.wildcardMatch( name, s ) ) { - okFieldNames.add(name); // Don't calculate it again - return true; - } - } - return false; - } - - public DocTransformer getTransformer() - { - return transformer; - } + /** Returns the DocTransformer used to modify documents, or null */ + public abstract DocTransformer getTransformer(); } diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 62d0e9ea6e1..ad8eff7c070 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -565,8 +565,8 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn /** * Retrieve the {@link Document} instance corresponding to the document id. - * - * Note: The document will have all fields accessable, but if a field + *

+ * Note: The document will have all fields accessible, but if a field * filter is provided, only the provided fields will be loaded (the * remainder will be available lazily). */ diff --git a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java new file mode 100644 index 00000000000..6e75d511621 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import org.apache.commons.io.FilenameUtils; +import org.apache.lucene.queries.function.FunctionQuery; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.QueryValueSource; +import org.apache.lucene.search.Query; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.MapSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.transform.DocTransformer; +import org.apache.solr.response.transform.DocTransformers; +import org.apache.solr.response.transform.RenameFieldTransformer; +import org.apache.solr.response.transform.ScoreAugmenter; +import org.apache.solr.response.transform.TransformerFactory; +import org.apache.solr.response.transform.ValueSourceAugmenter; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * The default implementation of return fields parsing for Solr. + */ +public class SolrReturnFields extends ReturnFields { + // Special Field Keys + public static final String SCORE = "score"; + + private final List globs = new ArrayList(1); + + // The lucene field names to request from the SolrIndexSearcher + // Order is important for CSVResponseWriter + private final Set fields = new LinkedHashSet(); + + // Field names that are OK to include in the response. + // This will include pseudo fields, lucene fields, and matching globs + private Set okFieldNames = new HashSet(); + + // The list of explicitly requested fields + private Set reqFieldNames = null; + + protected DocTransformer transformer; + protected boolean _wantsScore = false; + protected boolean _wantsAllFields = false; + + public SolrReturnFields() { + _wantsAllFields = true; + } + + public SolrReturnFields(SolrQueryRequest req) { + this( req.getParams().getParams(CommonParams.FL), req ); + } + + public SolrReturnFields(String fl, SolrQueryRequest req) { +// this( (fl==null)?null:SolrPluginUtils.split(fl), req ); + if( fl == null ) { + parseFieldList((String[])null, req); + } + else { + if( fl.trim().length() == 0 ) { + // legacy thing to support fl=' ' => fl=*,score! + // maybe time to drop support for this? + // See ConvertedLegacyTest + _wantsScore = true; + _wantsAllFields = true; + transformer = new ScoreAugmenter(SCORE); + } + else { + parseFieldList( new String[]{fl}, req); + } + } + } + + public SolrReturnFields(String[] fl, SolrQueryRequest req) { + parseFieldList(fl, req); + } + + private void parseFieldList(String[] fl, SolrQueryRequest req) { + _wantsScore = false; + _wantsAllFields = false; + if (fl == null || fl.length == 0 || fl.length == 1 && fl[0].length()==0) { + _wantsAllFields = true; + return; + } + + NamedList rename = new NamedList(); + DocTransformers augmenters = new DocTransformers(); + for (String fieldList : fl) { + add(fieldList,rename,augmenters,req); + } + for( int i=0; i(); + } + reqFieldNames.add(to); // don't rename our current target + } + } + } + augmenters.addTransformer( new RenameFieldTransformer( from, to, copy ) ); + } + + if( !_wantsAllFields ) { + if( !globs.isEmpty() ) { + // TODO??? need to fill up the fields with matching field names in the index + // and add them to okFieldNames? + // maybe just get all fields? + // this would disable field selection optimization... i think thatis OK + fields.clear(); // this will get all fields, and use wantsField to limit + } + okFieldNames.addAll( fields ); + } + + if( augmenters.size() == 1 ) { + transformer = augmenters.getTransformer(0); + } + else if( augmenters.size() > 1 ) { + transformer = augmenters; + } + } + + // like getId, but also accepts dashes for legacy fields + String getFieldName(QueryParsing.StrParser sp) { + sp.eatws(); + int id_start = sp.pos; + char ch; + if (sp.pos < sp.end && (ch = sp.val.charAt(sp.pos)) != '$' && Character.isJavaIdentifierStart(ch)) { + sp.pos++; + while (sp.pos < sp.end) { + ch = sp.val.charAt(sp.pos); + if (!Character.isJavaIdentifierPart(ch) && ch != '.' && ch != '-') { + break; + } + sp.pos++; + } + return sp.val.substring(id_start, sp.pos); + } + + return null; + } + + private void add(String fl, NamedList rename, DocTransformers augmenters, SolrQueryRequest req) { + if( fl == null ) { + return; + } + try { + QueryParsing.StrParser sp = new QueryParsing.StrParser(fl); + + for(;;) { + sp.opt(','); + sp.eatws(); + if (sp.pos >= sp.end) break; + + int start = sp.pos; + + // short circuit test for a really simple field name + String key = null; + String field = getFieldName(sp); + char ch = sp.ch(); + + if (field != null) { + if (sp.opt(':')) { + // this was a key, not a field name + key = field; + field = null; + sp.eatws(); + start = sp.pos; + } else { + if (Character.isWhitespace(ch) || ch == ',' || ch==0) { + addField( field, key, augmenters, req ); + continue; + } + // an invalid field name... reset the position pointer to retry + sp.pos = start; + field = null; + } + } + + if (key != null) { + // we read "key : " + field = sp.getId(null); + ch = sp.ch(); + if (field != null && (Character.isWhitespace(ch) || ch == ',' || ch==0)) { + rename.add(field, key); + addField( field, key, augmenters, req ); + continue; + } + // an invalid field name... reset the position pointer to retry + sp.pos = start; + field = null; + } + + if (field == null) { + // We didn't find a simple name, so let's see if it's a globbed field name. + // Globbing only works with field names of the recommended form (roughly like java identifiers) + + field = sp.getGlobbedId(null); + ch = sp.ch(); + if (field != null && (Character.isWhitespace(ch) || ch == ',' || ch==0)) { + // "*" looks and acts like a glob, but we give it special treatment + if ("*".equals(field)) { + _wantsAllFields = true; + } else { + globs.add(field); + } + continue; + } + + // an invalid glob + sp.pos = start; + } + + String funcStr = sp.val.substring(start); + + // Is it an augmenter of the form [augmenter_name foo=1 bar=myfield]? + // This is identical to localParams syntax except it uses [] instead of {!} + + if (funcStr.startsWith("[")) { + Map augmenterArgs = new HashMap(); + int end = QueryParsing.parseLocalParams(funcStr, 0, augmenterArgs, req.getParams(), "[", ']'); + sp.pos += end; + + // [foo] is short for [type=foo] in localParams syntax + String augmenterName = augmenterArgs.remove("type"); + String disp = key; + if( disp == null ) { + disp = '['+augmenterName+']'; + } + + TransformerFactory factory = req.getCore().getTransformerFactory( augmenterName ); + if( factory != null ) { + MapSolrParams augmenterParams = new MapSolrParams( augmenterArgs ); + augmenters.addTransformer( factory.create(disp, augmenterParams, req) ); + } + else { + // unknown transformer? + } + addField(field, disp, augmenters, req); + continue; + } + + + // let's try it as a function instead + QParser parser = QParser.getParser(funcStr, FunctionQParserPlugin.NAME, req); + Query q = null; + ValueSource vs = null; + + try { + if (parser instanceof FunctionQParser) { + FunctionQParser fparser = (FunctionQParser)parser; + fparser.setParseMultipleSources(false); + fparser.setParseToEnd(false); + + q = fparser.getQuery(); + + if (fparser.localParams != null) { + if (fparser.valFollowedParams) { + // need to find the end of the function query via the string parser + int leftOver = fparser.sp.end - fparser.sp.pos; + sp.pos = sp.end - leftOver; // reset our parser to the same amount of leftover + } else { + // the value was via the "v" param in localParams, so we need to find + // the end of the local params themselves to pick up where we left off + sp.pos = start + fparser.localParamsEnd; + } + } else { + // need to find the end of the function query via the string parser + int leftOver = fparser.sp.end - fparser.sp.pos; + sp.pos = sp.end - leftOver; // reset our parser to the same amount of leftover + } + } else { + // A QParser that's not for function queries. + // It must have been specified via local params. + q = parser.getQuery(); + + assert parser.getLocalParams() != null; + sp.pos = start + parser.localParamsEnd; + } + + + if (q instanceof FunctionQuery) { + vs = ((FunctionQuery)q).getValueSource(); + } else { + vs = new QueryValueSource(q, 0.0f); + } + + if (key==null) { + SolrParams localParams = parser.getLocalParams(); + if (localParams != null) { + key = localParams.get("key"); + } + if (key == null) { + // use the function name itself as the field name + key = sp.val.substring(start, sp.pos); + } + } + + + if (key==null) { + key = funcStr; + } + okFieldNames.add( key ); + okFieldNames.add( funcStr ); + augmenters.addTransformer( new ValueSourceAugmenter( key, parser, vs ) ); + } + catch (SyntaxError e) { + // try again, simple rules for a field name with no whitespace + sp.pos = start; + field = sp.getSimpleString(); + + if (req.getSchema().getFieldOrNull(field) != null) { + // OK, it was an oddly named field + fields.add(field); + if( key != null ) { + rename.add(field, key); + } + } else { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing fieldname: " + e.getMessage(), e); + } + } + + // end try as function + + } // end for(;;) + } catch (SyntaxError e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing fieldname", e); + } + } + + private void addField(String field, String key, DocTransformers augmenters, SolrQueryRequest req) + { + if(key==null) { + if(reqFieldNames==null) { + reqFieldNames = new HashSet(); + } + reqFieldNames.add(field); + } + + fields.add(field); // need to put in the map to maintain order for things like CSVResponseWriter + okFieldNames.add( field ); + okFieldNames.add( key ); + // a valid field name + if(SCORE.equals(field)) { + _wantsScore = true; + + String disp = (key==null) ? field : key; + augmenters.addTransformer( new ScoreAugmenter( disp ) ); + } + } + + @Override + public Set getLuceneFieldNames() + { + return (_wantsAllFields || fields.isEmpty()) ? null : fields; + } + + @Override + public boolean wantsField(String name) + { + if( _wantsAllFields || okFieldNames.contains( name ) ){ + return true; + } + for( String s : globs ) { + // TODO something better? + if( FilenameUtils.wildcardMatch(name, s) ) { + okFieldNames.add(name); // Don't calculate it again + return true; + } + } + return false; + } + + @Override + public boolean wantsAllFields() + { + return _wantsAllFields; + } + + @Override + public boolean wantsScore() + { + return _wantsScore; + } + + @Override + public DocTransformer getTransformer() + { + return transformer; + } +} diff --git a/solr/core/src/test/org/apache/solr/request/JSONWriterTest.java b/solr/core/src/test/org/apache/solr/request/JSONWriterTest.java index 5c2600ec201..84350beb237 100644 --- a/solr/core/src/test/org/apache/solr/request/JSONWriterTest.java +++ b/solr/core/src/test/org/apache/solr/request/JSONWriterTest.java @@ -34,6 +34,7 @@ import org.apache.solr.response.PythonResponseWriter; import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.RubyResponseWriter; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrReturnFields; import org.junit.BeforeClass; import org.junit.Test; @@ -105,7 +106,7 @@ public class JSONWriterTest extends SolrTestCaseJ4 { SolrQueryResponse rsp = new SolrQueryResponse(); JSONResponseWriter w = new JSONResponseWriter(); - ReturnFields returnFields = new ReturnFields(req); + ReturnFields returnFields = new SolrReturnFields(req); rsp.setReturnFields(returnFields); StringWriter buf = new StringWriter(); diff --git a/solr/core/src/test/org/apache/solr/request/TestBinaryResponseWriter.java b/solr/core/src/test/org/apache/solr/request/TestBinaryResponseWriter.java index 7abb1e94a48..559e166a990 100644 --- a/solr/core/src/test/org/apache/solr/request/TestBinaryResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/request/TestBinaryResponseWriter.java @@ -30,6 +30,7 @@ import org.apache.solr.response.BinaryQueryResponseWriter; import org.apache.solr.response.BinaryResponseWriter.Resolver; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.search.ReturnFields; +import org.apache.solr.search.SolrReturnFields; import org.apache.solr.util.AbstractSolrTestCase; import org.junit.BeforeClass; @@ -81,7 +82,7 @@ public class TestBinaryResponseWriter extends AbstractSolrTestCase { in.addField("ddd_s", "ddd"); in.addField("eee_s", "eee"); - Resolver r = new Resolver(req, new ReturnFields(req)); + Resolver r = new Resolver(req, new SolrReturnFields(req)); Object o = r.resolve(in, new JavaBinCodec()); assertNotNull("obj is null", o); diff --git a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java index 0db44b24aa9..532d1508f09 100644 --- a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java @@ -23,6 +23,7 @@ import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.util.DateUtil; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.ReturnFields; +import org.apache.solr.search.SolrReturnFields; import org.junit.*; import java.io.StringWriter; @@ -145,19 +146,19 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 { rsp.add("response", sdl); QueryResponseWriter w = new CSVResponseWriter(); - rsp.setReturnFields( new ReturnFields("id,foo_s", req) ); + rsp.setReturnFields( new SolrReturnFields("id,foo_s", req) ); StringWriter buf = new StringWriter(); w.write(buf, req, rsp); assertEquals("id,foo_s\n1,hi\n2,\n", buf.toString()); // try scores - rsp.setReturnFields( new ReturnFields("id,score,foo_s", req) ); + rsp.setReturnFields( new SolrReturnFields("id,score,foo_s", req) ); buf = new StringWriter(); w.write(buf, req, rsp); assertEquals("id,score,foo_s\n1,2.718,hi\n2,89.83,\n", buf.toString()); // get field values from docs... should be ordered and not include score unless requested - rsp.setReturnFields( new ReturnFields("*", req) ); + rsp.setReturnFields( new SolrReturnFields("*", req) ); buf = new StringWriter(); w.write(buf, req, rsp); assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss\n" + @@ -167,14 +168,14 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 { // get field values and scores - just check that the scores are there... we don't guarantee where - rsp.setReturnFields( new ReturnFields("*,score", req) ); + rsp.setReturnFields( new SolrReturnFields("*,score", req) ); buf = new StringWriter(); w.write(buf, req, rsp); String s = buf.toString(); assertTrue(s.indexOf("score") >=0 && s.indexOf("2.718") > 0 && s.indexOf("89.83") > 0 ); // Test field globs - rsp.setReturnFields( new ReturnFields("id,foo*", req) ); + rsp.setReturnFields( new SolrReturnFields("id,foo*", req) ); buf = new StringWriter(); w.write(buf, req, rsp); assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt\n" + @@ -182,7 +183,7 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 { "2,,,,,,,\n", buf.toString()); - rsp.setReturnFields( new ReturnFields("id,*_d*", req) ); + rsp.setReturnFields( new SolrReturnFields("id,*_d*", req) ); buf = new StringWriter(); w.write(buf, req, rsp); assertEquals("id,foo_d,foo_dt\n" + diff --git a/solr/core/src/test/org/apache/solr/search/ReturnFieldsTest.java b/solr/core/src/test/org/apache/solr/search/ReturnFieldsTest.java index 0ff673f836c..88f5a0e254e 100644 --- a/solr/core/src/test/org/apache/solr/search/ReturnFieldsTest.java +++ b/solr/core/src/test/org/apache/solr/search/ReturnFieldsTest.java @@ -97,7 +97,7 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { @Test public void testSeparators() { - ReturnFields rf = new ReturnFields( req("fl", "id name test subject score") ); + ReturnFields rf = new SolrReturnFields( req("fl", "id name test subject score") ); assertTrue( rf.wantsScore() ); assertTrue( rf.wantsField( "id" ) ); assertTrue( rf.wantsField( "name" ) ); @@ -108,7 +108,7 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { assertFalse( rf.wantsField( "xxx" ) ); assertTrue( rf.getTransformer() instanceof ScoreAugmenter); - rf = new ReturnFields( req("fl", "id,name,test,subject,score") ); + rf = new SolrReturnFields( req("fl", "id,name,test,subject,score") ); assertTrue( rf.wantsScore() ); assertTrue( rf.wantsField( "id" ) ); assertTrue( rf.wantsField( "name" ) ); @@ -119,7 +119,7 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { assertFalse( rf.wantsField( "xxx" ) ); assertTrue( rf.getTransformer() instanceof ScoreAugmenter); - rf = new ReturnFields( req("fl", "id,name test,subject score") ); + rf = new SolrReturnFields( req("fl", "id,name test,subject score") ); assertTrue( rf.wantsScore() ); assertTrue( rf.wantsField( "id" ) ); assertTrue( rf.wantsField( "name" ) ); @@ -130,7 +130,7 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { assertFalse( rf.wantsField( "xxx" ) ); assertTrue( rf.getTransformer() instanceof ScoreAugmenter); - rf = new ReturnFields( req("fl", "id, name test , subject,score") ); + rf = new SolrReturnFields( req("fl", "id, name test , subject,score") ); assertTrue( rf.wantsScore() ); assertTrue( rf.wantsField( "id" ) ); assertTrue( rf.wantsField( "name" ) ); @@ -144,20 +144,20 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { @Test public void testWilcards() { - ReturnFields rf = new ReturnFields( req("fl", "*") ); + ReturnFields rf = new SolrReturnFields( req("fl", "*") ); assertFalse( rf.wantsScore() ); assertTrue( rf.wantsField( "xxx" ) ); assertTrue( rf.wantsAllFields() ); assertNull( rf.getTransformer() ); - rf = new ReturnFields( req("fl", " * ") ); + rf = new SolrReturnFields( req("fl", " * ") ); assertFalse( rf.wantsScore() ); assertTrue( rf.wantsField( "xxx" ) ); assertTrue( rf.wantsAllFields() ); assertNull( rf.getTransformer() ); // Check that we want wildcards - rf = new ReturnFields( req("fl", "id,aaa*,*bbb") ); + rf = new SolrReturnFields( req("fl", "id,aaa*,*bbb") ); assertTrue( rf.wantsField( "id" ) ); assertTrue( rf.wantsField( "aaaxxx" ) ); assertFalse(rf.wantsField("xxxaaa")); @@ -169,7 +169,7 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { @Test public void testManyParameters() { - ReturnFields rf = new ReturnFields( req("fl", "id name", "fl", "test subject", "fl", "score") ); + ReturnFields rf = new SolrReturnFields( req("fl", "id name", "fl", "test subject", "fl", "score") ); assertTrue( rf.wantsScore() ); assertTrue( rf.wantsField( "id" ) ); assertTrue( rf.wantsField( "name" ) ); @@ -183,7 +183,7 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { @Test public void testFunctions() { - ReturnFields rf = new ReturnFields( req("fl", "id sum(1,1)") ); + ReturnFields rf = new SolrReturnFields( req("fl", "id sum(1,1)") ); assertFalse(rf.wantsScore()); assertTrue( rf.wantsField( "id" ) ); assertFalse( rf.wantsAllFields() ); @@ -194,41 +194,41 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { @Test public void testTransformers() { - ReturnFields rf = new ReturnFields( req("fl", "[explain]") ); + ReturnFields rf = new SolrReturnFields( req("fl", "[explain]") ); assertFalse( rf.wantsScore() ); assertFalse(rf.wantsField("id")); assertFalse(rf.wantsAllFields()); assertEquals( "[explain]", rf.getTransformer().getName() ); - rf = new ReturnFields( req("fl", "[shard],id") ); + rf = new SolrReturnFields( req("fl", "[shard],id") ); assertFalse( rf.wantsScore() ); assertTrue(rf.wantsField("id")); assertFalse(rf.wantsField("xxx")); assertFalse(rf.wantsAllFields()); assertEquals( "[shard]", rf.getTransformer().getName() ); - rf = new ReturnFields( req("fl", "[docid]") ); + rf = new SolrReturnFields( req("fl", "[docid]") ); assertFalse( rf.wantsScore() ); assertFalse( rf.wantsField( "id" ) ); assertFalse(rf.wantsField("xxx")); assertFalse(rf.wantsAllFields()); assertEquals( "[docid]", rf.getTransformer().getName() ); - rf = new ReturnFields( req("fl", "mydocid:[docid]") ); + rf = new SolrReturnFields( req("fl", "mydocid:[docid]") ); assertFalse( rf.wantsScore() ); assertFalse( rf.wantsField( "id" ) ); assertFalse(rf.wantsField("xxx")); assertFalse(rf.wantsAllFields()); assertEquals( "mydocid", rf.getTransformer().getName() ); - rf = new ReturnFields( req("fl", "[docid][shard]") ); + rf = new SolrReturnFields( req("fl", "[docid][shard]") ); assertFalse( rf.wantsScore() ); assertFalse(rf.wantsField("xxx")); assertFalse(rf.wantsAllFields()); assertTrue( rf.getTransformer() instanceof DocTransformers); assertEquals(2, ((DocTransformers)rf.getTransformer()).size()); - rf = new ReturnFields( req("fl", "[xxxxx]") ); + rf = new SolrReturnFields( req("fl", "[xxxxx]") ); assertFalse( rf.wantsScore() ); assertFalse( rf.wantsField( "id" ) ); assertFalse(rf.wantsAllFields()); @@ -237,7 +237,7 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { @Test public void testAliases() { - ReturnFields rf = new ReturnFields( req("fl", "newId:id newName:name newTest:test newSubject:subject") ); + ReturnFields rf = new SolrReturnFields( req("fl", "newId:id newName:name newTest:test newSubject:subject") ); assertTrue(rf.wantsField("id")); assertTrue(rf.wantsField("name")); assertTrue(rf.wantsField("test")); @@ -249,7 +249,7 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { assertFalse(rf.wantsField("xxx")); assertFalse(rf.wantsAllFields()); - rf = new ReturnFields( req("fl", "newId:id newName:name newTest:test newSubject:subject score") ); + rf = new SolrReturnFields( req("fl", "newId:id newName:name newTest:test newSubject:subject score") ); assertTrue(rf.wantsField("id")); assertTrue(rf.wantsField("name")); assertTrue(rf.wantsField("test")); @@ -268,7 +268,7 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { // the simplest case of fl=foo-bar to work @Test public void testHyphenInFieldName() { - ReturnFields rf = new ReturnFields(req("fl", "id-test")); + ReturnFields rf = new SolrReturnFields(req("fl", "id-test")); assertFalse(rf.wantsScore()); assertTrue(rf.wantsField("id-test")); assertFalse(rf.wantsField("xxx")); @@ -277,20 +277,20 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { @Test public void testTrailingDotInFieldName() { - ReturnFields rf = new ReturnFields(req("fl", "id.test")); + ReturnFields rf = new SolrReturnFields(req("fl", "id.test")); assertFalse(rf.wantsScore()); assertTrue(rf.wantsField("id.test")); assertFalse(rf.wantsField("xxx")); assertFalse(rf.wantsAllFields()); - rf = new ReturnFields(req("fl", "test:id.test")); + rf = new SolrReturnFields(req("fl", "test:id.test")); assertFalse(rf.wantsScore()); assertTrue(rf.wantsField("id.test")); assertTrue(rf.wantsField("test")); assertFalse(rf.wantsField("xxx")); assertFalse(rf.wantsAllFields()); - rf = new ReturnFields(req("fl", "test.id:id.test")); + rf = new SolrReturnFields(req("fl", "test.id:id.test")); assertFalse(rf.wantsScore()); assertTrue(rf.wantsField("id.test")); assertTrue(rf.wantsField("test.id")); @@ -300,7 +300,7 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { @Test public void testTrailingDollarInFieldName() { - ReturnFields rf = new ReturnFields(req("fl", "id$test")); + ReturnFields rf = new SolrReturnFields(req("fl", "id$test")); assertFalse(rf.wantsScore()); assertTrue(rf.wantsField("id$test")); assertFalse(rf.wantsField("xxx")); @@ -325,7 +325,7 @@ public class ReturnFieldsTest extends SolrTestCaseJ4 { _TestUtil.randomWhitespace(r, 0, 3); final String fl = id + (r.nextBoolean() ? "" : ",") + foo_i; - ReturnFields rf = new ReturnFields(req("fl", fl)); + ReturnFields rf = new SolrReturnFields(req("fl", fl)); assertFalse("score ("+fl+")", rf.wantsScore());