SOLR-7968: Make QueryComponent extensible

This commit is contained in:
David Smiley 2016-01-31 22:25:37 -05:00
parent 14a2c16ca4
commit 4cdce3db77
5 changed files with 211 additions and 188 deletions

View File

@ -583,6 +583,8 @@ Other Changes
* SOLR-8597: add default, no-op QParserPlugin.init(NamedList) method (Christine Poerschke)
* SOLR-7968: Make QueryComponent more extensible. (Markus Jelsma via David Smiley)
================== 5.4.1 ==================
Bug Fixes

View File

@ -235,7 +235,7 @@ public class QueryComponent extends SearchComponent
}
}
private void prepareGrouping(ResponseBuilder rb) throws IOException {
protected void prepareGrouping(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrParams params = req.getParams();
@ -671,7 +671,7 @@ public class QueryComponent extends SearchComponent
}
}
private int groupedDistributedProcess(ResponseBuilder rb) {
protected int groupedDistributedProcess(ResponseBuilder rb) {
int nextStage = ResponseBuilder.STAGE_DONE;
ShardRequestFactory shardRequestFactory = null;
@ -705,7 +705,7 @@ public class QueryComponent extends SearchComponent
return nextStage;
}
private int regularDistributedProcess(ResponseBuilder rb) {
protected int regularDistributedProcess(ResponseBuilder rb) {
if (rb.stage < ResponseBuilder.STAGE_PARSE_QUERY)
return ResponseBuilder.STAGE_PARSE_QUERY;
if (rb.stage == ResponseBuilder.STAGE_PARSE_QUERY) {
@ -734,7 +734,7 @@ public class QueryComponent extends SearchComponent
}
}
private void handleGroupedResponses(ResponseBuilder rb, ShardRequest sreq) {
protected void handleGroupedResponses(ResponseBuilder rb, ShardRequest sreq) {
ShardResponseProcessor responseProcessor = null;
if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_GROUPS) != 0) {
responseProcessor = new SearchGroupShardResponseProcessor();
@ -749,7 +749,7 @@ public class QueryComponent extends SearchComponent
}
}
private void handleRegularResponses(ResponseBuilder rb, ShardRequest sreq) {
protected void handleRegularResponses(ResponseBuilder rb, ShardRequest sreq) {
if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
mergeIds(rb, sreq);
}
@ -775,11 +775,11 @@ public class QueryComponent extends SearchComponent
}
}
private static final EndResultTransformer MAIN_END_RESULT_TRANSFORMER = new MainEndResultTransformer();
private static final EndResultTransformer SIMPLE_END_RESULT_TRANSFORMER = new SimpleEndResultTransformer();
protected static final EndResultTransformer MAIN_END_RESULT_TRANSFORMER = new MainEndResultTransformer();
protected static final EndResultTransformer SIMPLE_END_RESULT_TRANSFORMER = new SimpleEndResultTransformer();
@SuppressWarnings("unchecked")
private void groupedFinishStage(final ResponseBuilder rb) {
protected void groupedFinishStage(final ResponseBuilder rb) {
// To have same response as non-distributed request.
GroupingSpecification groupSpec = rb.getGroupingSpec();
if (rb.mergedTopGroups.isEmpty()) {
@ -814,24 +814,24 @@ public class QueryComponent extends SearchComponent
endResultTransformer.transform(combinedMap, rb, solrDocumentSource);
}
private void regularFinishStage(ResponseBuilder rb) {
protected void regularFinishStage(ResponseBuilder rb) {
// We may not have been able to retrieve all the docs due to an
// index change. Remove any null documents.
for (Iterator<SolrDocument> iter = rb._responseDocs.iterator(); iter.hasNext();) {
for (Iterator<SolrDocument> iter = rb.getResponseDocs().iterator(); iter.hasNext();) {
if (iter.next() == null) {
iter.remove();
rb._responseDocs.setNumFound(rb._responseDocs.getNumFound()-1);
rb.getResponseDocs().setNumFound(rb.getResponseDocs().getNumFound()-1);
}
}
rb.rsp.addResponse(rb._responseDocs);
rb.rsp.addResponse(rb.getResponseDocs());
if (null != rb.getNextCursorMark()) {
rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
rb.getNextCursorMark().getSerializedTotem());
}
}
private void createDistributedStats(ResponseBuilder rb) {
protected void createDistributedStats(ResponseBuilder rb) {
StatsCache cache = rb.req.getCore().getStatsCache();
if ( (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES)!=0 || rb.getSortSpec().includesScore()) {
ShardRequest sreq = cache.retrieveStatsRequest(rb);
@ -841,12 +841,12 @@ public class QueryComponent extends SearchComponent
}
}
private void updateStats(ResponseBuilder rb, ShardRequest sreq) {
protected void updateStats(ResponseBuilder rb, ShardRequest sreq) {
StatsCache cache = rb.req.getCore().getStatsCache();
cache.mergeToGlobalStats(rb.req, sreq.responses);
}
private void createMainQuery(ResponseBuilder rb) {
protected void createMainQuery(ResponseBuilder rb) {
ShardRequest sreq = new ShardRequest();
sreq.purpose = ShardRequest.PURPOSE_GET_TOP_IDS;
@ -931,13 +931,13 @@ public class QueryComponent extends SearchComponent
rb.addRequest(this, sreq);
}
private boolean addFL(StringBuilder fl, String field, boolean additionalAdded) {
protected boolean addFL(StringBuilder fl, String field, boolean additionalAdded) {
if (additionalAdded) fl.append(",");
fl.append(field);
return true;
}
private void mergeIds(ResponseBuilder rb, ShardRequest sreq) {
protected void mergeIds(ResponseBuilder rb, ShardRequest sreq) {
List<MergeStrategy> mergeStrategies = rb.getMergeStrategies();
if(mergeStrategies != null) {
Collections.sort(mergeStrategies, MergeStrategy.MERGE_COMP);
@ -1110,7 +1110,7 @@ public class QueryComponent extends SearchComponent
// again when retrieving stored fields.
// TODO: use ResponseBuilder (w/ comments) or the request context?
rb.resultIds = resultIds;
rb._responseDocs = responseDocs;
rb.setResponseDocs(responseDocs);
populateNextCursorMarkFromMergedShards(rb);
@ -1130,7 +1130,7 @@ public class QueryComponent extends SearchComponent
* <code>ShardDocs</code> in <code>resultIds</code>, may or may not be
* part of a Cursor based request (method will NOOP if not needed)
*/
private void populateNextCursorMarkFromMergedShards(ResponseBuilder rb) {
protected void populateNextCursorMarkFromMergedShards(ResponseBuilder rb) {
final CursorMark lastCursorMark = rb.getCursorMark();
if (null == lastCursorMark) {
@ -1172,7 +1172,7 @@ public class QueryComponent extends SearchComponent
rb.setNextCursorMark(nextCursorMark);
}
private NamedList unmarshalSortValues(SortSpec sortSpec,
protected NamedList unmarshalSortValues(SortSpec sortSpec,
NamedList sortFieldValues,
IndexSchema schema) {
NamedList unmarshalledSortValsPerField = new NamedList();
@ -1213,7 +1213,7 @@ public class QueryComponent extends SearchComponent
return unmarshalledSortValsPerField;
}
private void createRetrieveDocs(ResponseBuilder rb) {
protected void createRetrieveDocs(ResponseBuilder rb) {
// TODO: in a system with nTiers > 2, we could be passed "ids" here
// unless those requests always go to the final destination shard
@ -1267,7 +1267,7 @@ public class QueryComponent extends SearchComponent
}
private void returnFields(ResponseBuilder rb, ShardRequest sreq) {
protected void returnFields(ResponseBuilder rb, ShardRequest sreq) {
// Keep in mind that this could also be a shard in a multi-tiered system.
// TODO: if a multi-tiered system, it seems like some requests
// could/should bypass middlemen (like retrieving stored fields)
@ -1318,7 +1318,7 @@ public class QueryComponent extends SearchComponent
if (removeKeyField) {
doc.removeFields(keyFieldName);
}
rb._responseDocs.set(sdoc.positionInResponse, doc);
rb.getResponseDocs().set(sdoc.positionInResponse, doc);
}
}
}
@ -1344,7 +1344,7 @@ public class QueryComponent extends SearchComponent
*
* TODO: when SOLR-5595 is fixed, this wont be needed, as we dont need to recompute sort values here from the comparator
*/
private static class FakeScorer extends Scorer {
protected static class FakeScorer extends Scorer {
final int docid;
final float score;

View File

@ -263,6 +263,10 @@ public class ResponseBuilder
public void setResponseDocs(SolrDocumentList _responseDocs) {
this._responseDocs = _responseDocs;
}
public SolrDocumentList getResponseDocs() {
return this._responseDocs;
}
public boolean isDebugTrack() {
return debugTrack;

View File

@ -16,21 +16,9 @@
*/
package org.apache.solr.handler.component;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.PriorityQueue;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
public class ShardDoc extends FieldDoc {
public String shard;
public String shardAddress; // TODO
@ -44,7 +32,7 @@ public class ShardDoc extends FieldDoc {
// this is currently the uniqueKeyField but
// may be replaced with internal docid in a future release.
NamedList sortFieldValues;
public NamedList sortFieldValues;
// sort field values for *all* docs in a particular shard.
// this doc's values are in position orderInShard
@ -93,154 +81,4 @@ public class ShardDoc extends FieldDoc {
+" ,positionInResponse="+positionInResponse
+" ,sortFieldValues="+sortFieldValues;
}
}
// used by distributed search to merge results.
class ShardFieldSortedHitQueue extends PriorityQueue<ShardDoc> {
/** Stores a comparator corresponding to each field being sorted by */
protected Comparator<ShardDoc>[] comparators;
/** Stores the sort criteria being used. */
protected SortField[] fields;
/** The order of these fieldNames should correspond to the order of sort field values retrieved from the shard */
protected List<String> fieldNames = new ArrayList<>();
public ShardFieldSortedHitQueue(SortField[] fields, int size, IndexSearcher searcher) {
super(size);
final int n = fields.length;
//noinspection unchecked
comparators = new Comparator[n];
this.fields = new SortField[n];
for (int i = 0; i < n; ++i) {
// keep track of the named fields
SortField.Type type = fields[i].getType();
if (type!=SortField.Type.SCORE && type!=SortField.Type.DOC) {
fieldNames.add(fields[i].getField());
}
String fieldname = fields[i].getField();
comparators[i] = getCachedComparator(fields[i], searcher);
if (fields[i].getType() == SortField.Type.STRING) {
this.fields[i] = new SortField(fieldname, SortField.Type.STRING,
fields[i].getReverse());
} else {
this.fields[i] = new SortField(fieldname, fields[i].getType(),
fields[i].getReverse());
}
//System.out.println("%%%%%%%%%%%%%%%%%% got "+fields[i].getType() +" for "+ fieldname +" fields[i].getReverse(): "+fields[i].getReverse());
}
}
@Override
protected boolean lessThan(ShardDoc docA, ShardDoc docB) {
// If these docs are from the same shard, then the relative order
// is how they appeared in the response from that shard.
if (docA.shard == docB.shard) {
// if docA has a smaller position, it should be "larger" so it
// comes before docB.
// This will handle sorting by docid within the same shard
// comment this out to test comparators.
return !(docA.orderInShard < docB.orderInShard);
}
// run comparators
final int n = comparators.length;
int c = 0;
for (int i = 0; i < n && c == 0; i++) {
c = (fields[i].getReverse()) ? comparators[i].compare(docB, docA)
: comparators[i].compare(docA, docB);
}
// solve tiebreaks by comparing shards (similar to using docid)
// smaller docid's beat larger ids, so reverse the natural ordering
if (c == 0) {
c = -docA.shard.compareTo(docB.shard);
}
return c < 0;
}
Comparator<ShardDoc> getCachedComparator(SortField sortField, IndexSearcher searcher) {
SortField.Type type = sortField.getType();
if (type == SortField.Type.SCORE) {
return comparatorScore();
} else if (type == SortField.Type.REWRITEABLE) {
try {
sortField = sortField.rewrite(searcher);
} catch (IOException e) {
throw new SolrException(SERVER_ERROR, "Exception rewriting sort field " + sortField, e);
}
}
return comparatorFieldComparator(sortField);
}
abstract class ShardComparator implements Comparator<ShardDoc> {
final SortField sortField;
final String fieldName;
final int fieldNum;
public ShardComparator(SortField sortField) {
this.sortField = sortField;
this.fieldName = sortField.getField();
int fieldNum = 0;
for (int i=0; i<fieldNames.size(); i++) {
if (fieldNames.get(i).equals(fieldName)) {
fieldNum = i;
break;
}
}
this.fieldNum = fieldNum;
}
Object sortVal(ShardDoc shardDoc) {
assert(shardDoc.sortFieldValues.getName(fieldNum).equals(fieldName));
List lst = (List)shardDoc.sortFieldValues.getVal(fieldNum);
return lst.get(shardDoc.orderInShard);
}
}
static Comparator<ShardDoc> comparatorScore() {
return new Comparator<ShardDoc>() {
@Override
public final int compare(final ShardDoc o1, final ShardDoc o2) {
final float f1 = o1.score;
final float f2 = o2.score;
if (f1 < f2)
return -1;
if (f1 > f2)
return 1;
return 0;
}
};
}
Comparator<ShardDoc> comparatorFieldComparator(SortField sortField) {
final FieldComparator fieldComparator;
try {
fieldComparator = sortField.getComparator(0, 0);
} catch (IOException e) {
throw new RuntimeException("Unable to get FieldComparator for sortField " + sortField);
}
return new ShardComparator(sortField) {
// Since the PriorityQueue keeps the biggest elements by default,
// we need to reverse the field compare ordering so that the
// smallest elements are kept instead of the largest... hence
// the negative sign.
@Override
public int compare(final ShardDoc o1, final ShardDoc o2) {
//noinspection unchecked
return -fieldComparator.compareValues(sortVal(o1), sortVal(o2));
}
};
}
}
}

View File

@ -0,0 +1,179 @@
package org.apache.solr.handler.component;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.PriorityQueue;
import org.apache.solr.common.SolrException;
import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
// used by distributed search to merge results.
public class ShardFieldSortedHitQueue extends PriorityQueue<ShardDoc> {
/** Stores a comparator corresponding to each field being sorted by */
protected Comparator<ShardDoc>[] comparators;
/** Stores the sort criteria being used. */
protected SortField[] fields;
/** The order of these fieldNames should correspond to the order of sort field values retrieved from the shard */
protected List<String> fieldNames = new ArrayList<>();
public ShardFieldSortedHitQueue(SortField[] fields, int size, IndexSearcher searcher) {
super(size);
final int n = fields.length;
//noinspection unchecked
comparators = new Comparator[n];
this.fields = new SortField[n];
for (int i = 0; i < n; ++i) {
// keep track of the named fields
SortField.Type type = fields[i].getType();
if (type!=SortField.Type.SCORE && type!=SortField.Type.DOC) {
fieldNames.add(fields[i].getField());
}
String fieldname = fields[i].getField();
comparators[i] = getCachedComparator(fields[i], searcher);
if (fields[i].getType() == SortField.Type.STRING) {
this.fields[i] = new SortField(fieldname, SortField.Type.STRING,
fields[i].getReverse());
} else {
this.fields[i] = new SortField(fieldname, fields[i].getType(),
fields[i].getReverse());
}
//System.out.println("%%%%%%%%%%%%%%%%%% got "+fields[i].getType() +" for "+ fieldname +" fields[i].getReverse(): "+fields[i].getReverse());
}
}
@Override
protected boolean lessThan(ShardDoc docA, ShardDoc docB) {
// If these docs are from the same shard, then the relative order
// is how they appeared in the response from that shard.
if (docA.shard == docB.shard) {
// if docA has a smaller position, it should be "larger" so it
// comes before docB.
// This will handle sorting by docid within the same shard
// comment this out to test comparators.
return !(docA.orderInShard < docB.orderInShard);
}
// run comparators
final int n = comparators.length;
int c = 0;
for (int i = 0; i < n && c == 0; i++) {
c = (fields[i].getReverse()) ? comparators[i].compare(docB, docA)
: comparators[i].compare(docA, docB);
}
// solve tiebreaks by comparing shards (similar to using docid)
// smaller docid's beat larger ids, so reverse the natural ordering
if (c == 0) {
c = -docA.shard.compareTo(docB.shard);
}
return c < 0;
}
Comparator<ShardDoc> getCachedComparator(SortField sortField, IndexSearcher searcher) {
SortField.Type type = sortField.getType();
if (type == SortField.Type.SCORE) {
return comparatorScore();
} else if (type == SortField.Type.REWRITEABLE) {
try {
sortField = sortField.rewrite(searcher);
} catch (IOException e) {
throw new SolrException(SERVER_ERROR, "Exception rewriting sort field " + sortField, e);
}
}
return comparatorFieldComparator(sortField);
}
abstract class ShardComparator implements Comparator<ShardDoc> {
final SortField sortField;
final String fieldName;
final int fieldNum;
public ShardComparator(SortField sortField) {
this.sortField = sortField;
this.fieldName = sortField.getField();
int fieldNum = 0;
for (int i=0; i<fieldNames.size(); i++) {
if (fieldNames.get(i).equals(fieldName)) {
fieldNum = i;
break;
}
}
this.fieldNum = fieldNum;
}
Object sortVal(ShardDoc shardDoc) {
assert(shardDoc.sortFieldValues.getName(fieldNum).equals(fieldName));
List lst = (List)shardDoc.sortFieldValues.getVal(fieldNum);
return lst.get(shardDoc.orderInShard);
}
}
static Comparator<ShardDoc> comparatorScore() {
return new Comparator<ShardDoc>() {
@Override
public final int compare(final ShardDoc o1, final ShardDoc o2) {
final float f1 = o1.score;
final float f2 = o2.score;
if (f1 < f2)
return -1;
if (f1 > f2)
return 1;
return 0;
}
};
}
Comparator<ShardDoc> comparatorFieldComparator(SortField sortField) {
final FieldComparator fieldComparator;
try {
fieldComparator = sortField.getComparator(0, 0);
} catch (IOException e) {
throw new RuntimeException("Unable to get FieldComparator for sortField " + sortField);
}
return new ShardComparator(sortField) {
// Since the PriorityQueue keeps the biggest elements by default,
// we need to reverse the field compare ordering so that the
// smallest elements are kept instead of the largest... hence
// the negative sign.
@Override
public int compare(final ShardDoc o1, final ShardDoc o2) {
//noinspection unchecked
return -fieldComparator.compareValues(sortVal(o1), sortVal(o2));
}
};
}
}