mirror of https://github.com/apache/lucene.git
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
This commit is contained in:
commit
2b6c82ed32
|
@ -200,6 +200,10 @@ Bug Fixes
|
||||||
* LUCENE-7956: Fixed potential stack overflow error in ICUNormalizer2CharFilter.
|
* LUCENE-7956: Fixed potential stack overflow error in ICUNormalizer2CharFilter.
|
||||||
(Adrien Grand)
|
(Adrien Grand)
|
||||||
|
|
||||||
|
* LUCENE-7963: Remove useless getAttribute() in DefaultIndexingChain that
|
||||||
|
causes performance drop, introduced by LUCENE-7626. (Daniel Mitterdorfer
|
||||||
|
via Uwe Schindler)
|
||||||
|
|
||||||
Improvements
|
Improvements
|
||||||
|
|
||||||
* LUCENE-7489: Better storage of sparse doc-values fields with the default
|
* LUCENE-7489: Better storage of sparse doc-values fields with the default
|
||||||
|
|
|
@ -27,7 +27,6 @@ import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import org.apache.lucene.analysis.TokenStream;
|
import org.apache.lucene.analysis.TokenStream;
|
||||||
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
|
|
||||||
import org.apache.lucene.codecs.DocValuesConsumer;
|
import org.apache.lucene.codecs.DocValuesConsumer;
|
||||||
import org.apache.lucene.codecs.DocValuesFormat;
|
import org.apache.lucene.codecs.DocValuesFormat;
|
||||||
import org.apache.lucene.codecs.NormsConsumer;
|
import org.apache.lucene.codecs.NormsConsumer;
|
||||||
|
@ -733,7 +732,6 @@ final class DefaultIndexingChain extends DocConsumer {
|
||||||
stream.reset();
|
stream.reset();
|
||||||
invertState.setAttributeSource(stream);
|
invertState.setAttributeSource(stream);
|
||||||
termsHashPerField.start(field, first);
|
termsHashPerField.start(field, first);
|
||||||
CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
|
|
||||||
|
|
||||||
while (stream.incrementToken()) {
|
while (stream.incrementToken()) {
|
||||||
|
|
||||||
|
|
|
@ -197,6 +197,7 @@ Other Changes
|
||||||
* SOLR-11322: JSON Facet API: instead of returning NaN, min & max aggregations omit
|
* SOLR-11322: JSON Facet API: instead of returning NaN, min & max aggregations omit
|
||||||
the value for any bucket with no values in the numeric field. (yonik)
|
the value for any bucket with no values in the numeric field. (yonik)
|
||||||
|
|
||||||
|
* SOLR-10990: Breakup QueryComponent.process method for readability. (Christine Poerschke)
|
||||||
|
|
||||||
================== 7.0.0 ==================
|
================== 7.0.0 ==================
|
||||||
|
|
||||||
|
|
|
@ -306,12 +306,12 @@ public class QueryComponent extends SearchComponent
|
||||||
if (!params.getBool(COMPONENT_NAME, true)) {
|
if (!params.getBool(COMPONENT_NAME, true)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
SolrIndexSearcher searcher = req.getSearcher();
|
|
||||||
|
|
||||||
StatsCache statsCache = req.getCore().getStatsCache();
|
StatsCache statsCache = req.getCore().getStatsCache();
|
||||||
|
|
||||||
int purpose = params.getInt(ShardParams.SHARDS_PURPOSE, ShardRequest.PURPOSE_GET_TOP_IDS);
|
int purpose = params.getInt(ShardParams.SHARDS_PURPOSE, ShardRequest.PURPOSE_GET_TOP_IDS);
|
||||||
if ((purpose & ShardRequest.PURPOSE_GET_TERM_STATS) != 0) {
|
if ((purpose & ShardRequest.PURPOSE_GET_TERM_STATS) != 0) {
|
||||||
|
SolrIndexSearcher searcher = req.getSearcher();
|
||||||
statsCache.returnLocalStats(rb, searcher);
|
statsCache.returnLocalStats(rb, searcher);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -321,50 +321,11 @@ public class QueryComponent extends SearchComponent
|
||||||
statsCache.receiveGlobalStats(req);
|
statsCache.receiveGlobalStats(req);
|
||||||
}
|
}
|
||||||
|
|
||||||
SolrQueryResponse rsp = rb.rsp;
|
|
||||||
IndexSchema schema = searcher.getSchema();
|
|
||||||
|
|
||||||
// Optional: This could also be implemented by the top-level searcher sending
|
// Optional: This could also be implemented by the top-level searcher sending
|
||||||
// a filter that lists the ids... that would be transparent to
|
// a filter that lists the ids... that would be transparent to
|
||||||
// the request handler, but would be more expensive (and would preserve score
|
// the request handler, but would be more expensive (and would preserve score
|
||||||
// too if desired).
|
// too if desired).
|
||||||
String ids = params.get(ShardParams.IDS);
|
if (doProcessSearchByIds(rb)) {
|
||||||
if (ids != null) {
|
|
||||||
SchemaField idField = schema.getUniqueKeyField();
|
|
||||||
List<String> idArr = StrUtils.splitSmart(ids, ",", true);
|
|
||||||
int[] luceneIds = new int[idArr.size()];
|
|
||||||
int docs = 0;
|
|
||||||
if (idField.getType().isPointField()) {
|
|
||||||
for (int i=0; i<idArr.size(); i++) {
|
|
||||||
int id = searcher.search(
|
|
||||||
idField.getType().getFieldQuery(null, idField, idArr.get(i)), 1).scoreDocs[0].doc;
|
|
||||||
if (id >= 0) {
|
|
||||||
luceneIds[docs++] = id;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
for (int i=0; i<idArr.size(); i++) {
|
|
||||||
int id = searcher.getFirstMatch(
|
|
||||||
new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
|
|
||||||
if (id >= 0)
|
|
||||||
luceneIds[docs++] = id;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
DocListAndSet res = new DocListAndSet();
|
|
||||||
res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
|
|
||||||
if (rb.isNeedDocSet()) {
|
|
||||||
// TODO: create a cache for this!
|
|
||||||
List<Query> queries = new ArrayList<>();
|
|
||||||
queries.add(rb.getQuery());
|
|
||||||
List<Query> filters = rb.getFilters();
|
|
||||||
if (filters != null) queries.addAll(filters);
|
|
||||||
res.docSet = searcher.getDocSet(queries);
|
|
||||||
}
|
|
||||||
rb.setResults(res);
|
|
||||||
|
|
||||||
ResultContext ctx = new BasicResultContext(rb);
|
|
||||||
rsp.addResponse(ctx);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -395,145 +356,15 @@ public class QueryComponent extends SearchComponent
|
||||||
if (groupingSpec != null) {
|
if (groupingSpec != null) {
|
||||||
cmd.setSegmentTerminateEarly(false); // not supported, silently ignore any segmentTerminateEarly flag
|
cmd.setSegmentTerminateEarly(false); // not supported, silently ignore any segmentTerminateEarly flag
|
||||||
try {
|
try {
|
||||||
boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
|
|
||||||
if (params.getBool(GroupParams.GROUP_DISTRIBUTED_FIRST, false)) {
|
if (params.getBool(GroupParams.GROUP_DISTRIBUTED_FIRST, false)) {
|
||||||
CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder()
|
doProcessGroupedDistributedSearchFirstPhase(rb, result);
|
||||||
.setQueryCommand(cmd)
|
|
||||||
.setNeedDocSet(false) // Order matters here
|
|
||||||
.setIncludeHitCount(true)
|
|
||||||
.setSearcher(searcher);
|
|
||||||
|
|
||||||
for (String field : groupingSpec.getFields()) {
|
|
||||||
topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder()
|
|
||||||
.setField(schema.getField(field))
|
|
||||||
.setGroupSort(groupingSpec.getGroupSort())
|
|
||||||
.setTopNGroups(cmd.getOffset() + cmd.getLen())
|
|
||||||
.setIncludeGroupCount(groupingSpec.isIncludeGroupCount())
|
|
||||||
.build()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
CommandHandler commandHandler = topsGroupsActionBuilder.build();
|
|
||||||
commandHandler.execute();
|
|
||||||
SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher);
|
|
||||||
rsp.add("firstPhase", commandHandler.processResult(result, serializer));
|
|
||||||
rsp.add("totalHitCount", commandHandler.getTotalHitCount());
|
|
||||||
rb.setResult(result);
|
|
||||||
return;
|
return;
|
||||||
} else if (params.getBool(GroupParams.GROUP_DISTRIBUTED_SECOND, false)) {
|
} else if (params.getBool(GroupParams.GROUP_DISTRIBUTED_SECOND, false)) {
|
||||||
CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder()
|
doProcessGroupedDistributedSearchSecondPhase(rb, result);
|
||||||
.setQueryCommand(cmd)
|
|
||||||
.setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0)
|
|
||||||
.setSearcher(searcher);
|
|
||||||
|
|
||||||
int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc());
|
|
||||||
docsToCollect = Math.max(docsToCollect, 1);
|
|
||||||
|
|
||||||
for (String field : groupingSpec.getFields()) {
|
|
||||||
SchemaField schemaField = schema.getField(field);
|
|
||||||
String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field);
|
|
||||||
if (topGroupsParam == null) {
|
|
||||||
topGroupsParam = new String[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
List<SearchGroup<BytesRef>> topGroups = new ArrayList<>(topGroupsParam.length);
|
|
||||||
for (String topGroup : topGroupsParam) {
|
|
||||||
SearchGroup<BytesRef> searchGroup = new SearchGroup<>();
|
|
||||||
if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) {
|
|
||||||
BytesRefBuilder builder = new BytesRefBuilder();
|
|
||||||
schemaField.getType().readableToIndexed(topGroup, builder);
|
|
||||||
searchGroup.groupValue = builder.get();
|
|
||||||
}
|
|
||||||
topGroups.add(searchGroup);
|
|
||||||
}
|
|
||||||
|
|
||||||
secondPhaseBuilder.addCommandField(
|
|
||||||
new TopGroupsFieldCommand.Builder()
|
|
||||||
.setField(schemaField)
|
|
||||||
.setGroupSort(groupingSpec.getGroupSort())
|
|
||||||
.setSortWithinGroup(groupingSpec.getSortWithinGroup())
|
|
||||||
.setFirstPhaseGroups(topGroups)
|
|
||||||
.setMaxDocPerGroup(docsToCollect)
|
|
||||||
.setNeedScores(needScores)
|
|
||||||
.setNeedMaxScore(needScores)
|
|
||||||
.build()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (String query : groupingSpec.getQueries()) {
|
|
||||||
secondPhaseBuilder.addCommandField(new Builder()
|
|
||||||
.setDocsToCollect(docsToCollect)
|
|
||||||
.setSort(groupingSpec.getGroupSort())
|
|
||||||
.setQuery(query, rb.req)
|
|
||||||
.setDocSet(searcher)
|
|
||||||
.build()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
CommandHandler commandHandler = secondPhaseBuilder.build();
|
|
||||||
commandHandler.execute();
|
|
||||||
TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
|
|
||||||
rsp.add("secondPhase", commandHandler.processResult(result, serializer));
|
|
||||||
rb.setResult(result);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
|
doProcessGroupedSearch(rb, result);
|
||||||
boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
|
|
||||||
Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ?
|
|
||||||
Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
|
|
||||||
int limitDefault = cmd.getLen(); // this is normally from "rows"
|
|
||||||
Grouping grouping =
|
|
||||||
new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
|
|
||||||
grouping.setGroupSort(groupingSpec.getGroupSort())
|
|
||||||
.setWithinGroupSort(groupingSpec.getSortWithinGroup())
|
|
||||||
.setDefaultFormat(groupingSpec.getResponseFormat())
|
|
||||||
.setLimitDefault(limitDefault)
|
|
||||||
.setDefaultTotalCount(defaultTotalCount)
|
|
||||||
.setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit())
|
|
||||||
.setGroupOffsetDefault(groupingSpec.getWithinGroupOffset())
|
|
||||||
.setGetGroupedDocSet(groupingSpec.isTruncateGroups());
|
|
||||||
|
|
||||||
if (groupingSpec.getFields() != null) {
|
|
||||||
for (String field : groupingSpec.getFields()) {
|
|
||||||
grouping.addFieldCommand(field, rb.req);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (groupingSpec.getFunctions() != null) {
|
|
||||||
for (String groupByStr : groupingSpec.getFunctions()) {
|
|
||||||
grouping.addFunctionCommand(groupByStr, rb.req);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (groupingSpec.getQueries() != null) {
|
|
||||||
for (String groupByStr : groupingSpec.getQueries()) {
|
|
||||||
grouping.addQueryCommand(groupByStr, rb.req);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if( rb.isNeedDocList() || rb.isDebug() ){
|
|
||||||
// we need a single list of the returned docs
|
|
||||||
cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
|
|
||||||
}
|
|
||||||
|
|
||||||
grouping.execute();
|
|
||||||
if (grouping.isSignalCacheWarning()) {
|
|
||||||
rsp.add(
|
|
||||||
"cacheWarning",
|
|
||||||
String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
rb.setResult(result);
|
|
||||||
|
|
||||||
if (grouping.mainResult != null) {
|
|
||||||
ResultContext ctx = new BasicResultContext(rb, grouping.mainResult);
|
|
||||||
rsp.addResponse(ctx);
|
|
||||||
rsp.getToLog().add("hits", grouping.mainResult.matches());
|
|
||||||
} else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this.
|
|
||||||
rsp.add("grouped", result.groupedResults);
|
|
||||||
rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
|
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
} catch (SyntaxError e) {
|
} catch (SyntaxError e) {
|
||||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
|
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
|
||||||
|
@ -541,27 +372,7 @@ public class QueryComponent extends SearchComponent
|
||||||
}
|
}
|
||||||
|
|
||||||
// normal search result
|
// normal search result
|
||||||
searcher.search(result, cmd);
|
doProcessUngroupedSearch(rb, result);
|
||||||
rb.setResult(result);
|
|
||||||
|
|
||||||
ResultContext ctx = new BasicResultContext(rb);
|
|
||||||
rsp.addResponse(ctx);
|
|
||||||
rsp.getToLog().add("hits", rb.getResults().docList.matches());
|
|
||||||
|
|
||||||
if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
|
|
||||||
if (null != rb.getNextCursorMark()) {
|
|
||||||
rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
|
|
||||||
rb.getNextCursorMark().getSerializedTotem());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if(rb.mergeFieldHandler != null) {
|
|
||||||
rb.mergeFieldHandler.handleMergeFields(rb, searcher);
|
|
||||||
} else {
|
|
||||||
doFieldSortValues(rb, searcher);
|
|
||||||
}
|
|
||||||
|
|
||||||
doPrefetch(rb);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException
|
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException
|
||||||
|
@ -1385,6 +1196,272 @@ public class QueryComponent extends SearchComponent
|
||||||
return Category.QUERY;
|
return Category.QUERY;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private boolean doProcessSearchByIds(ResponseBuilder rb) throws IOException {
|
||||||
|
|
||||||
|
SolrQueryRequest req = rb.req;
|
||||||
|
SolrQueryResponse rsp = rb.rsp;
|
||||||
|
|
||||||
|
SolrParams params = req.getParams();
|
||||||
|
|
||||||
|
String ids = params.get(ShardParams.IDS);
|
||||||
|
if (ids == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
SolrIndexSearcher searcher = req.getSearcher();
|
||||||
|
IndexSchema schema = searcher.getSchema();
|
||||||
|
SchemaField idField = schema.getUniqueKeyField();
|
||||||
|
List<String> idArr = StrUtils.splitSmart(ids, ",", true);
|
||||||
|
int[] luceneIds = new int[idArr.size()];
|
||||||
|
int docs = 0;
|
||||||
|
if (idField.getType().isPointField()) {
|
||||||
|
for (int i=0; i<idArr.size(); i++) {
|
||||||
|
int id = searcher.search(
|
||||||
|
idField.getType().getFieldQuery(null, idField, idArr.get(i)), 1).scoreDocs[0].doc;
|
||||||
|
if (id >= 0) {
|
||||||
|
luceneIds[docs++] = id;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for (int i=0; i<idArr.size(); i++) {
|
||||||
|
int id = searcher.getFirstMatch(
|
||||||
|
new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
|
||||||
|
if (id >= 0)
|
||||||
|
luceneIds[docs++] = id;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DocListAndSet res = new DocListAndSet();
|
||||||
|
res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
|
||||||
|
if (rb.isNeedDocSet()) {
|
||||||
|
// TODO: create a cache for this!
|
||||||
|
List<Query> queries = new ArrayList<>();
|
||||||
|
queries.add(rb.getQuery());
|
||||||
|
List<Query> filters = rb.getFilters();
|
||||||
|
if (filters != null) queries.addAll(filters);
|
||||||
|
res.docSet = searcher.getDocSet(queries);
|
||||||
|
}
|
||||||
|
rb.setResults(res);
|
||||||
|
|
||||||
|
ResultContext ctx = new BasicResultContext(rb);
|
||||||
|
rsp.addResponse(ctx);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void doProcessGroupedDistributedSearchFirstPhase(ResponseBuilder rb, QueryResult result) throws IOException {
|
||||||
|
|
||||||
|
GroupingSpecification groupingSpec = rb.getGroupingSpec();
|
||||||
|
assert null != groupingSpec : "GroupingSpecification is null";
|
||||||
|
|
||||||
|
SolrQueryRequest req = rb.req;
|
||||||
|
SolrQueryResponse rsp = rb.rsp;
|
||||||
|
|
||||||
|
SolrIndexSearcher searcher = req.getSearcher();
|
||||||
|
IndexSchema schema = searcher.getSchema();
|
||||||
|
|
||||||
|
QueryCommand cmd = rb.getQueryCommand();
|
||||||
|
|
||||||
|
CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder()
|
||||||
|
.setQueryCommand(cmd)
|
||||||
|
.setNeedDocSet(false) // Order matters here
|
||||||
|
.setIncludeHitCount(true)
|
||||||
|
.setSearcher(searcher);
|
||||||
|
|
||||||
|
for (String field : groupingSpec.getFields()) {
|
||||||
|
topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder()
|
||||||
|
.setField(schema.getField(field))
|
||||||
|
.setGroupSort(groupingSpec.getGroupSort())
|
||||||
|
.setTopNGroups(cmd.getOffset() + cmd.getLen())
|
||||||
|
.setIncludeGroupCount(groupingSpec.isIncludeGroupCount())
|
||||||
|
.build()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
CommandHandler commandHandler = topsGroupsActionBuilder.build();
|
||||||
|
commandHandler.execute();
|
||||||
|
SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher);
|
||||||
|
|
||||||
|
rsp.add("firstPhase", commandHandler.processResult(result, serializer));
|
||||||
|
rsp.add("totalHitCount", commandHandler.getTotalHitCount());
|
||||||
|
rb.setResult(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void doProcessGroupedDistributedSearchSecondPhase(ResponseBuilder rb, QueryResult result) throws IOException, SyntaxError {
|
||||||
|
|
||||||
|
GroupingSpecification groupingSpec = rb.getGroupingSpec();
|
||||||
|
assert null != groupingSpec : "GroupingSpecification is null";
|
||||||
|
|
||||||
|
SolrQueryRequest req = rb.req;
|
||||||
|
SolrQueryResponse rsp = rb.rsp;
|
||||||
|
|
||||||
|
SolrParams params = req.getParams();
|
||||||
|
|
||||||
|
SolrIndexSearcher searcher = req.getSearcher();
|
||||||
|
IndexSchema schema = searcher.getSchema();
|
||||||
|
|
||||||
|
QueryCommand cmd = rb.getQueryCommand();
|
||||||
|
boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
|
||||||
|
|
||||||
|
CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder()
|
||||||
|
.setQueryCommand(cmd)
|
||||||
|
.setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0)
|
||||||
|
.setSearcher(searcher);
|
||||||
|
|
||||||
|
int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc());
|
||||||
|
docsToCollect = Math.max(docsToCollect, 1);
|
||||||
|
|
||||||
|
for (String field : groupingSpec.getFields()) {
|
||||||
|
SchemaField schemaField = schema.getField(field);
|
||||||
|
String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field);
|
||||||
|
if (topGroupsParam == null) {
|
||||||
|
topGroupsParam = new String[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
List<SearchGroup<BytesRef>> topGroups = new ArrayList<>(topGroupsParam.length);
|
||||||
|
for (String topGroup : topGroupsParam) {
|
||||||
|
SearchGroup<BytesRef> searchGroup = new SearchGroup<>();
|
||||||
|
if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) {
|
||||||
|
BytesRefBuilder builder = new BytesRefBuilder();
|
||||||
|
schemaField.getType().readableToIndexed(topGroup, builder);
|
||||||
|
searchGroup.groupValue = builder.get();
|
||||||
|
}
|
||||||
|
topGroups.add(searchGroup);
|
||||||
|
}
|
||||||
|
|
||||||
|
secondPhaseBuilder.addCommandField(
|
||||||
|
new TopGroupsFieldCommand.Builder()
|
||||||
|
.setField(schemaField)
|
||||||
|
.setGroupSort(groupingSpec.getGroupSort())
|
||||||
|
.setSortWithinGroup(groupingSpec.getSortWithinGroup())
|
||||||
|
.setFirstPhaseGroups(topGroups)
|
||||||
|
.setMaxDocPerGroup(docsToCollect)
|
||||||
|
.setNeedScores(needScores)
|
||||||
|
.setNeedMaxScore(needScores)
|
||||||
|
.build()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (String query : groupingSpec.getQueries()) {
|
||||||
|
secondPhaseBuilder.addCommandField(new Builder()
|
||||||
|
.setDocsToCollect(docsToCollect)
|
||||||
|
.setSort(groupingSpec.getGroupSort())
|
||||||
|
.setQuery(query, rb.req)
|
||||||
|
.setDocSet(searcher)
|
||||||
|
.build()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
CommandHandler commandHandler = secondPhaseBuilder.build();
|
||||||
|
commandHandler.execute();
|
||||||
|
TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
|
||||||
|
rsp.add("secondPhase", commandHandler.processResult(result, serializer));
|
||||||
|
rb.setResult(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void doProcessGroupedSearch(ResponseBuilder rb, QueryResult result) throws IOException, SyntaxError {
|
||||||
|
|
||||||
|
GroupingSpecification groupingSpec = rb.getGroupingSpec();
|
||||||
|
assert null != groupingSpec : "GroupingSpecification is null";
|
||||||
|
|
||||||
|
SolrQueryRequest req = rb.req;
|
||||||
|
SolrQueryResponse rsp = rb.rsp;
|
||||||
|
|
||||||
|
SolrParams params = req.getParams();
|
||||||
|
|
||||||
|
SolrIndexSearcher searcher = req.getSearcher();
|
||||||
|
|
||||||
|
QueryCommand cmd = rb.getQueryCommand();
|
||||||
|
|
||||||
|
int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
|
||||||
|
boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
|
||||||
|
Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ?
|
||||||
|
Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
|
||||||
|
int limitDefault = cmd.getLen(); // this is normally from "rows"
|
||||||
|
Grouping grouping =
|
||||||
|
new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
|
||||||
|
grouping.setGroupSort(groupingSpec.getGroupSort())
|
||||||
|
.setWithinGroupSort(groupingSpec.getSortWithinGroup())
|
||||||
|
.setDefaultFormat(groupingSpec.getResponseFormat())
|
||||||
|
.setLimitDefault(limitDefault)
|
||||||
|
.setDefaultTotalCount(defaultTotalCount)
|
||||||
|
.setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit())
|
||||||
|
.setGroupOffsetDefault(groupingSpec.getWithinGroupOffset())
|
||||||
|
.setGetGroupedDocSet(groupingSpec.isTruncateGroups());
|
||||||
|
|
||||||
|
if (groupingSpec.getFields() != null) {
|
||||||
|
for (String field : groupingSpec.getFields()) {
|
||||||
|
grouping.addFieldCommand(field, rb.req);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (groupingSpec.getFunctions() != null) {
|
||||||
|
for (String groupByStr : groupingSpec.getFunctions()) {
|
||||||
|
grouping.addFunctionCommand(groupByStr, rb.req);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (groupingSpec.getQueries() != null) {
|
||||||
|
for (String groupByStr : groupingSpec.getQueries()) {
|
||||||
|
grouping.addQueryCommand(groupByStr, rb.req);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if( rb.isNeedDocList() || rb.isDebug() ){
|
||||||
|
// we need a single list of the returned docs
|
||||||
|
cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
|
||||||
|
}
|
||||||
|
|
||||||
|
grouping.execute();
|
||||||
|
if (grouping.isSignalCacheWarning()) {
|
||||||
|
rsp.add(
|
||||||
|
"cacheWarning",
|
||||||
|
String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
rb.setResult(result);
|
||||||
|
|
||||||
|
if (grouping.mainResult != null) {
|
||||||
|
ResultContext ctx = new BasicResultContext(rb, grouping.mainResult);
|
||||||
|
rsp.addResponse(ctx);
|
||||||
|
rsp.getToLog().add("hits", grouping.mainResult.matches());
|
||||||
|
} else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this.
|
||||||
|
rsp.add("grouped", result.groupedResults);
|
||||||
|
rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void doProcessUngroupedSearch(ResponseBuilder rb, QueryResult result) throws IOException {
|
||||||
|
|
||||||
|
SolrQueryRequest req = rb.req;
|
||||||
|
SolrQueryResponse rsp = rb.rsp;
|
||||||
|
|
||||||
|
SolrIndexSearcher searcher = req.getSearcher();
|
||||||
|
|
||||||
|
QueryCommand cmd = rb.getQueryCommand();
|
||||||
|
|
||||||
|
searcher.search(result, cmd);
|
||||||
|
rb.setResult(result);
|
||||||
|
|
||||||
|
ResultContext ctx = new BasicResultContext(rb);
|
||||||
|
rsp.addResponse(ctx);
|
||||||
|
rsp.getToLog().add("hits", rb.getResults().docList.matches());
|
||||||
|
|
||||||
|
if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
|
||||||
|
if (null != rb.getNextCursorMark()) {
|
||||||
|
rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
|
||||||
|
rb.getNextCursorMark().getSerializedTotem());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(rb.mergeFieldHandler != null) {
|
||||||
|
rb.mergeFieldHandler.handleMergeFields(rb, searcher);
|
||||||
|
} else {
|
||||||
|
doFieldSortValues(rb, searcher);
|
||||||
|
}
|
||||||
|
|
||||||
|
doPrefetch(rb);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fake scorer for a single document
|
* Fake scorer for a single document
|
||||||
*
|
*
|
||||||
|
|
Loading…
Reference in New Issue