Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

This commit is contained in:
Noble Paul 2016-11-30 18:48:18 +05:30
parent 44cce6bc4c a7fa920b52
commit 9eaea79f5c
12 changed files with 163 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
solr/CHANGES.txt
View File

@ -216,6 +216,8 @@ Bug Fixes
* SOLR-5260: Facet search on a docvalue field in a multi shard collection (Trym Møller, Erick Erickson)
* SOLR-9768: RecordingJsonParser produces incomplete json (Wojciech Stryszyk via ab)
Other Changes
----------------------
@ -245,6 +247,12 @@ Other Changes
* SOLR-9801: Upgrade jetty to 9.3.14.v20161028 (shalin)
* SOLR-9783: (Search|Top)Group[s]ShardResponseProcessor.process: turned sortWithinGroup null check into assert.
(Christine Poerschke)
* SOLR-9660: in GroupingSpecification factor [group](sort|offset|limit) into [group](sortSpec)
(Judith Silverman, Christine Poerschke)
================== 6.3.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

34
solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
View File

@ -252,21 +252,27 @@ public class QueryComponent extends SearchComponent
final SortSpec sortSpec = rb.getSortSpec();
//TODO: move weighting of sort
Sort groupSort = searcher.weightSort(sortSpec.getSort());
if (groupSort == null) {
groupSort = Sort.RELEVANCE;
}
final SortSpec groupSortSpec = searcher.weightSortSpec(sortSpec, Sort.RELEVANCE);
// groupSort defaults to sort
String sortWithinGroupStr = params.get(GroupParams.GROUP_SORT);
//TODO: move weighting of sort
Sort sortWithinGroup = sortWithinGroupStr == null ? groupSort : searcher.weightSort(SortSpecParsing.parseSortSpec(sortWithinGroupStr, req).getSort());
if (sortWithinGroup == null) {
sortWithinGroup = Sort.RELEVANCE;
final SortSpec sortSpecWithinGroup;
if (sortWithinGroupStr != null) {
SortSpec parsedSortSpecWithinGroup = SortSpecParsing.parseSortSpec(sortWithinGroupStr, req);
sortSpecWithinGroup = searcher.weightSortSpec(parsedSortSpecWithinGroup, Sort.RELEVANCE);
} else {
sortSpecWithinGroup = new SortSpec(
groupSortSpec.getSort(),
groupSortSpec.getSchemaFields(),
groupSortSpec.getCount(),
groupSortSpec.getOffset());
}
sortSpecWithinGroup.setOffset(params.getInt(GroupParams.GROUP_OFFSET, 0));
sortSpecWithinGroup.setCount(params.getInt(GroupParams.GROUP_LIMIT, 1));
groupingSpec.setSortWithinGroup(sortWithinGroup);
groupingSpec.setGroupSort(groupSort);
groupingSpec.setSortSpecWithinGroup(sortSpecWithinGroup);
groupingSpec.setGroupSortSpec(groupSortSpec);
String formatStr = params.get(GroupParams.GROUP_FORMAT, Grouping.Format.grouped.name());
Grouping.Format responseFormat;
@ -280,10 +286,6 @@ public class QueryComponent extends SearchComponent
groupingSpec.setFields(params.getParams(GroupParams.GROUP_FIELD));
groupingSpec.setQueries(params.getParams(GroupParams.GROUP_QUERY));
groupingSpec.setFunctions(params.getParams(GroupParams.GROUP_FUNC));
groupingSpec.setGroupOffset(params.getInt(GroupParams.GROUP_OFFSET, 0));
groupingSpec.setGroupLimit(params.getInt(GroupParams.GROUP_LIMIT, 1));
groupingSpec.setOffset(sortSpec.getOffset());
groupingSpec.setLimit(sortSpec.getCount());
groupingSpec.setIncludeGroupCount(params.getBool(GroupParams.GROUP_TOTAL_COUNT, false));
groupingSpec.setMain(params.getBool(GroupParams.GROUP_MAIN, false));
groupingSpec.setNeedScore((rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0);
@ -415,7 +417,7 @@ public class QueryComponent extends SearchComponent
.setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0)
.setSearcher(searcher);
int docsToCollect = Grouping.getMax(groupingSpec.getGroupOffset(), groupingSpec.getGroupLimit(), searcher.maxDoc());
int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc());
docsToCollect = Math.max(docsToCollect, 1);
for (String field : groupingSpec.getFields()) {
@ -477,8 +479,8 @@ public class QueryComponent extends SearchComponent
.setDefaultFormat(groupingSpec.getResponseFormat())
.setLimitDefault(limitDefault)
.setDefaultTotalCount(defaultTotalCount)
.setDocsPerGroupDefault(groupingSpec.getGroupLimit())
.setGroupOffsetDefault(groupingSpec.getGroupOffset())
.setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit())
.setGroupOffsetDefault(groupingSpec.getWithinGroupOffset())
.setGetGroupedDocSet(groupingSpec.isTruncateGroups());
if (groupingSpec.getFields() != null) {

28
solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
View File

@ -460,15 +460,15 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
// alter the sorting in the grouping specification if there is one
GroupingSpecification groupingSpec = rb.getGroupingSpec();
if(groupingSpec != null) {
SortField[] groupSort = groupingSpec.getGroupSort().getSort();
Sort modGroupSort = this.modifySort(groupSort, force, comparator);
if(modGroupSort != null) {
groupingSpec.setGroupSort(modGroupSort);
SortSpec groupSortSpec = groupingSpec.getGroupSortSpec();
SortSpec modGroupSortSpec = this.modifySortSpec(groupSortSpec, force, comparator);
if (modGroupSortSpec != null) {
groupingSpec.setGroupSortSpec(modGroupSortSpec);
}
SortField[] withinGroupSort = groupingSpec.getSortWithinGroup().getSort();
Sort modWithinGroupSort = this.modifySort(withinGroupSort, force, comparator);
if(modWithinGroupSort != null) {
groupingSpec.setSortWithinGroup(modWithinGroupSort);
SortSpec withinGroupSortSpec = groupingSpec.getSortSpecWithinGroup();
SortSpec modWithinGroupSortSpec = this.modifySortSpec(withinGroupSortSpec, force, comparator);
if (modWithinGroupSortSpec != null) {
groupingSpec.setSortSpecWithinGroup(modWithinGroupSortSpec);
}
}
}
@ -494,12 +494,6 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
}
}
private Sort modifySort(SortField[] current, boolean force, ElevationComparatorSource comparator) {
SortSpec tmp = new SortSpec(new Sort(current), Arrays.asList(new SchemaField[current.length]));
tmp = modifySortSpec(tmp, force, comparator);
return null == tmp ? null : tmp.getSort();
}
private SortSpec modifySortSpec(SortSpec current, boolean force, ElevationComparatorSource comparator) {
boolean modify = false;
SortField[] currentSorts = current.getSort().getSort();
@ -526,9 +520,9 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
}
if (modify) {
SortSpec newSpec = new SortSpec(new Sort(sorts.toArray(new SortField[sorts.size()])),
fields);
newSpec.setOffset(current.getOffset());
newSpec.setCount(current.getCount());
fields,
current.getCount(),
current.getOffset());
return newSpec;
}
return null;

26
solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -924,6 +924,32 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
return (sort != null) ? sort.rewrite(this) : null;
}
/** Returns a weighted sort spec according to this searcher */
public SortSpec weightSortSpec(SortSpec originalSortSpec, Sort nullEquivalent) throws IOException {
return implWeightSortSpec(
originalSortSpec.getSort(),
originalSortSpec.getCount(),
originalSortSpec.getOffset(),
nullEquivalent);
}
/** Returns a weighted sort spec according to this searcher */
private SortSpec implWeightSortSpec(Sort originalSort, int num, int offset, Sort nullEquivalent) throws IOException {
Sort rewrittenSort = weightSort(originalSort);
if (rewrittenSort == null) {
rewrittenSort = nullEquivalent;
}
final SortField[] rewrittenSortFields = rewrittenSort.getSort();
final SchemaField[] rewrittenSchemaFields = new SchemaField[rewrittenSortFields.length];
for (int ii = 0; ii < rewrittenSortFields.length; ++ii) {
final String fieldName = rewrittenSortFields[ii].getField();
rewrittenSchemaFields[ii] = (fieldName == null ? null : schema.getFieldOrNull(fieldName));
}
return new SortSpec(rewrittenSort, rewrittenSchemaFields, num, offset);
}
/**
* Returns the first document number containing the term <code>t</code> Returns -1 if no document was found. This
* method is primarily intended for clients that want to fetch documents using a unique identifier."

10
solr/core/src/java/org/apache/solr/search/SortSpec.java
View File

@ -34,9 +34,19 @@ public class SortSpec
private int num = 10;
private int offset = 0;
public SortSpec(Sort sort, List<SchemaField> fields, int num, int offset) {
setSortAndFields(sort, fields);
this.num = num;
this.offset = offset;
}
public SortSpec(Sort sort, List<SchemaField> fields) {
setSortAndFields(sort, fields);
}
public SortSpec(Sort sort, SchemaField[] fields, int num, int offset) {
setSortAndFields(sort, Arrays.asList(fields));
this.num = num;
this.offset = offset;
}
public SortSpec(Sort sort, SchemaField[] fields) {
setSortAndFields(sort, Arrays.asList(fields));
}

70
solr/core/src/java/org/apache/solr/search/grouping/GroupingSpecification.java
View File

@ -18,6 +18,7 @@ package org.apache.solr.search.grouping;
import org.apache.lucene.search.Sort;
import org.apache.solr.search.Grouping;
import org.apache.solr.search.SortSpec;
/**
* Encapsulates the grouping options like fields group sort and more specified by clients.
@ -29,12 +30,8 @@ public class GroupingSpecification {
private String[] fields = new String[]{};
private String[] queries = new String[]{};
private String[] functions = new String[]{};
private int offset;
private int limit;
private int groupOffset;
private int groupLimit;
private Sort groupSort;
private Sort sortWithinGroup;
private SortSpec groupSortSpec;
private SortSpec sortSpecWithinGroup;
private boolean includeGroupCount;
private boolean main;
private Grouping.Format responseFormat;
@ -77,53 +74,49 @@ public class GroupingSpecification {
this.functions = functions;
}
@Deprecated
public int getWithinGroupOffset() {
return sortSpecWithinGroup.getOffset();
}
@Deprecated
public int getGroupOffset() {
return groupOffset;
return getWithinGroupOffset();
}
public void setGroupOffset(int groupOffset) {
this.groupOffset = groupOffset;
}
@Deprecated
public int getWithinGroupLimit() {
return sortSpecWithinGroup.getCount();
}
@Deprecated
public int getGroupLimit() {
return groupLimit;
return getWithinGroupLimit();
}
public void setGroupLimit(int groupLimit) {
this.groupLimit = groupLimit;
}
@Deprecated
public int getOffset() {
return offset;
return groupSortSpec.getOffset();
}
public void setOffset(int offset) {
this.offset = offset;
}
@Deprecated
public int getLimit() {
return limit;
return groupSortSpec.getCount();
}
public void setLimit(int limit) {
this.limit = limit;
}
@Deprecated
public Sort getGroupSort() {
return groupSort;
return groupSortSpec.getSort();
}
public void setGroupSort(Sort groupSort) {
this.groupSort = groupSort;
}
@Deprecated
public Sort getSortWithinGroup() {
return sortWithinGroup;
return sortSpecWithinGroup.getSort();
}
public void setSortWithinGroup(Sort sortWithinGroup) {
this.sortWithinGroup = sortWithinGroup;
}
public boolean isIncludeGroupCount() {
return includeGroupCount;
@ -164,4 +157,21 @@ public class GroupingSpecification {
public void setTruncateGroups(boolean truncateGroups) {
this.truncateGroups = truncateGroups;
}
public SortSpec getGroupSortSpec() {
return groupSortSpec;
}
public void setGroupSortSpec(SortSpec groupSortSpec) {
this.groupSortSpec = groupSortSpec;
}
public SortSpec getSortSpecWithinGroup() {
return sortSpecWithinGroup;
}
public void setSortSpecWithinGroup(SortSpec sortSpecWithinGroup) {
this.sortSpecWithinGroup = sortSpecWithinGroup;
}
}

2
solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java
View File

@ -124,7 +124,7 @@ public class QueryCommand implements Command<QueryCommandResult> {
@Override
public List<Collector> create() throws IOException {
if (sort == null || sort == Sort.RELEVANCE) {
if (sort == null || sort.equals(Sort.RELEVANCE)) {
collector = TopScoreDocCollector.create(docsToCollect);
} else {
collector = TopFieldCollector.create(sort, docsToCollect, true, needScores, needScores);

4
solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
View File

@ -52,9 +52,7 @@ public class SearchGroupShardResponseProcessor implements ShardResponseProcessor
Sort groupSort = rb.getGroupingSpec().getGroupSort();
final String[] fields = rb.getGroupingSpec().getFields();
Sort sortWithinGroup = rb.getGroupingSpec().getSortWithinGroup();
if (sortWithinGroup == null) { // TODO prevent it from being null in the first place
sortWithinGroup = Sort.RELEVANCE;
}
assert sortWithinGroup != null;
final Map<String, List<Collection<SearchGroup<BytesRef>>>> commandSearchGroups = new HashMap<>(fields.length, 1.0f);
final Map<String, Map<SearchGroup<BytesRef>, Set<String>>> tempSearchGroupToShards = new HashMap<>(fields.length, 1.0f);

8
solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
View File

@ -61,18 +61,16 @@ public class TopGroupsShardResponseProcessor implements ShardResponseProcessor {
String[] fields = rb.getGroupingSpec().getFields();
String[] queries = rb.getGroupingSpec().getQueries();
Sort sortWithinGroup = rb.getGroupingSpec().getSortWithinGroup();
if (sortWithinGroup == null) { // TODO prevent it from being null in the first place
sortWithinGroup = Sort.RELEVANCE;
}
assert sortWithinGroup != null;
// If group.format=simple group.offset doesn't make sense
int groupOffsetDefault;
if (rb.getGroupingSpec().getResponseFormat() == Grouping.Format.simple || rb.getGroupingSpec().isMain()) {
groupOffsetDefault = 0;
} else {
groupOffsetDefault = rb.getGroupingSpec().getGroupOffset();
groupOffsetDefault = rb.getGroupingSpec().getWithinGroupOffset();
}
int docsPerGroupDefault = rb.getGroupingSpec().getGroupLimit();
int docsPerGroupDefault = rb.getGroupingSpec().getWithinGroupLimit();
Map<String, List<TopGroups<BytesRef>>> commandTopGroups = new HashMap<>();
for (String field : fields) {

4
solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/GroupedEndResultTransformer.java
View File

@ -79,7 +79,7 @@ public class GroupedEndResultTransformer implements EndResultTransformer {
if (!Float.isNaN(group.maxScore)) {
docList.setMaxScore(group.maxScore);
}
docList.setStart(rb.getGroupingSpec().getGroupOffset());
docList.setStart(rb.getGroupingSpec().getWithinGroupOffset());
for (ScoreDoc scoreDoc : group.scoreDocs) {
docList.add(solrDocumentSource.retrieve(scoreDoc));
}
@ -97,7 +97,7 @@ public class GroupedEndResultTransformer implements EndResultTransformer {
if (!Float.isNaN(queryCommandResult.getTopDocs().getMaxScore())) {
docList.setMaxScore(queryCommandResult.getTopDocs().getMaxScore());
}
docList.setStart(rb.getGroupingSpec().getGroupOffset());
docList.setStart(rb.getGroupingSpec().getWithinGroupOffset());
for (ScoreDoc scoreDoc :queryCommandResult.getTopDocs().scoreDocs){
docList.add(solrDocumentSource.retrieve(scoreDoc));
}

17
solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java
View File

@ -29,7 +29,9 @@ public class RecordingJSONParser extends JSONParser {
private StringBuilder sb = new StringBuilder();
private boolean objectStarted = false;
public long lastMarkedPosition = 0;
private long lastMarkedPosition = 0;
private long lastGlobalPosition = 0;
private static final int BUFFER_SIZE = 8192;
public RecordingJSONParser(Reader in) {
@ -39,7 +41,7 @@ public class RecordingJSONParser extends JSONParser {
}
static char[] getChars() {
buf.set(new char[8192]);
buf.set(new char[BUFFER_SIZE]);
return buf.get();
}
@ -68,11 +70,22 @@ public class RecordingJSONParser extends JSONParser {
if(currPosition < 0){
System.out.println("ERROR");
}
if (currPosition > lastMarkedPosition) {
for (long i = lastMarkedPosition; i < currPosition; i++) {
recordChar(bufCopy[(int) i]);
}
} else if (currPosition < lastMarkedPosition) {
for (long i = 0; i < currPosition; i++) {
recordChar(bufCopy[(int) i]);
}
} else if (currPosition == BUFFER_SIZE && lastGlobalPosition != globalPosition) {
for (long i = 0; i < currPosition; i++) {
recordChar(bufCopy[(int) i]);
}
}
lastGlobalPosition = globalPosition;
lastMarkedPosition = currPosition;
}

35
solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
View File

@ -16,11 +16,6 @@
*/
package org.apache.solr.common.util;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.util.RecordingJSONParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.StringReader;
import java.lang.invoke.MethodHandles;
@ -31,6 +26,12 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.util.RecordingJSONParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TestJsonRecordReader extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ -129,18 +130,32 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 {
" \"nested_inside\" : \"check check check 1\"\n" +
" }\n" +
"}";
String json2 =
" {\n" +
" \"id\" : \"345\",\n" +
" \"payload\": \""+ StringUtils.repeat("0123456789", 819) +
"\",\n" +
" \"description\": \"Testing /json/docs srcField 2\",\n" +
"\n" +
" \"nested_data\" : {\n" +
" \"nested_inside\" : \"check check check 2\"\n" +
" }\n" +
"}";
JsonRecordReader streamer = JsonRecordReader.getInst("/", Arrays.asList("id:/id"));
RecordingJSONParser parser = new RecordingJSONParser(new StringReader(json + json2));
String json3 =
" {\n" +
" \"id\" : \"678\",\n" +
" \"description\": \"Testing /json/docs srcField 3\",\n" +
"\n" +
" \"nested_data\" : {\n" +
" \"nested_inside\" : \"check check check 3\"\n" +
" }\n" +
"}";
JsonRecordReader streamer = JsonRecordReader.getInst("/", Arrays.asList("id:/id"));
RecordingJSONParser parser = new RecordingJSONParser(new StringReader(json + json2 + json3));
streamer.streamRecords(parser, new JsonRecordReader.Handler() {
int count = 0;
@ -162,6 +177,12 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 {
assertEquals(m.get("description"), "Testing /json/docs srcField 2");
assertEquals(((Map) m.get("nested_data")).get("nested_inside"), "check check check 2");
}
if (count++ == 3) {
assertEquals(m.get("id"), "678");
assertEquals(m.get("description"), "Testing /json/docs srcField 3");
assertEquals(((Map) m.get("nested_data")).get("nested_inside"), "check check check 3");
}
}
});