mirror of https://github.com/apache/lucene.git
SOLR-6581: Efficient DocValues support and numeric collapse field implementations for Collapse and Expand
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1651087 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Joel Bernstein
parent
1f2d431273
commit
d385c653c2
|
|
@ -291,6 +291,9 @@ New Features
|
|||
|
||||
* SOLR-6916: Toggle payload support for the default highlighter via hl.payloads. It's auto
|
||||
enabled when the index has payloads. (David Smiley)
|
||||
|
||||
* SOLR-6581: Efficient DocValues support and numeric collapse field implementations
|
||||
for Collapse and Expand (Joel Bernstein)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
|
|
|||
|
|
@ -24,11 +24,29 @@ import java.util.ArrayList;
|
|||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Iterator;
|
||||
import java.util.Arrays;
|
||||
|
||||
import com.carrotsearch.hppc.IntObjectOpenHashMap;
|
||||
import com.carrotsearch.hppc.LongOpenHashSet;
|
||||
import com.carrotsearch.hppc.LongObjectOpenHashMap;
|
||||
import com.carrotsearch.hppc.cursors.IntObjectCursor;
|
||||
import com.carrotsearch.hppc.cursors.LongCursor;
|
||||
import com.carrotsearch.hppc.cursors.LongObjectCursor;
|
||||
import com.carrotsearch.hppc.IntOpenHashSet;
|
||||
import com.carrotsearch.hppc.cursors.ObjectCursor;
|
||||
import com.carrotsearch.hppc.LongObjectMap;
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.DocValuesType;
|
||||
import org.apache.lucene.index.FieldInfo;
|
||||
import org.apache.lucene.index.FieldInfos;
|
||||
import org.apache.lucene.index.FilterLeafReader;
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.index.MultiDocValues;
|
||||
import org.apache.lucene.index.NumericDocValues;
|
||||
import org.apache.lucene.index.SortedDocValues;
|
||||
import org.apache.lucene.queries.TermsFilter;
|
||||
import org.apache.lucene.search.Collector;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.search.LeafCollector;
|
||||
|
|
@ -40,35 +58,37 @@ import org.apache.lucene.search.TopDocs;
|
|||
import org.apache.lucene.search.TopDocsCollector;
|
||||
import org.apache.lucene.search.TopFieldCollector;
|
||||
import org.apache.lucene.search.TopScoreDocCollector;
|
||||
import org.apache.lucene.uninverting.UninvertingReader;
|
||||
import org.apache.lucene.util.BitSetIterator;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.BytesRefBuilder;
|
||||
import org.apache.lucene.util.CharsRefBuilder;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.LongValues;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.params.ExpandParams;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.core.PluginInfo;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.TrieFloatField;
|
||||
import org.apache.solr.schema.TrieIntField;
|
||||
import org.apache.solr.schema.TrieLongField;
|
||||
import org.apache.solr.schema.TrieDoubleField;
|
||||
import org.apache.solr.schema.StrField;
|
||||
import org.apache.solr.search.CollapsingQParserPlugin;
|
||||
import org.apache.solr.search.DocIterator;
|
||||
import org.apache.solr.search.DocList;
|
||||
import org.apache.solr.search.DocSlice;
|
||||
import org.apache.solr.search.QParser;
|
||||
import org.apache.solr.search.QueryParsing;
|
||||
import org.apache.solr.search.SolrConstantScoreQuery;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.util.plugin.PluginInfoInitialized;
|
||||
import org.apache.solr.util.plugin.SolrCoreAware;
|
||||
|
||||
import com.carrotsearch.hppc.IntObjectMap;
|
||||
import com.carrotsearch.hppc.IntObjectOpenHashMap;
|
||||
import com.carrotsearch.hppc.IntOpenHashSet;
|
||||
import com.carrotsearch.hppc.cursors.IntObjectCursor;
|
||||
import com.carrotsearch.hppc.cursors.ObjectCursor;
|
||||
|
||||
/**
|
||||
* The ExpandComponent is designed to work with the CollapsingPostFilter.
|
||||
* The CollapsingPostFilter collapses a result set on a field.
|
||||
|
|
@ -118,6 +138,7 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
|
|||
SolrParams params = req.getParams();
|
||||
|
||||
String field = params.get(ExpandParams.EXPAND_FIELD);
|
||||
String hint = null;
|
||||
if (field == null) {
|
||||
List<Query> filters = rb.getFilters();
|
||||
if (filters != null) {
|
||||
|
|
@ -125,6 +146,7 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
|
|||
if (q instanceof CollapsingQParserPlugin.CollapsingPostFilter) {
|
||||
CollapsingQParserPlugin.CollapsingPostFilter cp = (CollapsingQParserPlugin.CollapsingPostFilter) q;
|
||||
field = cp.getField();
|
||||
hint = cp.hint;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -183,26 +205,168 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
|
|||
|
||||
SolrIndexSearcher searcher = req.getSearcher();
|
||||
LeafReader reader = searcher.getLeafReader();
|
||||
SortedDocValues values = DocValues.getSorted(reader, field);
|
||||
FixedBitSet groupBits = new FixedBitSet(values.getValueCount());
|
||||
|
||||
FieldType fieldType = searcher.getSchema().getField(field).getType();
|
||||
|
||||
SortedDocValues values = null;
|
||||
long nullValue = 0;
|
||||
|
||||
if(fieldType instanceof StrField) {
|
||||
//Get The Top Level SortedDocValues
|
||||
if(CollapsingQParserPlugin.HINT_TOP_FC.equals(hint)) {
|
||||
Map<String, UninvertingReader.Type> mapping = new HashMap();
|
||||
mapping.put(field, UninvertingReader.Type.SORTED);
|
||||
UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getLeafReader(), field), mapping);
|
||||
values = uninvertingReader.getSortedDocValues(field);
|
||||
} else {
|
||||
values = DocValues.getSorted(reader, field);
|
||||
}
|
||||
} else {
|
||||
//Get the nullValue for the numeric collapse field
|
||||
String defaultValue = searcher.getSchema().getField(field).getDefaultValue();
|
||||
if(defaultValue != null) {
|
||||
if(fieldType instanceof TrieIntField || fieldType instanceof TrieLongField) {
|
||||
nullValue = Long.parseLong(defaultValue);
|
||||
} else if(fieldType instanceof TrieFloatField){
|
||||
nullValue = Float.floatToIntBits(Float.parseFloat(defaultValue));
|
||||
} else if(fieldType instanceof TrieDoubleField){
|
||||
nullValue = Double.doubleToLongBits(Double.parseDouble(defaultValue));
|
||||
}
|
||||
} else {
|
||||
if(fieldType instanceof TrieFloatField){
|
||||
nullValue = Float.floatToIntBits(0.0f);
|
||||
} else if(fieldType instanceof TrieDoubleField){
|
||||
nullValue = Double.doubleToLongBits(0.0f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
FixedBitSet groupBits = null;
|
||||
LongOpenHashSet groupSet = null;
|
||||
DocList docList = rb.getResults().docList;
|
||||
IntOpenHashSet collapsedSet = new IntOpenHashSet(docList.size() * 2);
|
||||
|
||||
//Gather the groups for the current page of documents
|
||||
DocIterator idit = docList.iterator();
|
||||
|
||||
int[] globalDocs = new int[docList.size()];
|
||||
int docsIndex = -1;
|
||||
while (idit.hasNext()) {
|
||||
int doc = idit.nextDoc();
|
||||
int ord = values.getOrd(doc);
|
||||
if (ord > -1) {
|
||||
groupBits.set(ord);
|
||||
collapsedSet.add(doc);
|
||||
globalDocs[++docsIndex] = idit.nextDoc();
|
||||
}
|
||||
|
||||
Arrays.sort(globalDocs);
|
||||
Query groupQuery = null;
|
||||
|
||||
/*
|
||||
* This code gathers the group information for the current page.
|
||||
*/
|
||||
List<LeafReaderContext> contexts = searcher.getTopReaderContext().leaves();
|
||||
int currentContext = 0;
|
||||
int currentDocBase = contexts.get(currentContext).docBase;
|
||||
int nextDocBase = (currentContext+1)<contexts.size() ? contexts.get(currentContext+1).docBase : Integer.MAX_VALUE;
|
||||
IntObjectOpenHashMap<BytesRef> ordBytes = null;
|
||||
if(values != null) {
|
||||
groupBits = new FixedBitSet(values.getValueCount());
|
||||
MultiDocValues.OrdinalMap ordinalMap = null;
|
||||
SortedDocValues[] sortedDocValues = null;
|
||||
LongValues segmentOrdinalMap = null;
|
||||
SortedDocValues currentValues = null;
|
||||
if(values instanceof MultiDocValues.MultiSortedDocValues) {
|
||||
ordinalMap = ((MultiDocValues.MultiSortedDocValues)values).mapping;
|
||||
sortedDocValues = ((MultiDocValues.MultiSortedDocValues)values).values;
|
||||
currentValues = sortedDocValues[currentContext];
|
||||
segmentOrdinalMap = ordinalMap.getGlobalOrds(currentContext);
|
||||
}
|
||||
int count = 0;
|
||||
|
||||
ordBytes = new IntObjectOpenHashMap();
|
||||
|
||||
for(int i=0; i<globalDocs.length; i++) {
|
||||
int globalDoc = globalDocs[i];
|
||||
while(globalDoc >= nextDocBase) {
|
||||
currentContext++;
|
||||
currentDocBase = contexts.get(currentContext).docBase;
|
||||
nextDocBase = (currentContext+1) < contexts.size() ? contexts.get(currentContext+1).docBase : Integer.MAX_VALUE;
|
||||
if(ordinalMap != null) {
|
||||
currentValues = sortedDocValues[currentContext];
|
||||
segmentOrdinalMap = ordinalMap.getGlobalOrds(currentContext);
|
||||
}
|
||||
}
|
||||
|
||||
int contextDoc = globalDoc - currentDocBase;
|
||||
if(ordinalMap != null) {
|
||||
int ord = currentValues.getOrd(contextDoc);
|
||||
if(ord > -1) {
|
||||
++count;
|
||||
BytesRef ref = currentValues.lookupOrd(ord);
|
||||
ord = (int)segmentOrdinalMap.get(ord);
|
||||
ordBytes.put(ord, BytesRef.deepCopyOf(ref));
|
||||
groupBits.set(ord);
|
||||
collapsedSet.add(globalDoc);
|
||||
}
|
||||
} else {
|
||||
int ord = values.getOrd(globalDoc);
|
||||
if(ord > -1) {
|
||||
++count;
|
||||
BytesRef ref = values.lookupOrd(ord);
|
||||
ordBytes.put(ord, BytesRef.deepCopyOf(ref));
|
||||
groupBits.set(ord);
|
||||
collapsedSet.add(globalDoc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(count > 0 && count < 200) {
|
||||
try {
|
||||
groupQuery = getGroupQuery(field, count, ordBytes);
|
||||
} catch(Exception e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
groupSet = new LongOpenHashSet((int)(docList.size()*1.25));
|
||||
NumericDocValues collapseValues = contexts.get(currentContext).reader().getNumericDocValues(field);
|
||||
int count = 0;
|
||||
for(int i=0; i<globalDocs.length; i++) {
|
||||
int globalDoc = globalDocs[i];
|
||||
while(globalDoc >= nextDocBase) {
|
||||
currentContext++;
|
||||
currentDocBase = contexts.get(currentContext).docBase;
|
||||
nextDocBase = currentContext+1 < contexts.size() ? contexts.get(currentContext+1).docBase : Integer.MAX_VALUE;
|
||||
collapseValues = contexts.get(currentContext).reader().getNumericDocValues(field);
|
||||
}
|
||||
int contextDoc = globalDoc - currentDocBase;
|
||||
long value = collapseValues.get(contextDoc);
|
||||
if(value != nullValue) {
|
||||
++count;
|
||||
groupSet.add(value);
|
||||
collapsedSet.add(globalDoc);
|
||||
}
|
||||
}
|
||||
|
||||
if(count > 0 && count < 200) {
|
||||
groupQuery = getGroupQuery(field, fieldType, count, groupSet);
|
||||
}
|
||||
}
|
||||
|
||||
Collector collector;
|
||||
if (sort != null)
|
||||
sort = sort.rewrite(searcher);
|
||||
GroupExpandCollector groupExpandCollector = new GroupExpandCollector(values, groupBits, collapsedSet, limit, sort);
|
||||
|
||||
|
||||
Collector groupExpandCollector = null;
|
||||
|
||||
if(values != null) {
|
||||
groupExpandCollector = new GroupExpandCollector(values, groupBits, collapsedSet, limit, sort);
|
||||
} else {
|
||||
groupExpandCollector = new NumericGroupExpandCollector(field, nullValue, groupSet, collapsedSet, limit, sort);
|
||||
}
|
||||
|
||||
if(groupQuery != null) {
|
||||
//Limits the results to documents that are in the same group as the documents in the page.
|
||||
newFilters.add(groupQuery);
|
||||
}
|
||||
|
||||
SolrIndexSearcher.ProcessedFilter pfilter = searcher.getProcessedFilter(null, newFilters);
|
||||
if (pfilter.postFilter != null) {
|
||||
pfilter.postFilter.setLastDelegate(groupExpandCollector);
|
||||
|
|
@ -212,12 +376,11 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
|
|||
}
|
||||
|
||||
searcher.search(query, pfilter.filter, collector);
|
||||
IntObjectMap groups = groupExpandCollector.getGroups();
|
||||
LongObjectMap groups = ((GroupCollector)groupExpandCollector).getGroups();
|
||||
Map<String, DocSlice> outMap = new HashMap<>();
|
||||
CharsRefBuilder charsRef = new CharsRefBuilder();
|
||||
FieldType fieldType = searcher.getSchema().getField(field).getType();
|
||||
for (IntObjectCursor cursor : (Iterable<IntObjectCursor>) groups) {
|
||||
int ord = cursor.key;
|
||||
for (LongObjectCursor cursor : (Iterable<LongObjectCursor>) groups) {
|
||||
long groupValue = cursor.key;
|
||||
TopDocsCollector topDocsCollector = (TopDocsCollector) cursor.value;
|
||||
TopDocs topDocs = topDocsCollector.topDocs();
|
||||
ScoreDoc[] scoreDocs = topDocs.scoreDocs;
|
||||
|
|
@ -230,10 +393,21 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
|
|||
scores[i] = scoreDoc.score;
|
||||
}
|
||||
DocSlice slice = new DocSlice(0, docs.length, docs, scores, topDocs.totalHits, topDocs.getMaxScore());
|
||||
final BytesRef bytesRef = values.lookupOrd(ord);
|
||||
fieldType.indexedToReadable(bytesRef, charsRef);
|
||||
String group = charsRef.toString();
|
||||
outMap.put(group, slice);
|
||||
|
||||
if(fieldType instanceof StrField) {
|
||||
final BytesRef bytesRef = ordBytes.get((int)groupValue);
|
||||
fieldType.indexedToReadable(bytesRef, charsRef);
|
||||
String group = charsRef.toString();
|
||||
outMap.put(group, slice);
|
||||
} else {
|
||||
if(fieldType instanceof TrieIntField || fieldType instanceof TrieLongField ) {
|
||||
outMap.put(Long.toString(groupValue), slice);
|
||||
} else if(fieldType instanceof TrieFloatField) {
|
||||
outMap.put(Float.toString(Float.intBitsToFloat((int)groupValue)), slice);
|
||||
} else if(fieldType instanceof TrieDoubleField) {
|
||||
outMap.put(Double.toString(Double.longBitsToDouble(groupValue)), slice);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -306,16 +480,20 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
|
|||
rb.rsp.add("expanded", expanded);
|
||||
}
|
||||
|
||||
private class GroupExpandCollector implements Collector {
|
||||
private class GroupExpandCollector implements Collector, GroupCollector {
|
||||
private SortedDocValues docValues;
|
||||
private IntObjectMap<Collector> groups;
|
||||
private int docBase;
|
||||
private MultiDocValues.OrdinalMap ordinalMap;
|
||||
private SortedDocValues segmentValues;
|
||||
private LongValues segmentOrdinalMap;
|
||||
private MultiDocValues.MultiSortedDocValues multiSortedDocValues;
|
||||
|
||||
private LongObjectMap<Collector> groups;
|
||||
private FixedBitSet groupBits;
|
||||
private IntOpenHashSet collapsedSet;
|
||||
|
||||
public GroupExpandCollector(SortedDocValues docValues, FixedBitSet groupBits, IntOpenHashSet collapsedSet, int limit, Sort sort) throws IOException {
|
||||
int numGroups = collapsedSet.size();
|
||||
groups = new IntObjectOpenHashMap<>(numGroups * 2);
|
||||
groups = new LongObjectOpenHashMap<>(numGroups * 2);
|
||||
DocIdSetIterator iterator = new BitSetIterator(groupBits, 0); // cost is not useful here
|
||||
int group;
|
||||
while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
|
||||
|
|
@ -326,12 +504,22 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
|
|||
this.collapsedSet = collapsedSet;
|
||||
this.groupBits = groupBits;
|
||||
this.docValues = docValues;
|
||||
if(docValues instanceof MultiDocValues.MultiSortedDocValues) {
|
||||
this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)docValues;
|
||||
this.ordinalMap = multiSortedDocValues.mapping;
|
||||
}
|
||||
}
|
||||
|
||||
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
|
||||
final int docBase = context.docBase;
|
||||
final IntObjectMap<LeafCollector> leafCollectors = new IntObjectOpenHashMap<>();
|
||||
for (IntObjectCursor<Collector> entry : groups) {
|
||||
|
||||
if(ordinalMap != null) {
|
||||
this.segmentValues = this.multiSortedDocValues.values[context.ord];
|
||||
this.segmentOrdinalMap = ordinalMap.getGlobalOrds(context.ord);
|
||||
}
|
||||
|
||||
final LongObjectMap<LeafCollector> leafCollectors = new LongObjectOpenHashMap<>();
|
||||
for (LongObjectCursor<Collector> entry : groups) {
|
||||
leafCollectors.put(entry.key, entry.value.getLeafCollector(context));
|
||||
}
|
||||
return new LeafCollector() {
|
||||
|
|
@ -345,9 +533,18 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
|
|||
|
||||
@Override
|
||||
public void collect(int docId) throws IOException {
|
||||
int doc = docId + docBase;
|
||||
int ord = docValues.getOrd(doc);
|
||||
if (ord > -1 && groupBits.get(ord) && !collapsedSet.contains(doc)) {
|
||||
int globalDoc = docId + docBase;
|
||||
int ord = -1;
|
||||
if(ordinalMap != null) {
|
||||
ord = segmentValues.getOrd(docId);
|
||||
if(ord > -1) {
|
||||
ord = (int)segmentOrdinalMap.get(ord);
|
||||
}
|
||||
} else {
|
||||
ord = docValues.getOrd(globalDoc);
|
||||
}
|
||||
|
||||
if (ord > -1 && groupBits.get(ord) && !collapsedSet.contains(globalDoc)) {
|
||||
LeafCollector c = leafCollectors.get(ord);
|
||||
c.collect(docId);
|
||||
}
|
||||
|
|
@ -360,12 +557,125 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
|
|||
};
|
||||
}
|
||||
|
||||
public IntObjectMap<Collector> getGroups() {
|
||||
public LongObjectMap<Collector> getGroups() {
|
||||
return groups;
|
||||
}
|
||||
}
|
||||
|
||||
private class NumericGroupExpandCollector implements Collector, GroupCollector {
|
||||
private NumericDocValues docValues;
|
||||
|
||||
|
||||
private String field;
|
||||
private LongObjectOpenHashMap<Collector> groups;
|
||||
|
||||
private IntOpenHashSet collapsedSet;
|
||||
private long nullValue;
|
||||
|
||||
public NumericGroupExpandCollector(String field, long nullValue, LongOpenHashSet groupSet, IntOpenHashSet collapsedSet, int limit, Sort sort) throws IOException {
|
||||
int numGroups = collapsedSet.size();
|
||||
this.nullValue = nullValue;
|
||||
groups = new LongObjectOpenHashMap(numGroups * 2);
|
||||
Iterator<LongCursor> iterator = groupSet.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
LongCursor cursor = iterator.next();
|
||||
Collector collector = (sort == null) ? TopScoreDocCollector.create(limit, true) : TopFieldCollector.create(sort, limit, false, false, false, true);
|
||||
groups.put(cursor.value, collector);
|
||||
}
|
||||
|
||||
this.field = field;
|
||||
this.collapsedSet = collapsedSet;
|
||||
}
|
||||
|
||||
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
|
||||
final int docBase = context.docBase;
|
||||
this.docValues = context.reader().getNumericDocValues(this.field);
|
||||
|
||||
final LongObjectOpenHashMap<LeafCollector> leafCollectors = new LongObjectOpenHashMap<>();
|
||||
|
||||
for (LongObjectCursor<Collector> entry : groups) {
|
||||
leafCollectors.put(entry.key, entry.value.getLeafCollector(context));
|
||||
}
|
||||
|
||||
return new LeafCollector() {
|
||||
|
||||
@Override
|
||||
public void setScorer(Scorer scorer) throws IOException {
|
||||
for (ObjectCursor<LeafCollector> c : leafCollectors.values()) {
|
||||
c.value.setScorer(scorer);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void collect(int docId) throws IOException {
|
||||
long value = docValues.get(docId);
|
||||
if (value != nullValue && leafCollectors.containsKey(value) && !collapsedSet.contains(docId + docBase)) {
|
||||
LeafCollector c = leafCollectors.lget();
|
||||
c.collect(docId);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean acceptsDocsOutOfOrder() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public LongObjectOpenHashMap<Collector> getGroups() {
|
||||
return groups;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private interface GroupCollector {
|
||||
public LongObjectMap getGroups();
|
||||
|
||||
}
|
||||
|
||||
private Query getGroupQuery(String fname,
|
||||
FieldType ft,
|
||||
int size,
|
||||
LongOpenHashSet groupSet) {
|
||||
|
||||
BytesRef[] bytesRefs = new BytesRef[size];
|
||||
BytesRefBuilder term = new BytesRefBuilder();
|
||||
Iterator<LongCursor> it = groupSet.iterator();
|
||||
int index = -1;
|
||||
String stringVal = null;
|
||||
while (it.hasNext()) {
|
||||
LongCursor cursor = it.next();
|
||||
if(ft instanceof TrieIntField || ft instanceof TrieLongField) {
|
||||
stringVal = Long.toString(cursor.value);
|
||||
} else {
|
||||
if(ft instanceof TrieFloatField) {
|
||||
stringVal = Float.toString(Float.intBitsToFloat((int)cursor.value));
|
||||
} else {
|
||||
stringVal = Double.toString(Double.longBitsToDouble(cursor.value));
|
||||
}
|
||||
}
|
||||
ft.readableToIndexed(stringVal, term);
|
||||
bytesRefs[++index] = term.toBytesRef();
|
||||
}
|
||||
|
||||
return new SolrConstantScoreQuery(new TermsFilter(fname, bytesRefs));
|
||||
}
|
||||
|
||||
private Query getGroupQuery(String fname,
|
||||
int size,
|
||||
IntObjectOpenHashMap<BytesRef> ordBytes) throws Exception {
|
||||
|
||||
BytesRef[] bytesRefs = new BytesRef[size];
|
||||
int index = -1;
|
||||
Iterator<IntObjectCursor<BytesRef>>it = ordBytes.iterator();
|
||||
while (it.hasNext()) {
|
||||
IntObjectCursor<BytesRef> cursor = it.next();
|
||||
bytesRefs[++index] = cursor.value;
|
||||
}
|
||||
return new SolrConstantScoreQuery(new TermsFilter(fname, bytesRefs));
|
||||
}
|
||||
|
||||
|
||||
////////////////////////////////////////////
|
||||
/// SolrInfoMBean
|
||||
////////////////////////////////////////////
|
||||
|
|
@ -385,4 +695,49 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
|
|||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private class ReaderWrapper extends FilterLeafReader {
|
||||
|
||||
private String field;
|
||||
|
||||
public ReaderWrapper(LeafReader leafReader, String field) {
|
||||
super(leafReader);
|
||||
this.field = field;
|
||||
}
|
||||
|
||||
public SortedDocValues getSortedDocValues(String field) {
|
||||
return null;
|
||||
}
|
||||
|
||||
public Object getCoreCacheKey() {
|
||||
return in.getCoreCacheKey();
|
||||
}
|
||||
|
||||
public FieldInfos getFieldInfos() {
|
||||
Iterator<FieldInfo> it = in.getFieldInfos().iterator();
|
||||
List<FieldInfo> newInfos = new ArrayList();
|
||||
while(it.hasNext()) {
|
||||
FieldInfo fieldInfo = it.next();
|
||||
|
||||
if(fieldInfo.name.equals(field)) {
|
||||
FieldInfo f = new FieldInfo(fieldInfo.name,
|
||||
fieldInfo.number,
|
||||
fieldInfo.hasVectors(),
|
||||
fieldInfo.hasNorms(),
|
||||
fieldInfo.hasPayloads(),
|
||||
fieldInfo.getIndexOptions(),
|
||||
DocValuesType.NONE,
|
||||
fieldInfo.getDocValuesGen(),
|
||||
fieldInfo.attributes());
|
||||
newInfos.add(f);
|
||||
|
||||
} else {
|
||||
newInfos.add(fieldInfo);
|
||||
}
|
||||
}
|
||||
FieldInfos infos = new FieldInfos(newInfos.toArray(new FieldInfo[newInfos.size()]));
|
||||
return infos;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -19,6 +19,7 @@ package org.apache.solr.handler.component;
|
|||
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.search.CollapsingQParserPlugin;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
|
@ -43,14 +44,46 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
assertU(commit());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void testExpand() throws Exception {
|
||||
final String group = (random().nextBoolean() ? "group_s" : "group_s_dv");
|
||||
|
||||
String[] doc = {"id","1", "term_s", "YYYY", group, "group1", "test_ti", "5", "test_tl", "10", "test_tf", "2000", "type_s", "parent"};
|
||||
List<String> groups = new ArrayList();
|
||||
groups.add("group_s");
|
||||
groups.add("group_s_dv");
|
||||
|
||||
Collections.shuffle(groups, random());
|
||||
String floatAppend = "";
|
||||
|
||||
String hint = (random().nextBoolean() ? " hint="+ CollapsingQParserPlugin.HINT_TOP_FC : "");
|
||||
|
||||
_testExpand(groups.get(0), floatAppend, hint);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNumericExpand() throws Exception {
|
||||
List<String> groups = new ArrayList();
|
||||
groups.add("group_i");
|
||||
groups.add("group_ti_dv");
|
||||
groups.add("group_f");
|
||||
groups.add("group_tf_dv");
|
||||
Collections.shuffle(groups, random());
|
||||
String floatAppend = "";
|
||||
if(groups.get(0).indexOf("f") > -1) {
|
||||
floatAppend = "."+random().nextInt(100); //Append the float
|
||||
}
|
||||
|
||||
String hint = "";
|
||||
|
||||
_testExpand(groups.get(0), floatAppend, hint);
|
||||
}
|
||||
|
||||
private void _testExpand(String group, String floatAppend, String hint) throws Exception {
|
||||
|
||||
String[] doc = {"id","1", "term_s", "YYYY", group, "1"+floatAppend, "test_ti", "5", "test_tl", "10", "test_tf", "2000", "type_s", "parent"};
|
||||
assertU(adoc(doc));
|
||||
assertU(commit());
|
||||
String[] doc1 = {"id","2", "term_s","YYYY", group, "group1", "test_ti", "50", "test_tl", "100", "test_tf", "200", "type_s", "child"};
|
||||
String[] doc1 = {"id","2", "term_s","YYYY", group, "1"+floatAppend, "test_ti", "50", "test_tl", "100", "test_tf", "200", "type_s", "child"};
|
||||
assertU(adoc(doc1));
|
||||
|
||||
String[] doc2 = {"id","3", "term_s", "YYYY", "test_ti", "5000", "test_tl", "100", "test_tf", "200"};
|
||||
|
|
@ -60,17 +93,17 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
assertU(adoc(doc3));
|
||||
|
||||
|
||||
String[] doc4 = {"id","5", "term_s", "YYYY", group, "group2", "test_ti", "4", "test_tl", "10", "test_tf", "2000", "type_s", "parent"};
|
||||
String[] doc4 = {"id","5", "term_s", "YYYY", group, "2"+floatAppend, "test_ti", "4", "test_tl", "10", "test_tf", "2000", "type_s", "parent"};
|
||||
assertU(adoc(doc4));
|
||||
assertU(commit());
|
||||
String[] doc5 = {"id","6", "term_s","YYYY", group, "group2", "test_ti", "10", "test_tl", "100", "test_tf", "200", "type_s", "child"};
|
||||
String[] doc5 = {"id","6", "term_s","YYYY", group, "2"+floatAppend, "test_ti", "10", "test_tl", "100", "test_tf", "200", "type_s", "child"};
|
||||
assertU(adoc(doc5));
|
||||
assertU(commit());
|
||||
|
||||
String[] doc6 = {"id","7", "term_s", "YYYY", group, "group1", "test_ti", "1", "test_tl", "100000", "test_tf", "2000", "type_s", "child"};
|
||||
String[] doc6 = {"id","7", "term_s", "YYYY", group, "1"+floatAppend, "test_ti", "1", "test_tl", "100000", "test_tf", "2000", "type_s", "child"};
|
||||
assertU(adoc(doc6));
|
||||
assertU(commit());
|
||||
String[] doc7 = {"id","8", "term_s","YYYY", group, "group2", "test_ti", "2", "test_tl", "100000", "test_tf", "200", "type_s", "child"};
|
||||
String[] doc7 = {"id","8", "term_s","YYYY", group, "2"+floatAppend, "test_ti", "2", "test_tl", "100000", "test_tf", "200", "type_s", "child"};
|
||||
assertU(adoc(doc7));
|
||||
|
||||
assertU(commit());
|
||||
|
|
@ -78,7 +111,7 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
//First basic test case.
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+"}");
|
||||
params.add("fq", "{!collapse field="+group+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("expand", "true");
|
||||
|
|
@ -86,17 +119,17 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
"*[count(/response/lst[@name='expanded']/result)=2]",
|
||||
"/response/result/doc[1]/float[@name='id'][.='2.0']",
|
||||
"/response/result/doc[2]/float[@name='id'][.='6.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[1]/float[@name='id'][.='1.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[2]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='5.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='8.0']"
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='1.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='5.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='8.0']"
|
||||
);
|
||||
|
||||
//Basic test case page 2
|
||||
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+"}");
|
||||
params.add("fq", "{!collapse field="+group+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("expand", "true");
|
||||
|
|
@ -105,14 +138,14 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
assertQ(req(params), "*[count(/response/result/doc)=1]",
|
||||
"*[count(/response/lst[@name='expanded']/result)=1]",
|
||||
"/response/result/doc[1]/float[@name='id'][.='6.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='5.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='8.0']"
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='5.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='8.0']"
|
||||
);
|
||||
|
||||
//Test expand.sort
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+"}");
|
||||
params.add("fq", "{!collapse field="+group+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("expand", "true");
|
||||
|
|
@ -121,17 +154,17 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
"*[count(/response/lst[@name='expanded']/result)=2]",
|
||||
"/response/result/doc[1]/float[@name='id'][.='2.0']",
|
||||
"/response/result/doc[2]/float[@name='id'][.='6.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[1]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[2]/float[@name='id'][.='1.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='8.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='5.0']"
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='1.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='8.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='5.0']"
|
||||
);
|
||||
|
||||
//Test with nullPolicy, ExpandComponent should ignore docs with null values in the collapse fields.
|
||||
//Main result set should include the doc with null value in the collapse field.
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=collapse}");
|
||||
params.add("fq", "{!collapse field="+group+hint+" nullPolicy=collapse}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("expand", "true");
|
||||
|
|
@ -141,10 +174,10 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
"/response/result/doc[1]/float[@name='id'][.='3.0']",
|
||||
"/response/result/doc[2]/float[@name='id'][.='2.0']",
|
||||
"/response/result/doc[3]/float[@name='id'][.='6.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[1]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[2]/float[@name='id'][.='1.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='8.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='5.0']"
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='1.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='8.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='5.0']"
|
||||
);
|
||||
|
||||
|
||||
|
|
@ -162,10 +195,10 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
"*[count(/response/lst[@name='expanded']/result)=2]",
|
||||
"/response/result/doc[1]/float[@name='id'][.='1.0']",
|
||||
"/response/result/doc[2]/float[@name='id'][.='5.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[1]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[2]/float[@name='id'][.='2.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='8.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='6.0']"
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='2.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='8.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='6.0']"
|
||||
);
|
||||
|
||||
|
||||
|
|
@ -184,10 +217,10 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
"*[count(/response/lst[@name='expanded']/result)=2]",
|
||||
"/response/result/doc[1]/float[@name='id'][.='1.0']",
|
||||
"/response/result/doc[2]/float[@name='id'][.='5.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[1]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[2]/float[@name='id'][.='2.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='8.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='6.0']"
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='2.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='8.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='6.0']"
|
||||
);
|
||||
|
||||
//Test overide expand.fq and expand.q
|
||||
|
|
@ -206,17 +239,17 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
"*[count(/response/lst[@name='expanded']/result)=2]",
|
||||
"/response/result/doc[1]/float[@name='id'][.='1.0']",
|
||||
"/response/result/doc[2]/float[@name='id'][.='5.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[1]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[2]/float[@name='id'][.='2.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='8.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='6.0']"
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='2.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='8.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='6.0']"
|
||||
);
|
||||
|
||||
//Test expand.rows
|
||||
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+"}");
|
||||
params.add("fq", "{!collapse field="+group+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("expand", "true");
|
||||
|
|
@ -224,12 +257,12 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
params.add("expand.rows", "1");
|
||||
assertQ(req(params), "*[count(/response/result/doc)=2]",
|
||||
"*[count(/response/lst[@name='expanded']/result)=2]",
|
||||
"*[count(/response/lst[@name='expanded']/result[@name='group1']/doc)=1]",
|
||||
"*[count(/response/lst[@name='expanded']/result[@name='group2']/doc)=1]",
|
||||
"*[count(/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc)=1]",
|
||||
"*[count(/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc)=1]",
|
||||
"/response/result/doc[1]/float[@name='id'][.='2.0']",
|
||||
"/response/result/doc[2]/float[@name='id'][.='6.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[1]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='8.0']"
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='8.0']"
|
||||
);
|
||||
|
||||
|
||||
|
|
@ -237,7 +270,7 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "test_ti:5");
|
||||
params.add("fq", "{!collapse field="+group+"}");
|
||||
params.add("fq", "{!collapse field="+group+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("expand", "true");
|
||||
|
|
@ -251,7 +284,7 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "test_ti:5532535");
|
||||
params.add("fq", "{!collapse field="+group+"}");
|
||||
params.add("fq", "{!collapse field="+group+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("expand", "true");
|
||||
|
|
@ -265,7 +298,7 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+"}");
|
||||
params.add("fq", "{!collapse field="+group+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("expand", "true");
|
||||
|
|
@ -274,10 +307,10 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
|
|||
"*[count(/response/lst[@name='expanded']/result)=2]",
|
||||
"/response/result/doc[1]/float[@name='id'][.='2.0']",
|
||||
"/response/result/doc[2]/float[@name='id'][.='6.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[1]/float[@name='id'][.='1.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group1']/doc[2]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='5.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='8.0']"
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='1.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='7.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='5.0']",
|
||||
"/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='8.0']"
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,11 @@
|
|||
|
||||
package org.apache.solr.search;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
|
@ -45,13 +50,38 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testCollapseQueries() throws Exception {
|
||||
final String group = (random().nextBoolean() ? "group_s" : "group_s_dv");
|
||||
|
||||
String[] doc = {"id","1", "term_s", "YYYY", group, "group1", "test_ti", "5", "test_tl", "10", "test_tf", "2000"};
|
||||
public void testStringCollapse() throws Exception {
|
||||
List<String> types = new ArrayList();
|
||||
types.add("group_s");
|
||||
types.add("group_s_dv");
|
||||
Collections.shuffle(types, random());
|
||||
String group = types.get(0);
|
||||
String hint = (random().nextBoolean() ? " hint="+CollapsingQParserPlugin.HINT_TOP_FC : "");
|
||||
testCollapseQueries(group, hint, false);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testNumericCollapse() throws Exception {
|
||||
List<String> types = new ArrayList();
|
||||
types.add("group_i");
|
||||
types.add("group_ti_dv");
|
||||
types.add("group_f");
|
||||
types.add("group_tf_dv");
|
||||
Collections.shuffle(types, random());
|
||||
String group = types.get(0);
|
||||
String hint = "";
|
||||
testCollapseQueries(group, hint, true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void testCollapseQueries(String group, String hint, boolean numeric) throws Exception {
|
||||
|
||||
String[] doc = {"id","1", "term_s", "YYYY", group, "1", "test_ti", "5", "test_tl", "10", "test_tf", "2000"};
|
||||
assertU(adoc(doc));
|
||||
assertU(commit());
|
||||
String[] doc1 = {"id","2", "term_s","YYYY", group, "group1", "test_ti", "50", "test_tl", "100", "test_tf", "200"};
|
||||
String[] doc1 = {"id","2", "term_s","YYYY", group, "1", "test_ti", "50", "test_tl", "100", "test_tf", "200"};
|
||||
assertU(adoc(doc1));
|
||||
|
||||
|
||||
|
|
@ -63,19 +93,25 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
assertU(adoc(doc3));
|
||||
|
||||
|
||||
String[] doc4 = {"id","5", "term_s", "YYYY", group, "group2", "test_ti", "4", "test_tl", "10", "test_tf", "2000"};
|
||||
String[] doc4 = {"id","5", "term_s", "YYYY", group, "2", "test_ti", "4", "test_tl", "10", "test_tf", "2000"};
|
||||
assertU(adoc(doc4));
|
||||
assertU(commit());
|
||||
String[] doc5 = {"id","6", "term_s","YYYY", group, "group2", "test_ti", "10", "test_tl", "100", "test_tf", "200"};
|
||||
String[] doc5 = {"id","6", "term_s","YYYY", group, "2", "test_ti", "10", "test_tl", "100", "test_tf", "200"};
|
||||
assertU(adoc(doc5));
|
||||
assertU(commit());
|
||||
|
||||
String[] doc6 = {"id","7", "term_s", "YYYY", group, "1", "test_ti", "8", "test_tl", "50", "test_tf", "300"};
|
||||
assertU(adoc(doc6));
|
||||
assertU(commit());
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
//Test collapse by score and following sort by score
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+"}");
|
||||
params.add("fq", "{!collapse field="+group+""+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
assertQ(req(params, "indent", "on"), "*[count(//doc)=2]",
|
||||
|
|
@ -87,7 +123,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
// SOLR-5544 test ordering with empty sort param
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=expand min=test_tf}");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=expand min=test_tf"+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("sort","");
|
||||
|
|
@ -101,7 +137,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
// Test value source collapse criteria
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=collapse min=field(test_ti)}");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=collapse min=field(test_ti)"+hint+"}");
|
||||
params.add("sort", "test_ti desc");
|
||||
assertQ(req(params), "*[count(//doc)=3]",
|
||||
"//result/doc[1]/float[@name='id'][.='4.0']",
|
||||
|
|
@ -112,7 +148,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
// Test value source collapse criteria with cscore function
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=collapse min=cscore()}");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=collapse min=cscore()"+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
assertQ(req(params), "*[count(//doc)=3]",
|
||||
|
|
@ -124,7 +160,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
// Test value source collapse criteria with compound cscore function
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=collapse min=sum(cscore(),field(test_ti))}");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=collapse min=sum(cscore(),field(test_ti))"+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
assertQ(req(params), "*[count(//doc)=3]",
|
||||
|
|
@ -137,7 +173,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "YYYY");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=collapse}");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=collapse"+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("qf", "term_s");
|
||||
|
|
@ -151,7 +187,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
//Test SOLR-5773 with score collapse criteria
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "YYYY");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=collapse}");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=collapse"+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("qf", "term_s");
|
||||
|
|
@ -165,7 +201,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
//Test SOLR-5773 with max field collapse criteria
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "YYYY");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_ti nullPolicy=collapse}");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_ti nullPolicy=collapse"+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("qf", "term_s");
|
||||
|
|
@ -180,7 +216,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
//Test SOLR-5773 elevating documents with null group
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "YYYY");
|
||||
params.add("fq", "{!collapse field="+group+"}");
|
||||
params.add("fq", "{!collapse field="+group+""+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("qf", "term_s");
|
||||
|
|
@ -197,7 +233,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
//Test collapse by min int field and sort
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_ti}");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_ti"+hint+"}");
|
||||
params.add("sort", "id desc");
|
||||
assertQ(req(params), "*[count(//doc)=2]",
|
||||
"//result/doc[1]/float[@name='id'][.='5.0']",
|
||||
|
|
@ -205,7 +241,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_ti}");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_ti"+hint+"}");
|
||||
params.add("sort", "id asc");
|
||||
assertQ(req(params), "*[count(//doc)=2]",
|
||||
"//result/doc[1]/float[@name='id'][.='1.0']",
|
||||
|
|
@ -213,15 +249,17 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_ti}");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_ti"+hint+"}");
|
||||
params.add("sort", "test_tl asc,id desc");
|
||||
assertQ(req(params), "*[count(//doc)=2]",
|
||||
"//result/doc[1]/float[@name='id'][.='5.0']",
|
||||
"//result/doc[2]/float[@name='id'][.='1.0']");
|
||||
|
||||
|
||||
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_ti}");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_ti"+hint+"}");
|
||||
params.add("sort", "score desc,id asc");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(id)");
|
||||
|
|
@ -235,39 +273,43 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
//Test collapse by max int field
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" max=test_ti}");
|
||||
params.add("fq", "{!collapse field="+group+" max=test_ti"+hint+"}");
|
||||
params.add("sort", "test_ti asc");
|
||||
assertQ(req(params), "*[count(//doc)=2]",
|
||||
"//result/doc[1]/float[@name='id'][.='6.0']",
|
||||
"//result/doc[2]/float[@name='id'][.='2.0']"
|
||||
);
|
||||
|
||||
try {
|
||||
//Test collapse by min long field
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_tl"+hint+"}");
|
||||
params.add("sort", "test_ti desc");
|
||||
assertQ(req(params), "*[count(//doc)=2]",
|
||||
"//result/doc[1]/float[@name='id'][.='1.0']",
|
||||
"//result/doc[2]/float[@name='id'][.='5.0']");
|
||||
|
||||
|
||||
//Test collapse by min long field
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_tl}");
|
||||
params.add("sort", "test_ti desc");
|
||||
assertQ(req(params), "*[count(//doc)=2]",
|
||||
"//result/doc[1]/float[@name='id'][.='1.0']",
|
||||
"//result/doc[2]/float[@name='id'][.='5.0']");
|
||||
|
||||
|
||||
//Test collapse by max long field
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" max=test_tl}");
|
||||
params.add("sort", "test_ti desc");
|
||||
assertQ(req(params), "*[count(//doc)=2]",
|
||||
"//result/doc[1]/float[@name='id'][.='2.0']",
|
||||
"//result/doc[2]/float[@name='id'][.='6.0']");
|
||||
//Test collapse by max long field
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" max=test_tl"+hint+"}");
|
||||
params.add("sort", "test_ti desc");
|
||||
assertQ(req(params), "*[count(//doc)=2]",
|
||||
"//result/doc[1]/float[@name='id'][.='2.0']",
|
||||
"//result/doc[2]/float[@name='id'][.='6.0']");
|
||||
} catch (Exception e) {
|
||||
if(!numeric) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//Test collapse by min float field
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_tf}");
|
||||
params.add("fq", "{!collapse field="+group+" min=test_tf"+hint+"}");
|
||||
params.add("sort", "test_ti desc");
|
||||
assertQ(req(params), "*[count(//doc)=2]",
|
||||
"//result/doc[1]/float[@name='id'][.='2.0']",
|
||||
|
|
@ -279,7 +321,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
//Test collapse by min float field
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" max=test_tf}");
|
||||
params.add("fq", "{!collapse field="+group+" max=test_tf"+hint+"}");
|
||||
params.add("sort", "test_ti asc");
|
||||
assertQ(req(params), "*[count(//doc)=2]",
|
||||
"//result/doc[1]/float[@name='id'][.='5.0']",
|
||||
|
|
@ -288,7 +330,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
//Test collapse by min float field sort by score
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" max=test_tf}");
|
||||
params.add("fq", "{!collapse field="+group+" max=test_tf"+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(id)");
|
||||
params.add("fl", "score, id");
|
||||
|
|
@ -304,7 +346,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
//Test nullPolicy expand
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" max=test_tf nullPolicy=expand}");
|
||||
params.add("fq", "{!collapse field="+group+" max=test_tf nullPolicy=expand"+hint+"}");
|
||||
params.add("sort", "id desc");
|
||||
assertQ(req(params), "*[count(//doc)=4]",
|
||||
"//result/doc[1]/float[@name='id'][.='5.0']",
|
||||
|
|
@ -316,7 +358,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+" max=test_tf nullPolicy=collapse}");
|
||||
params.add("fq", "{!collapse field="+group+" max=test_tf nullPolicy=collapse"+hint+"}");
|
||||
params.add("sort", "id desc");
|
||||
assertQ(req(params), "*[count(//doc)=3]",
|
||||
"//result/doc[1]/float[@name='id'][.='5.0']",
|
||||
|
|
@ -326,7 +368,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+"}");
|
||||
params.add("fq", "{!collapse field="+group+hint+"}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("fq","{!tag=test_ti}id:5");
|
||||
|
|
@ -338,7 +380,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
// SOLR-5230 - ensure CollapsingFieldValueCollector.finish() is called
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "*:*");
|
||||
params.add("fq", "{!collapse field="+group+"}");
|
||||
params.add("fq", "{!collapse field="+group+hint+"}");
|
||||
params.add("group", "true");
|
||||
params.add("group.field", "id");
|
||||
assertQ(req(params), "*[count(//doc)=2]");
|
||||
|
|
@ -350,14 +392,15 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
assertU(commit());
|
||||
params = new ModifiableSolrParams();
|
||||
params.add("q", "YYYY");
|
||||
params.add("fq", "{!collapse field="+group+" nullPolicy=collapse}");
|
||||
params.add("fq", "{!collapse field="+group+hint+" nullPolicy=collapse}");
|
||||
params.add("defType", "edismax");
|
||||
params.add("bf", "field(test_ti)");
|
||||
params.add("qf", "term_s");
|
||||
params.add("qt", "/elevate");
|
||||
assertQ(req(params), "*[count(//doc)=2]",
|
||||
assertQ(req(params), "*[count(//doc)=3]",
|
||||
"//result/doc[1]/float[@name='id'][.='3.0']",
|
||||
"//result/doc[2]/float[@name='id'][.='6.0']");
|
||||
"//result/doc[2]/float[@name='id'][.='6.0']",
|
||||
"//result/doc[3]/float[@name='id'][.='7.0']");
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue