Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make use of the diamond operator in grouping module. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1467946 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Martijn van Groningen 2013-04-15 11:50:08 +00:00
parent 5de5540557
commit be363e6d1c
20 changed files with 248 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractDistinctValuesCollector.java
View File

@ -57,7 +57,7 @@ public abstract class AbstractDistinctValuesCollector<GC extends AbstractDistinc
public GroupCount(GROUP_VALUE_TYPE groupValue) {
this.groupValue = groupValue;
this.uniqueValues = new HashSet<GROUP_VALUE_TYPE>();
this.uniqueValues = new HashSet<>();
}
}

10
lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java
View File

@ -85,7 +85,7 @@ abstract public class AbstractFirstPassGroupingCollector<GROUP_VALUE_TYPE> exten
}
spareSlot = topNGroups;
groupMap = new HashMap<GROUP_VALUE_TYPE, CollectedSearchGroup<GROUP_VALUE_TYPE>>(topNGroups);
groupMap = new HashMap<>(topNGroups);
}
/**
@ -113,7 +113,7 @@ abstract public class AbstractFirstPassGroupingCollector<GROUP_VALUE_TYPE> exten
buildSortedSet();
}
final Collection<SearchGroup<GROUP_VALUE_TYPE>> result = new ArrayList<SearchGroup<GROUP_VALUE_TYPE>>();
final Collection<SearchGroup<GROUP_VALUE_TYPE>> result = new ArrayList<>();
int upto = 0;
final int sortFieldCount = groupSort.getSort().length;
for(CollectedSearchGroup<GROUP_VALUE_TYPE> group : orderedGroups) {
@ -121,7 +121,7 @@ abstract public class AbstractFirstPassGroupingCollector<GROUP_VALUE_TYPE> exten
continue;
}
//System.out.println(" group=" + (group.groupValue == null ? "null" : group.groupValue.utf8ToString()));
SearchGroup<GROUP_VALUE_TYPE> searchGroup = new SearchGroup<GROUP_VALUE_TYPE>();
SearchGroup<GROUP_VALUE_TYPE> searchGroup = new SearchGroup<>();
searchGroup.groupValue = group.groupValue;
if (fillFields) {
searchGroup.sortValues = new Object[sortFieldCount];
@ -193,7 +193,7 @@ abstract public class AbstractFirstPassGroupingCollector<GROUP_VALUE_TYPE> exten
// just keep collecting them
// Add a new CollectedSearchGroup:
CollectedSearchGroup<GROUP_VALUE_TYPE> sg = new CollectedSearchGroup<GROUP_VALUE_TYPE>();
CollectedSearchGroup<GROUP_VALUE_TYPE> sg = new CollectedSearchGroup<>();
sg.groupValue = copyDocGroupValue(groupValue, null);
sg.comparatorSlot = groupMap.size();
sg.topDoc = docBase + doc;
@ -311,7 +311,7 @@ abstract public class AbstractFirstPassGroupingCollector<GROUP_VALUE_TYPE> exten
}
};
orderedGroups = new TreeSet<CollectedSearchGroup<GROUP_VALUE_TYPE>>(comparator);
orderedGroups = new TreeSet<>(comparator);
orderedGroups.addAll(groupMap.values());
assert orderedGroups.size() > 0;

6
lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractGroupFacetCollector.java
View File

@ -46,7 +46,7 @@ public abstract class AbstractGroupFacetCollector extends Collector {
this.groupField = groupField;
this.facetField = facetField;
this.facetPrefix = facetPrefix;
segmentResults = new ArrayList<SegmentResult>();
segmentResults = new ArrayList<>();
}
/**
@ -148,7 +148,7 @@ public abstract class AbstractGroupFacetCollector extends Collector {
private int currentMin;
public GroupedFacetResult(int size, int minCount, boolean orderByCount, int totalCount, int totalMissingCount) {
this.facetEntries = new TreeSet<FacetEntry>(orderByCount ? orderByCountAndValue : orderByValue);
this.facetEntries = new TreeSet<>(orderByCount ? orderByCountAndValue : orderByValue);
this.totalMissingCount = totalMissingCount;
this.totalCount = totalCount;
maxSize = size;
@ -183,7 +183,7 @@ public abstract class AbstractGroupFacetCollector extends Collector {
* @return a list of facet entries to be rendered based on the specified offset and limit
*/
public List<FacetEntry> getFacetEntries(int offset, int limit) {
List<FacetEntry> entries = new LinkedList<FacetEntry>();
List<FacetEntry> entries = new LinkedList<>();
int skipped = 0;
int included = 0;

8
lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractSecondPassGroupingCollector.java
View File

@ -62,7 +62,7 @@ public abstract class AbstractSecondPassGroupingCollector<GROUP_VALUE_TYPE> exte
this.withinGroupSort = withinGroupSort;
this.groups = groups;
this.maxDocsPerGroup = maxDocsPerGroup;
groupMap = new HashMap<GROUP_VALUE_TYPE, SearchGroupDocs<GROUP_VALUE_TYPE>>(groups.size());
groupMap = new HashMap<>(groups.size());
for (SearchGroup<GROUP_VALUE_TYPE> group : groups) {
//System.out.println(" prep group=" + (group.groupValue == null ? "null" : group.groupValue.utf8ToString()));
@ -75,7 +75,7 @@ public abstract class AbstractSecondPassGroupingCollector<GROUP_VALUE_TYPE> exte
collector = TopFieldCollector.create(withinGroupSort, maxDocsPerGroup, fillSortFields, getScores, getMaxScores, true);
}
groupMap.put(group.groupValue,
new SearchGroupDocs<GROUP_VALUE_TYPE>(group.groupValue,
new SearchGroupDocs<>(group.groupValue,
collector));
}
}
@ -128,7 +128,7 @@ public abstract class AbstractSecondPassGroupingCollector<GROUP_VALUE_TYPE> exte
for(SearchGroup<?> group : groups) {
final SearchGroupDocs<GROUP_VALUE_TYPE> groupDocs = groupMap.get(group.groupValue);
final TopDocs topDocs = groupDocs.collector.topDocs(withinGroupOffset, maxDocsPerGroup);
groupDocsResult[groupIDX++] = new GroupDocs<GROUP_VALUE_TYPE>(Float.NaN,
groupDocsResult[groupIDX++] = new GroupDocs<>(Float.NaN,
topDocs.getMaxScore(),
topDocs.totalHits,
topDocs.scoreDocs,
@ -137,7 +137,7 @@ public abstract class AbstractSecondPassGroupingCollector<GROUP_VALUE_TYPE> exte
maxScore = Math.max(maxScore, topDocs.getMaxScore());
}
return new TopGroups<GROUP_VALUE_TYPE>(groupSort.getSort(),
return new TopGroups<>(groupSort.getSort(),
withinGroupSort == null ? null : withinGroupSort.getSort(),
totalHitCount, totalGroupedHitCount, groupDocsResult,
maxScore);

4
lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java
View File

@ -365,7 +365,7 @@ public class BlockGroupingCollector extends Collector {
// TODO: we could aggregate scores across children
// by Sum/Avg instead of passing NaN:
groups[downTo] = new GroupDocs<Object>(Float.NaN,
groups[downTo] = new GroupDocs<>(Float.NaN,
topDocs.getMaxScore(),
og.count,
topDocs.scoreDocs,
@ -382,7 +382,7 @@ public class BlockGroupingCollector extends Collector {
}
*/
return new TopGroups<Object>(new TopGroups<Object>(groupSort.getSort(),
return new TopGroups<>(new TopGroups<>(groupSort.getSort(),
withinGroupSort == null ? null : withinGroupSort.getSort(),
totalHitCount, totalGroupedHitCount, groups, maxScore),
totalGroupCount);

22
lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java
View File

@ -17,11 +17,16 @@ package org.apache.lucene.search.grouping;
* limitations under the License.
*/
import java.io.IOException;
import java.util.*;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.*;
import org.apache.lucene.search.CachingCollector;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MultiCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.grouping.function.FunctionAllGroupHeadsCollector;
import org.apache.lucene.search.grouping.function.FunctionAllGroupsCollector;
import org.apache.lucene.search.grouping.function.FunctionFirstPassGroupingCollector;
@ -34,6 +39,13 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.mutable.MutableValue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
/**
* Convenience class to perform grouping in a non distributed environment.
*
@ -173,7 +185,7 @@ public class GroupingSearch {
final Collector firstRound;
if (allGroupHeads || allGroups) {
List<Collector> collectors = new ArrayList<Collector>();
List<Collector> collectors = new ArrayList<>();
collectors.add(firstPassCollector);
if (allGroups) {
collectors.add(allGroupsCollector);

6
lucene/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java
View File

@ -198,9 +198,9 @@ public class SearchGroup<GROUP_VALUE_TYPE> {
private final Map<T,MergedGroup<T>> groupsSeen;
public GroupMerger(Sort groupSort) throws IOException {
groupComp = new GroupComparator<T>(groupSort);
queue = new TreeSet<MergedGroup<T>>(groupComp);
groupsSeen = new HashMap<T,MergedGroup<T>>();
groupComp = new GroupComparator<>(groupSort);
queue = new TreeSet<>(groupComp);
groupsSeen = new HashMap<>();
}
@SuppressWarnings({"unchecked","rawtypes"})

10
lucene/grouping/src/java/org/apache/lucene/search/grouping/TopGroups.java
View File

@ -78,7 +78,7 @@ public class TopGroups<GROUP_VALUE_TYPE> {
Total,
/* Avg score across all shards for this group. */
Avg,
};
}
/** Merges an array of TopGroups, for example obtained
* from the second-pass collector across multiple
@ -202,7 +202,7 @@ public class TopGroups<GROUP_VALUE_TYPE> {
}
//System.out.println("SHARDS=" + Arrays.toString(mergedTopDocs.shardIndex));
mergedGroupDocs[groupIDX] = new GroupDocs<T>(groupScore,
mergedGroupDocs[groupIDX] = new GroupDocs<>(groupScore,
maxScore,
totalHits,
mergedScoreDocs,
@ -212,15 +212,15 @@ public class TopGroups<GROUP_VALUE_TYPE> {
}
if (totalGroupCount != null) {
TopGroups<T> result = new TopGroups<T>(groupSort.getSort(),
TopGroups<T> result = new TopGroups<>(groupSort.getSort(),
docSort == null ? null : docSort.getSort(),
totalHitCount,
totalGroupedHitCount,
mergedGroupDocs,
totalMaxScore);
return new TopGroups<T>(result, totalGroupCount);
return new TopGroups<>(result, totalGroupCount);
} else {
return new TopGroups<T>(groupSort.getSort(),
return new TopGroups<>(groupSort.getSort(),
docSort == null ? null : docSort.getSort(),
totalHitCount,
totalGroupedHitCount,

2
lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionAllGroupHeadsCollector.java
View File

@ -59,7 +59,7 @@ public class FunctionAllGroupHeadsCollector extends AbstractAllGroupHeadsCollect
*/
public FunctionAllGroupHeadsCollector(ValueSource groupBy, Map<?, ?> vsContext, Sort sortWithinGroup) {
super(sortWithinGroup.getSort().length);
groups = new HashMap<MutableValue, GroupHead>();
groups = new HashMap<>();
this.sortWithinGroup = sortWithinGroup;
this.groupBy = groupBy;
this.vsContext = vsContext;

4
lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionDistinctValuesCollector.java
View File

@ -48,7 +48,7 @@ public class FunctionDistinctValuesCollector extends AbstractDistinctValuesColle
this.vsContext = vsContext;
this.groupSource = groupSource;
this.countSource = countSource;
groupMap = new LinkedHashMap<MutableValue, GroupCount>();
groupMap = new LinkedHashMap<>();
for (SearchGroup<MutableValue> group : groups) {
groupMap.put(group.groupValue, new GroupCount(group.groupValue));
}
@ -56,7 +56,7 @@ public class FunctionDistinctValuesCollector extends AbstractDistinctValuesColle
@Override
public List<GroupCount> getGroups() {
return new ArrayList<GroupCount>(groupMap.values());
return new ArrayList<>(groupMap.values());
}
@Override

24
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
View File

@ -17,16 +17,24 @@ package org.apache.lucene.search.grouping.term;
* limitations under the License.
*/
import java.io.IOException;
import java.util.*;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.*;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* A base implementation of {@link org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector} for retrieving the most relevant groups when grouping
* on a string based group field. More specifically this all concrete implementations of this base implementation
@ -115,7 +123,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
GeneralAllGroupHeadsCollector(String groupField, Sort sortWithinGroup) {
super(groupField, sortWithinGroup.getSort().length);
this.sortWithinGroup = sortWithinGroup;
groups = new HashMap<BytesRef, GroupHead>();
groups = new HashMap<>();
final SortField[] sortFields = sortWithinGroup.getSort();
for (int i = 0; i < sortFields.length; i++) {
@ -219,7 +227,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
OrdScoreAllGroupHeadsCollector(String groupField, Sort sortWithinGroup, int initialSize) {
super(groupField, sortWithinGroup.getSort().length);
ordSet = new SentinelIntSet(initialSize, -2);
collectedGroups = new ArrayList<GroupHead>(initialSize);
collectedGroups = new ArrayList<>(initialSize);
final SortField[] sortFields = sortWithinGroup.getSort();
fields = new SortField[sortFields.length];
@ -388,7 +396,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
OrdAllGroupHeadsCollector(String groupField, Sort sortWithinGroup, int initialSize) {
super(groupField, sortWithinGroup.getSort().length);
ordSet = new SentinelIntSet(initialSize, -2);
collectedGroups = new ArrayList<GroupHead>(initialSize);
collectedGroups = new ArrayList<>(initialSize);
final SortField[] sortFields = sortWithinGroup.getSort();
fields = new SortField[sortFields.length];
@ -531,7 +539,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
ScoreAllGroupHeadsCollector(String groupField, Sort sortWithinGroup, int initialSize) {
super(groupField, sortWithinGroup.getSort().length);
ordSet = new SentinelIntSet(initialSize, -2);
collectedGroups = new ArrayList<GroupHead>(initialSize);
collectedGroups = new ArrayList<>(initialSize);
final SortField[] sortFields = sortWithinGroup.getSort();
fields = new SortField[sortFields.length];

12
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
View File

@ -17,11 +17,6 @@ package org.apache.lucene.search.grouping.term;
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache;
@ -29,6 +24,11 @@ import org.apache.lucene.search.grouping.AbstractAllGroupsCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/**
* A collector that collects all groups that match the
* query. Only the group value is collected, and the order
@ -66,7 +66,7 @@ public class TermAllGroupsCollector extends AbstractAllGroupsCollector<BytesRef>
*/
public TermAllGroupsCollector(String groupField, int initialSize) {
ordSet = new SentinelIntSet(initialSize, -2);
groups = new ArrayList<BytesRef>(initialSize);
groups = new ArrayList<>(initialSize);
this.groupField = groupField;
}

11
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java
View File

@ -17,9 +17,6 @@ package org.apache.lucene.search.grouping.term;
* limitations under the License.
*/
import java.io.IOException;
import java.util.*;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache;
@ -28,6 +25,12 @@ import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
/**
* A term based implementation of {@link org.apache.lucene.search.grouping.AbstractDistinctValuesCollector} that relies
* on {@link SortedDocValues} to count the distinct values per group.
@ -55,7 +58,7 @@ public class TermDistinctValuesCollector extends AbstractDistinctValuesCollector
public TermDistinctValuesCollector(String groupField, String countField, Collection<SearchGroup<BytesRef>> groups) {
this.groupField = groupField;
this.countField = countField;
this.groups = new ArrayList<GroupCount>(groups.size());
this.groups = new ArrayList<>(groups.size());
for (SearchGroup<BytesRef> group : groups) {
this.groups.add(new GroupCount(group.groupValue));
}

15
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
View File

@ -17,18 +17,19 @@ package org.apache.lucene.search.grouping.term;
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.grouping.AbstractGroupFacetCollector;
import org.apache.lucene.util.*;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
import org.apache.lucene.util.UnicodeUtil;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* An implementation of {@link AbstractGroupFacetCollector} that computes grouped facets based on the indexed terms
@ -70,7 +71,7 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
TermGroupFacetCollector(String groupField, String facetField, BytesRef facetPrefix, int initialSize) {
super(groupField, facetField, facetPrefix);
groupedFacetHits = new ArrayList<GroupedFacetHit>(initialSize);
groupedFacetHits = new ArrayList<>(initialSize);
segmentGroupedFacetHits = new SentinelIntSet(initialSize, Integer.MIN_VALUE);
}

69
lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
View File

@ -17,11 +17,13 @@ package org.apache.lucene.search.grouping;
* limitations under the License.
*/
import java.io.IOException;
import java.util.*;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.IndexReader;
@ -30,7 +32,14 @@ import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.search.*;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.grouping.function.FunctionAllGroupHeadsCollector;
import org.apache.lucene.search.grouping.term.TermAllGroupHeadsCollector;
import org.apache.lucene.store.Directory;
@ -39,6 +48,16 @@ import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
public class AllGroupHeadsCollectorTest extends LuceneTestCase {
private static final DocValuesType[] vts = new DocValuesType[]{
@ -121,30 +140,30 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
int maxDoc = reader.maxDoc();
Sort sortWithinGroup = new Sort(new SortField("id", SortField.Type.INT, true));
AbstractAllGroupHeadsCollector<?> allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup, canUseIDV, valueType);
AbstractAllGroupHeadsCollector<?> allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup);
indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupHeadsCollector);
assertTrue(arrayContains(new int[]{2, 3, 5, 7}, allGroupHeadsCollector.retrieveGroupHeads()));
assertTrue(openBitSetContains(new int[]{2, 3, 5, 7}, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc));
allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup, canUseIDV, valueType);
allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup);
indexSearcher.search(new TermQuery(new Term("content", "some")), allGroupHeadsCollector);
assertTrue(arrayContains(new int[]{2, 3, 4}, allGroupHeadsCollector.retrieveGroupHeads()));
assertTrue(openBitSetContains(new int[]{2, 3, 4}, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc));
allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup, canUseIDV, valueType);
allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup);
indexSearcher.search(new TermQuery(new Term("content", "blob")), allGroupHeadsCollector);
assertTrue(arrayContains(new int[]{1, 5}, allGroupHeadsCollector.retrieveGroupHeads()));
assertTrue(openBitSetContains(new int[]{1, 5}, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc));
// STRING sort type triggers different implementation
Sort sortWithinGroup2 = new Sort(new SortField("id", SortField.Type.STRING, true));
allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup2, canUseIDV, valueType);
allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup2);
indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupHeadsCollector);
assertTrue(arrayContains(new int[]{2, 3, 5, 7}, allGroupHeadsCollector.retrieveGroupHeads()));
assertTrue(openBitSetContains(new int[]{2, 3, 5, 7}, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc));
Sort sortWithinGroup3 = new Sort(new SortField("id", SortField.Type.STRING, false));
allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup3, canUseIDV, valueType);
allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup3);
indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupHeadsCollector);
// 7 b/c higher doc id wins, even if order of field is in not in reverse.
assertTrue(arrayContains(new int[]{0, 3, 4, 6}, allGroupHeadsCollector.retrieveGroupHeads()));
@ -168,7 +187,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
System.out.println("TEST: numDocs=" + numDocs + " numGroups=" + numGroups);
}
final List<BytesRef> groups = new ArrayList<BytesRef>();
final List<BytesRef> groups = new ArrayList<>();
for (int i = 0; i < numGroups; i++) {
String randomValue;
do {
@ -201,7 +220,6 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())));
boolean canUseIDV = true;
DocValuesType valueType = vts[random().nextInt(vts.length)];
Document doc = new Document();
@ -209,8 +227,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
Field group = newStringField("group", "", Field.Store.NO);
doc.add(group);
Field valuesField = null;
if (canUseIDV) {
switch(valueType) {
switch(valueType) {
case BINARY:
valuesField = new BinaryDocValuesField("group_dv", new BytesRef());
break;
@ -219,9 +236,8 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
break;
default:
fail("unhandled type");
}
doc.add(valuesField);
}
doc.add(valuesField);
Field sort1 = newStringField("sort1", "", Field.Store.NO);
doc.add(sort1);
docNoGroup.add(sort1);
@ -264,9 +280,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
groupDocs[i] = groupDoc;
if (groupDoc.group != null) {
group.setStringValue(groupDoc.group.utf8ToString());
if (canUseIDV) {
valuesField.setBytesValue(new BytesRef(groupDoc.group.utf8ToString()));
}
valuesField.setBytesValue(new BytesRef(groupDoc.group.utf8ToString()));
}
sort1.setStringValue(groupDoc.sort1.utf8ToString());
sort2.setStringValue(groupDoc.sort2.utf8ToString());
@ -293,11 +307,6 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
try {
final IndexSearcher s = newSearcher(r);
if (SlowCompositeReaderWrapper.class.isAssignableFrom(s.getIndexReader().getClass())) {
canUseIDV = false;
} else {
canUseIDV = true;
}
for (int contentID = 0; contentID < 3; contentID++) {
final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real" + contentID)), numDocs).scoreDocs;
@ -323,7 +332,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
final String searchTerm = "real" + random().nextInt(3);
boolean sortByScoreOnly = random().nextBoolean();
Sort sortWithinGroup = getRandomSort(sortByScoreOnly);
AbstractAllGroupHeadsCollector<?> allGroupHeadsCollector = createRandomCollector("group", sortWithinGroup, canUseIDV, valueType);
AbstractAllGroupHeadsCollector<?> allGroupHeadsCollector = createRandomCollector("group", sortWithinGroup);
s.search(new TermQuery(new Term("content", searchTerm)), allGroupHeadsCollector);
int[] expectedGroupHeads = createExpectedGroupHeads(searchTerm, groupDocs, sortWithinGroup, sortByScoreOnly, fieldIdToDocID);
int[] actualGroupHeads = allGroupHeadsCollector.retrieveGroupHeads();
@ -426,14 +435,14 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
}
private int[] createExpectedGroupHeads(String searchTerm, GroupDoc[] groupDocs, Sort docSort, boolean sortByScoreOnly, int[] fieldIdToDocID) {
Map<BytesRef, List<GroupDoc>> groupHeads = new HashMap<BytesRef, List<GroupDoc>>();
Map<BytesRef, List<GroupDoc>> groupHeads = new HashMap<>();
for (GroupDoc groupDoc : groupDocs) {
if (!groupDoc.content.startsWith(searchTerm)) {
continue;
}
if (!groupHeads.containsKey(groupDoc.group)) {
List<GroupDoc> list = new ArrayList<GroupDoc>();
List<GroupDoc> list = new ArrayList<>();
list.add(groupDoc);
groupHeads.put(groupDoc.group, list);
continue;
@ -453,7 +462,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
}
private Sort getRandomSort(boolean scoreOnly) {
final List<SortField> sortFields = new ArrayList<SortField>();
final List<SortField> sortFields = new ArrayList<>();
if (random().nextInt(7) == 2 || scoreOnly) {
sortFields.add(SortField.FIELD_SCORE);
} else {
@ -514,11 +523,11 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
}
@SuppressWarnings({"unchecked","rawtypes"})
private AbstractAllGroupHeadsCollector<?> createRandomCollector(String groupField, Sort sortWithinGroup, boolean canUseIDV, DocValuesType valueType) {
private AbstractAllGroupHeadsCollector<?> createRandomCollector(String groupField, Sort sortWithinGroup) {
AbstractAllGroupHeadsCollector<? extends AbstractAllGroupHeadsCollector.GroupHead> collector;
if (random().nextBoolean()) {
ValueSource vs = new BytesRefFieldSource(groupField);
collector = new FunctionAllGroupHeadsCollector(vs, new HashMap<Object, Object>(), sortWithinGroup);
collector = new FunctionAllGroupHeadsCollector(vs, new HashMap<>(), sortWithinGroup);
} else {
collector = TermAllGroupHeadsCollector.create(groupField, sortWithinGroup);
}

36
lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupsCollectorTest.java
View File

@ -18,10 +18,13 @@ package org.apache.lucene.search.grouping;
*/
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.search.IndexSearcher;
@ -48,25 +51,24 @@ public class AllGroupsCollectorTest extends LuceneTestCase {
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
boolean canUseIDV = true;
// 0
Document doc = new Document();
addGroupField(doc, groupField, "author1", canUseIDV);
addGroupField(doc, groupField, "author1");
doc.add(new TextField("content", "random text", Field.Store.YES));
doc.add(new Field("id", "1", customType));
w.addDocument(doc);
// 1
doc = new Document();
addGroupField(doc, groupField, "author1", canUseIDV);
addGroupField(doc, groupField, "author1");
doc.add(new TextField("content", "some more random text blob", Field.Store.YES));
doc.add(new Field("id", "2", customType));
w.addDocument(doc);
// 2
doc = new Document();
addGroupField(doc, groupField, "author1", canUseIDV);
addGroupField(doc, groupField, "author1");
doc.add(new TextField("content", "some more random textual data", Field.Store.YES));
doc.add(new Field("id", "3", customType));
w.addDocument(doc);
@ -74,21 +76,21 @@ public class AllGroupsCollectorTest extends LuceneTestCase {
// 3
doc = new Document();
addGroupField(doc, groupField, "author2", canUseIDV);
addGroupField(doc, groupField, "author2");
doc.add(new TextField("content", "some random text", Field.Store.YES));
doc.add(new Field("id", "4", customType));
w.addDocument(doc);
// 4
doc = new Document();
addGroupField(doc, groupField, "author3", canUseIDV);
addGroupField(doc, groupField, "author3");
doc.add(new TextField("content", "some more random text", Field.Store.YES));
doc.add(new Field("id", "5", customType));
w.addDocument(doc);
// 5
doc = new Document();
addGroupField(doc, groupField, "author3", canUseIDV);
addGroupField(doc, groupField, "author3");
doc.add(new TextField("content", "random blob", Field.Store.YES));
doc.add(new Field("id", "6", customType));
w.addDocument(doc);
@ -102,15 +104,15 @@ public class AllGroupsCollectorTest extends LuceneTestCase {
IndexSearcher indexSearcher = new IndexSearcher(w.getReader());
w.close();
AbstractAllGroupsCollector<?> allGroupsCollector = createRandomCollector(groupField, canUseIDV);
AbstractAllGroupsCollector<?> allGroupsCollector = createRandomCollector(groupField);
indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupsCollector);
assertEquals(4, allGroupsCollector.getGroupCount());
allGroupsCollector = createRandomCollector(groupField, canUseIDV);
allGroupsCollector = createRandomCollector(groupField);
indexSearcher.search(new TermQuery(new Term("content", "some")), allGroupsCollector);
assertEquals(3, allGroupsCollector.getGroupCount());
allGroupsCollector = createRandomCollector(groupField, canUseIDV);
allGroupsCollector = createRandomCollector(groupField);
indexSearcher.search(new TermQuery(new Term("content", "blob")), allGroupsCollector);
assertEquals(2, allGroupsCollector.getGroupCount());
@ -118,20 +120,18 @@ public class AllGroupsCollectorTest extends LuceneTestCase {
dir.close();
}
private void addGroupField(Document doc, String groupField, String value, boolean canUseIDV) {
private void addGroupField(Document doc, String groupField, String value) {
doc.add(new TextField(groupField, value, Field.Store.YES));
if (canUseIDV) {
doc.add(new SortedDocValuesField(groupField, new BytesRef(value)));
}
doc.add(new SortedDocValuesField(groupField, new BytesRef(value)));
}
private AbstractAllGroupsCollector<?> createRandomCollector(String groupField, boolean canUseIDV) {
private AbstractAllGroupsCollector<?> createRandomCollector(String groupField) {
AbstractAllGroupsCollector<?> selected;
if (random().nextBoolean()) {
selected = new TermAllGroupsCollector(groupField);
} else {
ValueSource vs = new BytesRefFieldSource(groupField);
selected = new FunctionAllGroupsCollector(vs, new HashMap<Object, Object>());
selected = new FunctionAllGroupsCollector(vs, new HashMap<>());
}
if (VERBOSE) {

50
lucene/grouping/src/test/org/apache/lucene/search/grouping/DistinctValuesCollectorTest.java
View File

@ -17,13 +17,19 @@ package org.apache.lucene.search.grouping;
* limitations under the License.
*/
import java.io.IOException;
import java.util.*;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.index.*;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Sort;
@ -39,6 +45,20 @@ import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueStr;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Random;
import java.util.Set;
public class DistinctValuesCollectorTest extends AbstractGroupingTestCase {
private final static NullComparator nullComparator = new NullComparator();
@ -267,9 +287,9 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase {
AbstractDistinctValuesCollector.GroupCount<Comparable<?>> actual = actualResult.get(i);
assertValues(expected.groupValue, actual.groupValue);
assertEquals(expected.uniqueValues.size(), actual.uniqueValues.size());
List<Comparable<?>> expectedUniqueValues = new ArrayList<Comparable<?>>(expected.uniqueValues);
List<Comparable<?>> expectedUniqueValues = new ArrayList<>(expected.uniqueValues);
Collections.sort(expectedUniqueValues, nullComparator);
List<Comparable<?>> actualUniqueValues = new ArrayList<Comparable<?>>(actual.uniqueValues);
List<Comparable<?>> actualUniqueValues = new ArrayList<>(actual.uniqueValues);
Collections.sort(actualUniqueValues, nullComparator);
for (int j = 0; j < expectedUniqueValues.size(); j++) {
assertValues(expectedUniqueValues.get(j), actualUniqueValues.get(j));
@ -373,7 +393,7 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase {
Random random = random();
Collection<SearchGroup<T>> searchGroups = firstPassGroupingCollector.getTopGroups(0, false);
if (FunctionFirstPassGroupingCollector.class.isAssignableFrom(firstPassGroupingCollector.getClass())) {
return (AbstractDistinctValuesCollector) new FunctionDistinctValuesCollector(new HashMap<Object, Object>(), new BytesRefFieldSource(groupField), new BytesRefFieldSource(countField), (Collection) searchGroups);
return (AbstractDistinctValuesCollector) new FunctionDistinctValuesCollector(new HashMap<>(), new BytesRefFieldSource(groupField), new BytesRefFieldSource(countField), (Collection) searchGroups);
} else {
return (AbstractDistinctValuesCollector) new TermDistinctValuesCollector(groupField, countField, (Collection) searchGroups);
}
@ -384,13 +404,13 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase {
Random random = random();
if (dvType != null) {
if (random.nextBoolean()) {
return (AbstractFirstPassGroupingCollector<T>) new FunctionFirstPassGroupingCollector(new BytesRefFieldSource(groupField), new HashMap<Object, Object>(), groupSort, topNGroups);
return (AbstractFirstPassGroupingCollector<T>) new FunctionFirstPassGroupingCollector(new BytesRefFieldSource(groupField), new HashMap<>(), groupSort, topNGroups);
} else {
return (AbstractFirstPassGroupingCollector<T>) new TermFirstPassGroupingCollector(groupField, groupSort, topNGroups);
}
} else {
if (random.nextBoolean()) {
return (AbstractFirstPassGroupingCollector<T>) new FunctionFirstPassGroupingCollector(new BytesRefFieldSource(groupField), new HashMap<Object, Object>(), groupSort, topNGroups);
return (AbstractFirstPassGroupingCollector<T>) new FunctionFirstPassGroupingCollector(new BytesRefFieldSource(groupField), new HashMap<>(), groupSort, topNGroups);
} else {
return (AbstractFirstPassGroupingCollector<T>) new TermFirstPassGroupingCollector(groupField, groupSort, topNGroups);
}
@ -413,7 +433,7 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase {
if (topN <= i++) {
break;
}
Set<BytesRef> uniqueValues = new HashSet<BytesRef>();
Set<BytesRef> uniqueValues = new HashSet<>();
for (String val : groupCounts.get(group)) {
uniqueValues.add(val != null ? new BytesRef(val) : null);
}
@ -450,8 +470,8 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase {
countValues[i] = generateRandomNonEmptyString();
}
List<String> contentStrings = new ArrayList<String>();
Map<String, Map<String, Set<String>>> searchTermToGroupCounts = new HashMap<String, Map<String, Set<String>>>();
List<String> contentStrings = new ArrayList<>();
Map<String, Map<String, Set<String>>> searchTermToGroupCounts = new HashMap<>();
for (int i = 1; i <= numDocs; i++) {
String groupValue = random.nextInt(23) == 14 ? null : groupValues[random.nextInt(groupValues.length)];
String countValue = random.nextInt(21) == 13 ? null : countValues[random.nextInt(countValues.length)];
@ -459,13 +479,13 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase {
Map<String, Set<String>> groupToCounts = searchTermToGroupCounts.get(content);
if (groupToCounts == null) {
// Groups sort always DOCID asc...
searchTermToGroupCounts.put(content, groupToCounts = new LinkedHashMap<String, Set<String>>());
searchTermToGroupCounts.put(content, groupToCounts = new LinkedHashMap<>());
contentStrings.add(content);
}
Set<String> countsVals = groupToCounts.get(groupValue);
if (countsVals == null) {
groupToCounts.put(groupValue, countsVals = new HashSet<String>());
groupToCounts.put(groupValue, countsVals = new HashSet<>());
}
countsVals.add(countValue);

45
lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupFacetCollectorTest.java
View File

@ -18,7 +18,11 @@ package org.apache.lucene.search.grouping;
*/
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.RandomIndexWriter;
@ -28,12 +32,22 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.grouping.term.TermGroupFacetCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util._TestUtil;
import java.io.IOException;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.NavigableSet;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
public class GroupFacetCollectorTest extends AbstractGroupingTestCase {
@ -48,8 +62,7 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase {
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
boolean canUseDV = true;
boolean useDv = canUseDV && random().nextBoolean();
boolean useDv = random().nextBoolean();
// 0
Document doc = new Document();
@ -89,9 +102,9 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase {
IndexSearcher indexSearcher = new IndexSearcher(w.getReader());
List<TermGroupFacetCollector.FacetEntry> entries = null;
AbstractGroupFacetCollector groupedAirportFacetCollector = null;
TermGroupFacetCollector.GroupedFacetResult airportResult = null;
List<TermGroupFacetCollector.FacetEntry> entries;
AbstractGroupFacetCollector groupedAirportFacetCollector;
TermGroupFacetCollector.GroupedFacetResult airportResult;
for (int limit : new int[] { 2, 10, 100, Integer.MAX_VALUE }) {
// any of these limits is plenty for the data we have
@ -473,11 +486,11 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase {
System.out.println("TEST: numDocs=" + numDocs + " numGroups=" + numGroups);
}
final List<String> groups = new ArrayList<String>();
final List<String> groups = new ArrayList<>();
for (int i = 0; i < numGroups; i++) {
groups.add(generateRandomNonEmptyString());
}
final List<String> facetValues = new ArrayList<String>();
final List<String> facetValues = new ArrayList<>();
for (int i = 0; i < numFacets; i++) {
facetValues.add(generateRandomNonEmptyString());
}
@ -540,7 +553,7 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase {
docNoFacet.add(content);
docNoGroupNoFacet.add(content);
NavigableSet<String> uniqueFacetValues = new TreeSet<String>(new Comparator<String>() {
NavigableSet<String> uniqueFacetValues = new TreeSet<>(new Comparator<String>() {
@Override
public int compare(String a, String b) {
@ -556,7 +569,7 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase {
}
});
Map<String, Map<String, Set<String>>> searchTermToFacetToGroups = new HashMap<String, Map<String, Set<String>>>();
Map<String, Map<String, Set<String>>> searchTermToFacetToGroups = new HashMap<>();
int facetWithMostGroups = 0;
for (int i = 0; i < numDocs; i++) {
final String groupValue;
@ -578,7 +591,7 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase {
}
Map<String, Set<String>> facetToGroups = searchTermToFacetToGroups.get(contentStr);
List<String> facetVals = new ArrayList<String>();
List<String> facetVals = new ArrayList<>();
if (useDv || random.nextInt(24) != 18) {
if (useDv) {
String facetValue = facetValues.get(random.nextInt(facetValues.size()));
@ -656,14 +669,14 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase {
private GroupedFacetResult createExpectedFacetResult(String searchTerm, IndexContext context, int offset, int limit, int minCount, final boolean orderByCount, String facetPrefix) {
Map<String, Set<String>> facetGroups = context.searchTermToFacetGroups.get(searchTerm);
if (facetGroups == null) {
facetGroups = new HashMap<String, Set<String>>();
facetGroups = new HashMap<>();
}
int totalCount = 0;
int totalMissCount = 0;
Set<String> facetValues;
if (facetPrefix != null) {
facetValues = new HashSet<String>();
facetValues = new HashSet<>();
for (String facetValue : context.facetValues) {
if (facetValue != null && facetValue.startsWith(facetPrefix)) {
facetValues.add(facetValue);
@ -673,7 +686,7 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase {
facetValues = context.facetValues;
}
List<TermGroupFacetCollector.FacetEntry> entries = new ArrayList<TermGroupFacetCollector.FacetEntry>(facetGroups.size());
List<TermGroupFacetCollector.FacetEntry> entries = new ArrayList<>(facetGroups.size());
// also includes facets with count 0
for (String facetValue : facetValues) {
if (facetValue == null) {

4
lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java
View File

@ -60,7 +60,7 @@ public class GroupingSearchTest extends LuceneTestCase {
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
boolean canUseIDV = true;
List<Document> documents = new ArrayList<Document>();
List<Document> documents = new ArrayList<>();
// 0
Document doc = new Document();
addGroupField(doc, groupField, "author1", canUseIDV);
@ -207,7 +207,7 @@ public class GroupingSearchTest extends LuceneTestCase {
GroupingSearch groupingSearch;
if (random().nextBoolean()) {
ValueSource vs = new BytesRefFieldSource(groupField);
groupingSearch = new GroupingSearch(vs, new HashMap<Object, Object>());
groupingSearch = new GroupingSearch(vs, new HashMap<>());
} else {
groupingSearch = new GroupingSearch(groupField);
}

93
lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
View File

@ -67,45 +67,44 @@ public class TestGrouping extends LuceneTestCase {
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
boolean canUseIDV = true;
// 0
Document doc = new Document();
addGroupField(doc, groupField, "author1", canUseIDV);
addGroupField(doc, groupField, "author1");
doc.add(new TextField("content", "random text", Field.Store.YES));
doc.add(new Field("id", "1", customType));
w.addDocument(doc);
// 1
doc = new Document();
addGroupField(doc, groupField, "author1", canUseIDV);
addGroupField(doc, groupField, "author1");
doc.add(new TextField("content", "some more random text", Field.Store.YES));
doc.add(new Field("id", "2", customType));
w.addDocument(doc);
// 2
doc = new Document();
addGroupField(doc, groupField, "author1", canUseIDV);
addGroupField(doc, groupField, "author1");
doc.add(new TextField("content", "some more random textual data", Field.Store.YES));
doc.add(new Field("id", "3", customType));
w.addDocument(doc);
// 3
doc = new Document();
addGroupField(doc, groupField, "author2", canUseIDV);
addGroupField(doc, groupField, "author2");
doc.add(new TextField("content", "some random text", Field.Store.YES));
doc.add(new Field("id", "4", customType));
w.addDocument(doc);
// 4
doc = new Document();
addGroupField(doc, groupField, "author3", canUseIDV);
addGroupField(doc, groupField, "author3");
doc.add(new TextField("content", "some more random text", Field.Store.YES));
doc.add(new Field("id", "5", customType));
w.addDocument(doc);
// 5
doc = new Document();
addGroupField(doc, groupField, "author3", canUseIDV);
addGroupField(doc, groupField, "author3");
doc.add(new TextField("content", "random", Field.Store.YES));
doc.add(new Field("id", "6", customType));
w.addDocument(doc);
@ -121,7 +120,7 @@ public class TestGrouping extends LuceneTestCase {
final Sort groupSort = Sort.RELEVANCE;
if (canUseIDV && random().nextBoolean()) {
if (random().nextBoolean()) {
groupField += "_dv";
}
@ -172,18 +171,16 @@ public class TestGrouping extends LuceneTestCase {
dir.close();
}
private void addGroupField(Document doc, String groupField, String value, boolean canUseIDV) {
private void addGroupField(Document doc, String groupField, String value) {
doc.add(new TextField(groupField, value, Field.Store.YES));
if (canUseIDV) {
doc.add(new SortedDocValuesField(groupField + "_dv", new BytesRef(value)));
}
doc.add(new SortedDocValuesField(groupField + "_dv", new BytesRef(value)));
}
private AbstractFirstPassGroupingCollector<?> createRandomFirstPassCollector(String groupField, Sort groupSort, int topDocs) throws IOException {
AbstractFirstPassGroupingCollector<?> selected;
if (random().nextBoolean()) {
ValueSource vs = new BytesRefFieldSource(groupField);
selected = new FunctionFirstPassGroupingCollector(vs, new HashMap<Object, Object>(), groupSort, topDocs);
selected = new FunctionFirstPassGroupingCollector(vs, new HashMap<>(), groupSort, topDocs);
} else {
selected = new TermFirstPassGroupingCollector(groupField, groupSort, topDocs);
}
@ -196,7 +193,7 @@ public class TestGrouping extends LuceneTestCase {
private AbstractFirstPassGroupingCollector<?> createFirstPassCollector(String groupField, Sort groupSort, int topDocs, AbstractFirstPassGroupingCollector<?> firstPassGroupingCollector) throws IOException {
if (TermFirstPassGroupingCollector.class.isAssignableFrom(firstPassGroupingCollector.getClass())) {
ValueSource vs = new BytesRefFieldSource(groupField);
return new FunctionFirstPassGroupingCollector(vs, new HashMap<Object, Object>(), groupSort, topDocs);
return new FunctionFirstPassGroupingCollector(vs, new HashMap<>(), groupSort, topDocs);
} else {
return new TermFirstPassGroupingCollector(groupField, groupSort, topDocs);
}
@ -238,9 +235,9 @@ public class TestGrouping extends LuceneTestCase {
return new TermSecondPassGroupingCollector(groupField, searchGroups, groupSort, sortWithinGroup, maxDocsPerGroup , getScores, getMaxScores, fillSortFields);
} else {
ValueSource vs = new BytesRefFieldSource(groupField);
List<SearchGroup<MutableValue>> mvalSearchGroups = new ArrayList<SearchGroup<MutableValue>>(searchGroups.size());
List<SearchGroup<MutableValue>> mvalSearchGroups = new ArrayList<>(searchGroups.size());
for (SearchGroup<BytesRef> mergedTopGroup : searchGroups) {
SearchGroup<MutableValue> sg = new SearchGroup<MutableValue>();
SearchGroup<MutableValue> sg = new SearchGroup<>();
MutableValueStr groupValue = new MutableValueStr();
if (mergedTopGroup.groupValue != null) {
groupValue.value = mergedTopGroup.groupValue;
@ -253,7 +250,7 @@ public class TestGrouping extends LuceneTestCase {
mvalSearchGroups.add(sg);
}
return new FunctionSecondPassGroupingCollector(mvalSearchGroups, groupSort, sortWithinGroup, maxDocsPerGroup, getScores, getMaxScores, fillSortFields, vs, new HashMap<Object, Object>());
return new FunctionSecondPassGroupingCollector(mvalSearchGroups, groupSort, sortWithinGroup, maxDocsPerGroup, getScores, getMaxScores, fillSortFields, vs, new HashMap<>());
}
}
@ -263,7 +260,7 @@ public class TestGrouping extends LuceneTestCase {
return new TermAllGroupsCollector(groupField);
} else {
ValueSource vs = new BytesRefFieldSource(groupField);
return new FunctionAllGroupsCollector(vs, new HashMap<Object, Object>());
return new FunctionAllGroupsCollector(vs, new HashMap<>());
}
}
@ -299,9 +296,9 @@ public class TestGrouping extends LuceneTestCase {
return null;
}
List<SearchGroup<BytesRef>> groups = new ArrayList<SearchGroup<BytesRef>>(mutableValueGroups.size());
List<SearchGroup<BytesRef>> groups = new ArrayList<>(mutableValueGroups.size());
for (SearchGroup<MutableValue> mutableValueGroup : mutableValueGroups) {
SearchGroup<BytesRef> sg = new SearchGroup<BytesRef>();
SearchGroup<BytesRef> sg = new SearchGroup<>();
sg.groupValue = mutableValueGroup.groupValue.exists() ? ((MutableValueStr) mutableValueGroup.groupValue).value : null;
sg.sortValues = mutableValueGroup.sortValues;
groups.add(sg);
@ -318,10 +315,10 @@ public class TestGrouping extends LuceneTestCase {
return ((TermSecondPassGroupingCollector) c).getTopGroups(withinGroupOffset);
} else if (c.getClass().isAssignableFrom(FunctionSecondPassGroupingCollector.class)) {
TopGroups<MutableValue> mvalTopGroups = ((FunctionSecondPassGroupingCollector) c).getTopGroups(withinGroupOffset);
List<GroupDocs<BytesRef>> groups = new ArrayList<GroupDocs<BytesRef>>(mvalTopGroups.groups.length);
List<GroupDocs<BytesRef>> groups = new ArrayList<>(mvalTopGroups.groups.length);
for (GroupDocs<MutableValue> mvalGd : mvalTopGroups.groups) {
BytesRef groupValue = mvalGd.groupValue.exists() ? ((MutableValueStr) mvalGd.groupValue).value : null;
groups.add(new GroupDocs<BytesRef>(Float.NaN, mvalGd.maxScore, mvalGd.totalHits, mvalGd.scoreDocs, groupValue, mvalGd.groupSortValues));
groups.add(new GroupDocs<>(Float.NaN, mvalGd.maxScore, mvalGd.totalHits, mvalGd.scoreDocs, groupValue, mvalGd.groupSortValues));
}
return new TopGroups<BytesRef>(mvalTopGroups.groupSort, mvalTopGroups.withinGroupSort, mvalTopGroups.totalHitCount, mvalTopGroups.totalGroupedHitCount, groups.toArray(new GroupDocs[groups.size()]), Float.NaN);
}
@ -349,7 +346,7 @@ public class TestGrouping extends LuceneTestCase {
}
private Sort getRandomSort() {
final List<SortField> sortFields = new ArrayList<SortField>();
final List<SortField> sortFields = new ArrayList<>();
if (random().nextInt(7) == 2) {
sortFields.add(SortField.FIELD_SCORE);
} else {
@ -411,14 +408,14 @@ public class TestGrouping extends LuceneTestCase {
final Comparable<?> c;
final SortField sf = sortFields[fieldIDX];
if (sf.getType() == SortField.Type.SCORE) {
c = new Float(d.score);
c = d.score;
} else if (sf.getField().equals("sort1")) {
c = d.sort1;
} else if (sf.getField().equals("sort2")) {
c = d.sort2;
} else {
assertEquals("id", sf.getField());
c = new Integer(d.id);
c = d.id;
}
fields[fieldIDX] = c;
}
@ -449,12 +446,12 @@ public class TestGrouping extends LuceneTestCase {
final Comparator<GroupDoc> groupSortComp = getComparator(groupSort);
Arrays.sort(groupDocs, groupSortComp);
final HashMap<BytesRef,List<GroupDoc>> groups = new HashMap<BytesRef,List<GroupDoc>>();
final List<BytesRef> sortedGroups = new ArrayList<BytesRef>();
final List<Comparable<?>[]> sortedGroupFields = new ArrayList<Comparable<?>[]>();
final HashMap<BytesRef,List<GroupDoc>> groups = new HashMap<>();
final List<BytesRef> sortedGroups = new ArrayList<>();
final List<Comparable<?>[]> sortedGroupFields = new ArrayList<>();
int totalHitCount = 0;
Set<BytesRef> knownGroups = new HashSet<BytesRef>();
Set<BytesRef> knownGroups = new HashSet<>();
//System.out.println("TEST: slowGrouping");
for(GroupDoc d : groupDocs) {
@ -479,7 +476,7 @@ public class TestGrouping extends LuceneTestCase {
if (fillFields) {
sortedGroupFields.add(fillFields(d, groupSort));
}
l = new ArrayList<GroupDoc>();
l = new ArrayList<>();
groups.put(d.group, l);
}
l.add(d);
@ -519,7 +516,7 @@ public class TestGrouping extends LuceneTestCase {
hits = new ScoreDoc[0];
}
result[idx-groupOffset] = new GroupDocs<BytesRef>(Float.NaN,
result[idx-groupOffset] = new GroupDocs<>(Float.NaN,
0.0f,
docs.size(),
hits,
@ -528,20 +525,20 @@ public class TestGrouping extends LuceneTestCase {
}
if (doAllGroups) {
return new TopGroups<BytesRef>(
new TopGroups<BytesRef>(groupSort.getSort(), docSort.getSort(), totalHitCount, totalGroupedHitCount, result, Float.NaN),
knownGroups.size()
return new TopGroups<>(
new TopGroups<>(groupSort.getSort(), docSort.getSort(), totalHitCount, totalGroupedHitCount, result, Float.NaN),
knownGroups.size()
);
} else {
return new TopGroups<BytesRef>(groupSort.getSort(), docSort.getSort(), totalHitCount, totalGroupedHitCount, result, Float.NaN);
return new TopGroups<>(groupSort.getSort(), docSort.getSort(), totalHitCount, totalGroupedHitCount, result, Float.NaN);
}
}
private DirectoryReader getDocBlockReader(Directory dir, GroupDoc[] groupDocs) throws IOException {
// Coalesce by group, but in random order:
Collections.shuffle(Arrays.asList(groupDocs), random());
final Map<BytesRef,List<GroupDoc>> groupMap = new HashMap<BytesRef,List<GroupDoc>>();
final List<BytesRef> groupValues = new ArrayList<BytesRef>();
final Map<BytesRef,List<GroupDoc>> groupMap = new HashMap<>();
final List<BytesRef> groupValues = new ArrayList<>();
for(GroupDoc groupDoc : groupDocs) {
if (!groupMap.containsKey(groupDoc.group)) {
@ -557,7 +554,7 @@ public class TestGrouping extends LuceneTestCase {
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())));
final List<List<Document>> updateDocs = new ArrayList<List<Document>>();
final List<List<Document>> updateDocs = new ArrayList<>();
FieldType groupEndType = new FieldType(StringField.TYPE_NOT_STORED);
groupEndType.setIndexOptions(IndexOptions.DOCS_ONLY);
@ -565,7 +562,7 @@ public class TestGrouping extends LuceneTestCase {
//System.out.println("TEST: index groups");
for(BytesRef group : groupValues) {
final List<Document> docs = new ArrayList<Document>();
final List<Document> docs = new ArrayList<>();
//System.out.println("TEST: group=" + (group == null ? "null" : group.utf8ToString()));
for(GroupDoc groupValue : groupMap.get(group)) {
Document doc = new Document();
@ -637,7 +634,7 @@ public class TestGrouping extends LuceneTestCase {
System.out.println("TEST: numDocs=" + numDocs + " numGroups=" + numGroups);
}
final List<BytesRef> groups = new ArrayList<BytesRef>();
final List<BytesRef> groups = new ArrayList<>();
for(int i=0;i<numGroups;i++) {
String randomValue;
do {
@ -790,14 +787,14 @@ public class TestGrouping extends LuceneTestCase {
// ReaderBlocks only increases maxDoc() vs reader, which
// means a monotonic shift in scores, so we can
// reliably remap them w/ Map:
final Map<String,Map<Float,Float>> scoreMap = new HashMap<String,Map<Float,Float>>();
final Map<String,Map<Float,Float>> scoreMap = new HashMap<>();
// Tricky: must separately set .score2, because the doc
// block index was created with possible deletions!
//System.out.println("fixup score2");
for(int contentID=0;contentID<3;contentID++) {
//System.out.println(" term=real" + contentID);
final Map<Float,Float> termScoreMap = new HashMap<Float,Float>();
final Map<Float,Float> termScoreMap = new HashMap<>();
scoreMap.put("real"+contentID, termScoreMap);
//System.out.println("term=real" + contentID + " dfold=" + s.docFreq(new Term("content", "real"+contentID)) +
//" dfnew=" + sBlocks.docFreq(new Term("content", "real"+contentID)));
@ -939,7 +936,7 @@ public class TestGrouping extends LuceneTestCase {
// Get 1st pass top groups using shards
ValueHolder<Boolean> idvBasedImplsUsedSharded = new ValueHolder<Boolean>(false);
ValueHolder<Boolean> idvBasedImplsUsedSharded = new ValueHolder<>(false);
final TopGroups<BytesRef> topGroupsShards = searchShards(s, shards.subSearchers, query, groupSort, docSort,
groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, canUseIDV, false, idvBasedImplsUsedSharded);
final AbstractSecondPassGroupingCollector<?> c2;
@ -971,7 +968,7 @@ public class TestGrouping extends LuceneTestCase {
if (doAllGroups) {
TopGroups<BytesRef> tempTopGroups = getTopGroups(c2, docOffset);
groupsResult = new TopGroups<BytesRef>(tempTopGroups, allGroupsCollector.getGroupCount());
groupsResult = new TopGroups<>(tempTopGroups, allGroupsCollector.getGroupCount());
} else {
groupsResult = getTopGroups(c2, docOffset);
}
@ -1058,7 +1055,7 @@ public class TestGrouping extends LuceneTestCase {
final TopGroups<BytesRef> groupsResultBlocks;
if (doAllGroups && tempTopGroupsBlocks != null) {
assertEquals((int) tempTopGroupsBlocks.totalGroupCount, allGroupsCollector2.getGroupCount());
groupsResultBlocks = new TopGroups<BytesRef>(tempTopGroupsBlocks, allGroupsCollector2.getGroupCount());
groupsResultBlocks = new TopGroups<>(tempTopGroupsBlocks, allGroupsCollector2.getGroupCount());
} else {
groupsResultBlocks = tempTopGroupsBlocks;
}
@ -1086,7 +1083,7 @@ public class TestGrouping extends LuceneTestCase {
// Block index does not index DocValues so we pass
// false for canUseIDV:
final TopGroups<BytesRef> topGroupsBlockShards = searchShards(sBlocks, shardsBlocks.subSearchers, query,
groupSort, docSort, groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, false, false, new ValueHolder<Boolean>(false));
groupSort, docSort, groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, false, false, new ValueHolder<>(false));
if (expectedGroups != null) {
// Fixup scores for reader2
@ -1168,8 +1165,8 @@ public class TestGrouping extends LuceneTestCase {
}
// Run 1st pass collector to get top groups per shard
final Weight w = topSearcher.createNormalizedWeight(query);
final List<Collection<SearchGroup<BytesRef>>> shardGroups = new ArrayList<Collection<SearchGroup<BytesRef>>>();
List<AbstractFirstPassGroupingCollector<?>> firstPassGroupingCollectors = new ArrayList<AbstractFirstPassGroupingCollector<?>>();
final List<Collection<SearchGroup<BytesRef>>> shardGroups = new ArrayList<>();
List<AbstractFirstPassGroupingCollector<?>> firstPassGroupingCollectors = new ArrayList<>();
AbstractFirstPassGroupingCollector<?> firstPassCollector = null;
boolean shardsCanUseIDV;
if (canUseIDV) {