Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-5416: CollapsingQParserPlugin bug with tagging 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1551999 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Joel Bernstein 2013-12-18 16:06:13 +00:00
parent 22baafc512
commit a18470e858
4 changed files with 252 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
View File

@ -118,21 +118,15 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
}
private class CollapsingPostFilter extends ExtendedQueryBase implements PostFilter {
public class CollapsingPostFilter extends ExtendedQueryBase implements PostFilter, ScoreFilter {
private Object cacheId;
private String field;
private int leafCount;
private SortedDocValues docValues;
private int maxDoc;
private String max;
private String min;
private FieldType fieldType;
private boolean needsScores = true;
private int nullPolicy;
private SolrIndexSearcher searcher;
private SolrParams solrParams;
private Map context;
private IndexSchema schema;
public static final int NULL_POLICY_IGNORE = 0;
public static final int NULL_POLICY_COLLAPSE = 1;
public static final int NULL_POLICY_EXPAND = 2;
@ -180,7 +174,13 @@ public class CollapsingQParserPlugin extends QParserPlugin {
public CollapsingPostFilter(SolrParams localParams, SolrParams params, SolrQueryRequest request) throws IOException {
this.cacheId = new Object();
this.field = localParams.get("field");
this.solrParams = params;
this.max = localParams.get("max");
this.min = localParams.get("min");
this.context = request.getContext();
if(this.min != null || this.max != null) {
this.needsScores = needsScores(params);
}
String nPolicy = localParams.get("nullPolicy", NULL_IGNORE);
if(nPolicy.equals(NULL_IGNORE)) {
this.nullPolicy = NULL_POLICY_IGNORE;
@ -191,34 +191,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
} else {
throw new IOException("Invalid nullPolicy:"+nPolicy);
}
this.searcher = request.getSearcher();
this.leafCount = searcher.getTopReaderContext().leaves().size();
this.maxDoc = searcher.maxDoc();
this.schema = searcher.getSchema();
SchemaField schemaField = schema.getField(this.field);
if(schemaField.hasDocValues()) {
this.docValues = searcher.getAtomicReader().getSortedDocValues(this.field);
} else {
this.docValues = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), this.field);
}
this.max = localParams.get("max");
if(this.max != null) {
this.fieldType = searcher.getSchema().getField(this.max).getType();
}
this.min = localParams.get("min");
if(this.min != null) {
this.fieldType = searcher.getSchema().getField(this.min).getType();
}
this.context = request.getContext();
}
private IntOpenHashSet getBoostDocs(IndexSearcher indexSearcher, Set<String> boosted) throws IOException {
private IntOpenHashSet getBoostDocs(SolrIndexSearcher indexSearcher, Set<String> boosted) throws IOException {
IntOpenHashSet boostDocs = null;
if(boosted != null) {
SchemaField idField = this.schema.getUniqueKeyField();
SchemaField idField = indexSearcher.getSchema().getUniqueKeyField();
String fieldName = idField.getName();
HashSet<BytesRef> localBoosts = new HashSet(boosted.size()*2);
Iterator<String> boostedIt = boosted.iterator();
@ -258,22 +236,47 @@ public class CollapsingQParserPlugin extends QParserPlugin {
public DelegatingCollector getFilterCollector(IndexSearcher indexSearcher) {
try {
IntOpenHashSet boostDocs = getBoostDocs(indexSearcher, (Set<String>) (this.context.get(QueryElevationComponent.BOOSTED)));
SolrIndexSearcher searcher = (SolrIndexSearcher)indexSearcher;
IndexSchema schema = searcher.getSchema();
SchemaField schemaField = schema.getField(this.field);
SortedDocValues docValues = null;
if(schemaField.hasDocValues()) {
docValues = searcher.getAtomicReader().getSortedDocValues(this.field);
} else {
docValues = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), this.field);
}
FieldType fieldType = null;
if(this.max != null) {
fieldType = searcher.getSchema().getField(this.max).getType();
}
if(this.min != null) {
fieldType = searcher.getSchema().getField(this.min).getType();
}
int maxDoc = searcher.maxDoc();
int leafCount = searcher.getTopReaderContext().leaves().size();
IntOpenHashSet boostDocs = getBoostDocs(searcher, (Set<String>) (this.context.get(QueryElevationComponent.BOOSTED)));
if(this.min != null || this.max != null) {
return new CollapsingFieldValueCollector(this.maxDoc,
this.leafCount,
this.docValues,
this.searcher,
return new CollapsingFieldValueCollector(maxDoc,
leafCount,
docValues,
this.nullPolicy,
max != null ? this.max : this.min,
max != null,
needsScores(this.solrParams),
this.fieldType,
this.needsScores,
fieldType,
boostDocs);
} else {
return new CollapsingScoreCollector(this.maxDoc, this.leafCount, this.docValues, this.nullPolicy, boostDocs);
return new CollapsingScoreCollector(maxDoc, leafCount, docValues, this.nullPolicy, boostDocs);
}
} catch (Exception e) {
throw new RuntimeException(e);
@ -283,7 +286,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private boolean needsScores(SolrParams params) {
String sortSpec = params.get("sort");
if(sortSpec != null) {
if(sortSpec != null && sortSpec.length()!=0) {
String[] sorts = sortSpec.split(",");
for(String s: sorts) {
String parts[] = s.split(" ");
@ -500,7 +503,6 @@ public class CollapsingQParserPlugin extends QParserPlugin {
public CollapsingFieldValueCollector(int maxDoc,
int segments,
SortedDocValues values,
SolrIndexSearcher searcher,
int nullPolicy,
String field,
boolean max,
@ -516,11 +518,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.needsScores = needsScores;
this.boostDocs = boostDocs;
if(fieldType instanceof TrieIntField) {
this.fieldValueCollapse = new IntValueCollapse(searcher, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
this.fieldValueCollapse = new IntValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
} else if(fieldType instanceof TrieLongField) {
this.fieldValueCollapse = new LongValueCollapse(searcher, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
this.fieldValueCollapse = new LongValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
} else if(fieldType instanceof TrieFloatField) {
this.fieldValueCollapse = new FloatValueCollapse(searcher, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
this.fieldValueCollapse = new FloatValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
} else {
throw new IOException("min/max must be either TrieInt, TrieLong or TrieFloat.");
}
@ -616,7 +618,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
public abstract void collapse(int ord, int contextDoc, int globalDoc) throws IOException;
public abstract void setNextReader(AtomicReaderContext context) throws IOException;
public FieldValueCollapse(SolrIndexSearcher searcher,
public FieldValueCollapse(int maxDoc,
String field,
int nullPolicy,
boolean max,
@ -626,7 +628,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.nullPolicy = nullPolicy;
this.max = max;
this.needsScores = needsScores;
this.collapsedSet = new OpenBitSet(searcher.maxDoc());
this.collapsedSet = new OpenBitSet(maxDoc);
this.boostDocs = boostDocs;
if(this.boostDocs != null) {
Iterator<IntCursor> it = boostDocs.iterator();
@ -676,14 +678,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private int nullVal;
private int[] ordVals;
public IntValueCollapse(SolrIndexSearcher searcher,
public IntValueCollapse(int maxDoc,
String field,
int nullPolicy,
int[] ords,
boolean max,
boolean needsScores,
IntOpenHashSet boostDocs) throws IOException {
super(searcher, field, nullPolicy, max, needsScores, boostDocs);
super(maxDoc, field, nullPolicy, max, needsScores, boostDocs);
this.ords = ords;
this.ordVals = new int[ords.length];
Arrays.fill(ords, -1);
@ -745,14 +747,13 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private long nullVal;
private long[] ordVals;
public LongValueCollapse(SolrIndexSearcher searcher,
String field,
public LongValueCollapse(int maxDoc, String field,
int nullPolicy,
int[] ords,
boolean max,
boolean needsScores,
IntOpenHashSet boostDocs) throws IOException {
super(searcher, field, nullPolicy, max, needsScores, boostDocs);
super(maxDoc, field, nullPolicy, max, needsScores, boostDocs);
this.ords = ords;
this.ordVals = new long[ords.length];
Arrays.fill(ords, -1);
@ -814,14 +815,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private float nullVal;
private float[] ordVals;
public FloatValueCollapse(SolrIndexSearcher searcher,
public FloatValueCollapse(int maxDoc,
String field,
int nullPolicy,
int[] ords,
boolean max,
boolean needsScores,
IntOpenHashSet boostDocs) throws IOException {
super(searcher, field, nullPolicy, max, needsScores, boostDocs);
super(maxDoc, field, nullPolicy, max, needsScores, boostDocs);
this.ords = ords;
this.ordVals = new float[ords.length];
Arrays.fill(ords, -1);

22
solr/core/src/java/org/apache/solr/search/ScoreFilter.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
public interface ScoreFilter {
}

28
solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -863,6 +863,25 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
}
};
private DocSet getDocSetScore(List<Query> queries) throws IOException {
Query main = queries.remove(0);
ProcessedFilter pf = getProcessedFilter(null, queries);
DocSetCollector setCollector = new DocSetCollector(maxDoc()>>6, maxDoc());
Collector collector = setCollector;
if (pf.postFilter != null) {
pf.postFilter.setLastDelegate(collector);
collector = pf.postFilter;
}
search(main, pf.filter, collector);
if(collector instanceof DelegatingCollector) {
((DelegatingCollector) collector).finish();
}
DocSet docSet = setCollector.getDocSet();
return docSet;
}
/**
* Returns the set of document ids matching all queries.
@ -873,6 +892,15 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
* The DocSet returned should <b>not</b> be modified.
*/
public DocSet getDocSet(List<Query> queries) throws IOException {
if(queries != null) {
for(Query q : queries) {
if(q instanceof ScoreFilter) {
return getDocSetScore(queries);
}
}
}
ProcessedFilter pf = getProcessedFilter(null, queries);
if (pf.answer != null) return pf.answer;

156
solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
View File

@ -47,24 +47,54 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
public void testCollapseQueries() throws Exception {
String[] doc = {"id","1", "term_s", "YYYY", "group_s", "group1", "test_ti", "5", "test_tl", "10", "test_tf", "2000"};
assertU(adoc(doc));
assertU(commit());
String[] doc1 = {"id","2", "term_s","YYYY", "group_s", "group1", "test_ti", "50", "test_tl", "100", "test_tf", "200"};
assertU(adoc(doc1));
String[] doc2 = {"id","3", "term_s", "YYYY", "test_ti", "5000", "test_tl", "100", "test_tf", "200"};
assertU(adoc(doc2));
assertU(commit());
String[] doc3 = {"id","4", "term_s", "YYYY", "test_ti", "500", "test_tl", "1000", "test_tf", "2000"};
assertU(adoc(doc3));
String[] doc4 = {"id","5", "term_s", "YYYY", "group_s", "group2", "test_ti", "4", "test_tl", "10", "test_tf", "2000"};
assertU(adoc(doc4));
assertU(commit());
String[] doc5 = {"id","6", "term_s","YYYY", "group_s", "group2", "test_ti", "10", "test_tl", "100", "test_tf", "200"};
assertU(adoc(doc5));
assertU(commit());
//Test collapse by score
//Test collapse by score and following sort by score
ModifiableSolrParams params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='50']");
assertQ(req(params), "*[count(//doc)=2]",
"//result/doc[1]/float[@name='id'][.='2.0']",
"//result/doc[2]/float[@name='id'][.='6.0']"
);
// SOLR-5544 test ordering with empty sort param
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s nullPolicy=expand min=test_tf}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
params.add("sort","");
assertQ(req(params), "*[count(//doc)=4]",
"//result/doc[1]/float[@name='id'][.='3.0']",
"//result/doc[2]/float[@name='id'][.='4.0']",
"//result/doc[3]/float[@name='id'][.='2.0']",
"//result/doc[4]/float[@name='id'][.='6.0']"
);
//Test collapse by score with elevation
@ -75,54 +105,152 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
params.add("bf", "field(test_ti)");
params.add("qf", "term_s");
params.add("qt", "/elevate");
assertQ(req(params), "*[count(//doc)=3]", "//doc[./int[1][@name='test_ti']='5']");
assertQ(req(params), "*[count(//doc)=4]",
"//result/doc[1]/float[@name='id'][.='1.0']");
//Test collapse by min int field
//Test collapse by min int field and sort
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s min=test_ti}");
assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='5']");
params.add("sort", "id desc");
assertQ(req(params), "*[count(//doc)=2]",
"//result/doc[1]/float[@name='id'][.='5.0']",
"//result/doc[2]/float[@name='id'][.='1.0']");
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s min=test_ti}");
params.add("sort", "id asc");
assertQ(req(params), "*[count(//doc)=2]",
"//result/doc[1]/float[@name='id'][.='1.0']",
"//result/doc[2]/float[@name='id'][.='5.0']");
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s min=test_ti}");
params.add("sort", "test_tl asc,id desc");
assertQ(req(params), "*[count(//doc)=2]",
"//result/doc[1]/float[@name='id'][.='5.0']",
"//result/doc[2]/float[@name='id'][.='1.0']");
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s min=test_ti}");
params.add("sort", "score desc,id asc");
params.add("defType", "edismax");
params.add("bf", "field(id)");
assertQ(req(params), "*[count(//doc)=2]",
"//result/doc[1]/float[@name='id'][.='5.0']",
"//result/doc[2]/float[@name='id'][.='1.0']");
//Test collapse by max int field
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s max=test_ti}");
assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='50']");
params.add("sort", "test_ti asc");
assertQ(req(params), "*[count(//doc)=2]",
"//result/doc[1]/float[@name='id'][.='6.0']",
"//result/doc[2]/float[@name='id'][.='2.0']"
);
//Test collapse by min long field
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s min=test_tl}");
assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='5']");
params.add("sort", "test_ti desc");
assertQ(req(params), "*[count(//doc)=2]",
"//result/doc[1]/float[@name='id'][.='1.0']",
"//result/doc[2]/float[@name='id'][.='5.0']");
//Test collapse by max long field
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s max=test_tl}");
assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='50']");
params.add("sort", "test_ti desc");
assertQ(req(params), "*[count(//doc)=2]",
"//result/doc[1]/float[@name='id'][.='2.0']",
"//result/doc[2]/float[@name='id'][.='6.0']");
//Test collapse by min float field
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s min=test_tf}");
assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='50']");
params.add("sort", "test_ti desc");
assertQ(req(params), "*[count(//doc)=2]",
"//result/doc[1]/float[@name='id'][.='2.0']",
"//result/doc[2]/float[@name='id'][.='6.0']");
//Test collapse by min float field
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s max=test_tf}");
assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='5']");
params.add("sort", "test_ti asc");
assertQ(req(params), "*[count(//doc)=2]",
"//result/doc[1]/float[@name='id'][.='5.0']",
"//result/doc[2]/float[@name='id'][.='1.0']");
//Test collapse by min float field sort by score
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s max=test_tf}");
params.add("defType", "edismax");
params.add("bf", "field(id)");
params.add("fl", "score, id");
params.add("facet","true");
params.add("fq", "{!tag=test}term_s:YYYY");
params.add("facet.field", "{!ex=test}term_s");
assertQ(req(params), "*[count(//doc)=2]",
"//result/doc[1]/float[@name='id'][.='5.0']",
"//result/doc[2]/float[@name='id'][.='1.0']");
//Test nullPolicy expand
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s max=test_tf nullPolicy=expand}");
assertQ(req(params), "*[count(//doc)=3]");
params.add("sort", "id desc");
assertQ(req(params), "*[count(//doc)=4]",
"//result/doc[1]/float[@name='id'][.='5.0']",
"//result/doc[2]/float[@name='id'][.='4.0']",
"//result/doc[3]/float[@name='id'][.='3.0']",
"//result/doc[4]/float[@name='id'][.='1.0']");
//Test nullPolicy collapse
params = new ModifiableSolrParams();
params.add("q", "test_ti:(500 5000)");
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s max=test_tf nullPolicy=collapse}");
assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='500']");
params.add("sort", "id desc");
assertQ(req(params), "*[count(//doc)=3]",
"//result/doc[1]/float[@name='id'][.='5.0']",
"//result/doc[2]/float[@name='id'][.='4.0']",
"//result/doc[3]/float[@name='id'][.='1.0']");
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
params.add("fq","{!tag=test_ti}id:5");
params.add("facet","true");
params.add("facet.field","{!ex=test_ti}test_ti");
params.add("facet.mincount", "1");
assertQ(req(params), "*[count(//doc)=1]", "*[count(//lst[@name='facet_fields']/lst[@name='test_ti']/int)=2]");
}
}