mirror of https://github.com/apache/lucene.git
SOLR-911: add filter tags, facet local params, with excludes, output keys, and more efficient facet refinement requests
git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@727560 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley
parent
c11997b456
commit
aa9b5753ad
|
|
@ -112,6 +112,13 @@ New Features
|
|||
of solrconfig.xml
|
||||
(Noble Paul, Akshay Ukey via shalin)
|
||||
|
||||
24. SOLR-911: Add support for multi-select faceting by allowing filters to be
|
||||
tagged and facet commands to exclude certain filters. This patch also
|
||||
added the ability to change the output key for facets in the response, and
|
||||
optimized distributed faceting refinement by lowering parsing overhead and
|
||||
by making requests and responses smaller.
|
||||
|
||||
|
||||
Optimizations
|
||||
----------------------
|
||||
1. SOLR-374: Use IndexReader.reopen to save resources by re-using parts of the
|
||||
|
|
|
|||
|
|
@ -115,5 +115,14 @@ public interface CommonParams {
|
|||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
public static final String EXCLUDE = "ex";
|
||||
public static final String TAG = "tag";
|
||||
public static final String TERMS = "terms";
|
||||
public static final String OUTPUT_KEY = "key";
|
||||
public static final String FIELD = "f";
|
||||
public static final String VALUE = "v";
|
||||
public static final String TRUE = Boolean.TRUE.toString();
|
||||
public static final String FALSE = Boolean.FALSE.toString();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -27,10 +27,10 @@ import org.apache.solr.common.params.SolrParams;
|
|||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.request.SimpleFacets;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.search.QueryParsing;
|
||||
import org.apache.lucene.queryParser.ParseException;
|
||||
|
||||
|
|
@ -64,13 +64,15 @@ public class FacetComponent extends SearchComponent
|
|||
SolrParams params = rb.req.getParams();
|
||||
SimpleFacets f = new SimpleFacets(rb.req,
|
||||
rb.getResults().docSet,
|
||||
params );
|
||||
params,
|
||||
rb );
|
||||
|
||||
// TODO ???? add this directly to the response, or to the builder?
|
||||
rb.rsp.add( "facet_counts", f.getFacetCounts() );
|
||||
}
|
||||
}
|
||||
|
||||
private static final String commandPrefix = "{!" + CommonParams.TERMS + "=$";
|
||||
|
||||
@Override
|
||||
public int distributedProcess(ResponseBuilder rb) throws IOException {
|
||||
|
|
@ -86,12 +88,42 @@ public class FacetComponent extends SearchComponent
|
|||
// We do this in distributedProcess so we can look at all of the
|
||||
// requests in the outgoing queue at once.
|
||||
|
||||
|
||||
|
||||
for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
|
||||
List<String> fqueries = rb._facetInfo._toRefine[shardNum];
|
||||
if (fqueries == null || fqueries.size()==0) continue;
|
||||
List<String> refinements = null;
|
||||
|
||||
for (DistribFieldFacet dff : rb._facetInfo.facets.values()) {
|
||||
if (!dff.needRefinements) continue;
|
||||
List<String> refList = dff._toRefine[shardNum];
|
||||
if (refList == null | refList.size()==0) continue;
|
||||
|
||||
String key = dff.getKey(); // reuse the same key that was used for the main facet
|
||||
String termsKey = key + "__terms";
|
||||
String termsVal = StrUtils.join(refList, ',');
|
||||
|
||||
String facetCommand;
|
||||
// add terms into the original facet.field command
|
||||
// do it via parameter reference to avoid another layer of encoding.
|
||||
if (dff.localParams != null) {
|
||||
facetCommand = commandPrefix+termsKey+dff.facetStr.substring(2);
|
||||
} else {
|
||||
facetCommand = commandPrefix+termsKey+'}'+dff.field;
|
||||
}
|
||||
|
||||
if (refinements == null) {
|
||||
refinements = new ArrayList<String>();
|
||||
}
|
||||
|
||||
refinements.add(facetCommand);
|
||||
refinements.add(termsKey);
|
||||
refinements.add(termsVal);
|
||||
}
|
||||
|
||||
if (refinements == null) continue;
|
||||
|
||||
|
||||
String shard = rb.shards[shardNum];
|
||||
|
||||
ShardRequest refine = null;
|
||||
boolean newRequest = false;
|
||||
|
||||
|
|
@ -100,7 +132,7 @@ public class FacetComponent extends SearchComponent
|
|||
// scalability.
|
||||
for (ShardRequest sreq : rb.outgoing) {
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS)!=0
|
||||
&& sreq.shards != null
|
||||
&& sreq.shards != null
|
||||
&& sreq.shards.length==1
|
||||
&& sreq.shards[0].equals(shard))
|
||||
{
|
||||
|
|
@ -124,8 +156,16 @@ public class FacetComponent extends SearchComponent
|
|||
refine.purpose |= ShardRequest.PURPOSE_REFINE_FACETS;
|
||||
refine.params.set(FacetParams.FACET, "true");
|
||||
refine.params.remove(FacetParams.FACET_FIELD);
|
||||
// TODO: perhaps create a more compact facet.terms method?
|
||||
refine.params.set(FacetParams.FACET_QUERY, fqueries.toArray(new String[fqueries.size()]));
|
||||
refine.params.remove(FacetParams.FACET_QUERY);
|
||||
|
||||
for (int i=0; i<refinements.size();) {
|
||||
String facetCommand=refinements.get(i++);
|
||||
String termsKey=refinements.get(i++);
|
||||
String termsVal=refinements.get(i++);
|
||||
|
||||
refine.params.add(FacetParams.FACET_FIELD, facetCommand);
|
||||
refine.params.set(termsKey, termsVal);
|
||||
}
|
||||
|
||||
if (newRequest) {
|
||||
rb.addRequest(this, refine);
|
||||
|
|
@ -148,7 +188,7 @@ public class FacetComponent extends SearchComponent
|
|||
rb._facetInfo = fi = new FacetInfo();
|
||||
fi.parse(rb.req.getParams(), rb);
|
||||
// should already be true...
|
||||
// sreq.params.set(FacetParams.FACET, FacetParams.FACET_SORT_COUNT_LEGACY);
|
||||
// sreq.params.set(FacetParams.FACET, "true");
|
||||
}
|
||||
|
||||
sreq.params.remove(FacetParams.FACET_MINCOUNT);
|
||||
|
|
@ -206,18 +246,17 @@ public class FacetComponent extends SearchComponent
|
|||
NamedList facet_queries = (NamedList)facet_counts.get("facet_queries");
|
||||
if (facet_queries != null) {
|
||||
for (int i=0; i<facet_queries.size(); i++) {
|
||||
String facet_q = (String)facet_queries.getName(i);
|
||||
String returnedKey = (String)facet_queries.getName(i);
|
||||
long count = ((Number)facet_queries.getVal(i)).longValue();
|
||||
Long prevCount = fi.queryFacets.get(facet_q);
|
||||
if (prevCount != null) count += prevCount;
|
||||
fi.queryFacets.put(facet_q, count);
|
||||
QueryFacet qf = fi.queryFacets.get(returnedKey);
|
||||
qf.count += count;
|
||||
}
|
||||
}
|
||||
|
||||
// step through each facet.field, adding results from this shard
|
||||
NamedList facet_fields = (NamedList)facet_counts.get("facet_fields");
|
||||
for (DistribFieldFacet dff : fi.facets.values()) {
|
||||
dff.add(shardNum, (NamedList)facet_fields.get(dff.field), dff.initialLimit);
|
||||
dff.add(shardNum, (NamedList)facet_fields.get(dff.getKey()), dff.initialLimit);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -228,24 +267,17 @@ public class FacetComponent extends SearchComponent
|
|||
// otherwise we would need to wait until all facet responses were received.
|
||||
//
|
||||
|
||||
// list of queries to send each shard
|
||||
List<String>[] toRefine = new List[rb.shards.length];
|
||||
fi._toRefine = toRefine;
|
||||
for (int i=0; i<toRefine.length; i++) {
|
||||
toRefine[i] = new ArrayList<String>();
|
||||
}
|
||||
|
||||
|
||||
for (DistribFieldFacet dff : fi.facets.values()) {
|
||||
if (dff.limit <= 0) continue; // no need to check these facets for refinement
|
||||
if (dff.minCount <= 1 && dff.sort.equals(FacetParams.FACET_SORT_LEX)) continue;
|
||||
|
||||
dff._toRefine = new List[rb.shards.length];
|
||||
ShardFacetCount[] counts = dff.getCountSorted();
|
||||
int ntop = Math.min(counts.length, dff.offset + dff.limit);
|
||||
long smallestCount = counts.length == 0 ? 0 : counts[ntop-1].count;
|
||||
|
||||
for (int i=0; i<counts.length; i++) {
|
||||
ShardFacetCount sfc = counts[i];
|
||||
String query = null;
|
||||
boolean needRefinement = false;
|
||||
|
||||
if (i<ntop) {
|
||||
|
|
@ -274,8 +306,11 @@ public class FacetComponent extends SearchComponent
|
|||
OpenBitSet obs = dff.counted[shardNum];
|
||||
if (!obs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) {
|
||||
dff.needRefinements = true;
|
||||
if (query==null) query = dff.makeQuery(sfc);
|
||||
toRefine[shardNum].add(query);
|
||||
List<String> lst = dff._toRefine[shardNum];
|
||||
if (lst == null) {
|
||||
lst = dff._toRefine[shardNum] = new ArrayList<String>();
|
||||
}
|
||||
lst.add(sfc.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -290,44 +325,20 @@ public class FacetComponent extends SearchComponent
|
|||
for (ShardResponse srsp: sreq.responses) {
|
||||
// int shardNum = rb.getShardNum(srsp.shard);
|
||||
NamedList facet_counts = (NamedList)srsp.getSolrResponse().getResponse().get("facet_counts");
|
||||
NamedList facet_queries = (NamedList)facet_counts.get("facet_queries");
|
||||
NamedList facet_fields = (NamedList)facet_counts.get("facet_fields");
|
||||
|
||||
// These are single term queries used to fill in missing counts
|
||||
// for facet.field queries
|
||||
for (int i=0; i<facet_queries.size(); i++) {
|
||||
try {
|
||||
|
||||
String facet_q = (String)facet_queries.getName(i);
|
||||
long count = ((Number)facet_queries.getVal(i)).longValue();
|
||||
for (int i=0; i<facet_fields.size(); i++) {
|
||||
String key = facet_fields.getName(i);
|
||||
DistribFieldFacet dff = (DistribFieldFacet)fi.facets.get(key);
|
||||
if (dff == null) continue;
|
||||
|
||||
// expect {!field f=field}value style params
|
||||
SolrParams qparams = QueryParsing.getLocalParams(facet_q,null);
|
||||
if (qparams == null) continue; // not a refinement
|
||||
String field = qparams.get(QueryParsing.F);
|
||||
String val = qparams.get(QueryParsing.V);
|
||||
|
||||
// Find the right field.facet for this field
|
||||
DistribFieldFacet dff = fi.facets.get(field);
|
||||
if (dff == null) continue; // maybe this wasn't for facet count refinement
|
||||
|
||||
// Find the right constraint count for this value
|
||||
ShardFacetCount sfc = dff.counts.get(val);
|
||||
|
||||
if (sfc == null) {
|
||||
continue;
|
||||
// Just continue, since other components might have added
|
||||
// this facet.query for other purposes. But if there are charset
|
||||
// issues then the values coming back may not match the values sent.
|
||||
}
|
||||
|
||||
// TODO REMOVE
|
||||
// System.out.println("Got " + facet_q + " , refining count: " + sfc + " += " + count);
|
||||
NamedList shardCounts = (NamedList)facet_fields.getVal(i);
|
||||
|
||||
for (int j=0; j<shardCounts.size(); j++) {
|
||||
String name = shardCounts.getName(j);
|
||||
long count = ((Number)shardCounts.getVal(j)).longValue();
|
||||
ShardFacetCount sfc = dff.counts.get(name);
|
||||
sfc.count += count;
|
||||
|
||||
} catch (ParseException e) {
|
||||
// shouldn't happen, so fail for now rather than covering it up
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -345,8 +356,8 @@ public class FacetComponent extends SearchComponent
|
|||
NamedList facet_counts = new SimpleOrderedMap();
|
||||
NamedList facet_queries = new SimpleOrderedMap();
|
||||
facet_counts.add("facet_queries",facet_queries);
|
||||
for (Map.Entry<String,Long> entry : fi.queryFacets.entrySet()) {
|
||||
facet_queries.add(entry.getKey(), num(entry.getValue()));
|
||||
for (QueryFacet qf : fi.queryFacets.values()) {
|
||||
facet_queries.add(qf.getKey(), num(qf.count));
|
||||
}
|
||||
|
||||
NamedList facet_fields = new SimpleOrderedMap();
|
||||
|
|
@ -354,7 +365,7 @@ public class FacetComponent extends SearchComponent
|
|||
|
||||
for (DistribFieldFacet dff : fi.facets.values()) {
|
||||
NamedList fieldCounts = new NamedList(); // order is more important for facets
|
||||
facet_fields.add(dff.field, fieldCounts);
|
||||
facet_fields.add(dff.getKey(), fieldCounts);
|
||||
|
||||
ShardFacetCount[] counts;
|
||||
if (dff.sort.equals(FacetParams.FACET_SORT_COUNT)) {
|
||||
|
|
@ -379,7 +390,6 @@ public class FacetComponent extends SearchComponent
|
|||
}
|
||||
}
|
||||
|
||||
// TODO: list facets (sorted by natural order)
|
||||
// TODO: facet dates
|
||||
facet_counts.add("facet_dates", new SimpleOrderedMap());
|
||||
|
||||
|
|
@ -433,36 +443,76 @@ public class FacetComponent extends SearchComponent
|
|||
|
||||
|
||||
class FacetInfo {
|
||||
List<String>[] _toRefine;
|
||||
|
||||
LinkedHashMap<String,QueryFacet> queryFacets;
|
||||
LinkedHashMap<String,DistribFieldFacet> facets;
|
||||
|
||||
void parse(SolrParams params, ResponseBuilder rb) {
|
||||
queryFacets = new LinkedHashMap<String,Long>();
|
||||
queryFacets = new LinkedHashMap<String,QueryFacet>();
|
||||
facets = new LinkedHashMap<String,DistribFieldFacet>();
|
||||
|
||||
String[] facetQs = params.getParams(FacetParams.FACET_QUERY);
|
||||
if (facetQs != null) {
|
||||
for (String query : facetQs) {
|
||||
queryFacets.put(query,0L);
|
||||
QueryFacet queryFacet = new QueryFacet(rb, query);
|
||||
queryFacets.put(queryFacet.getKey(), queryFacet);
|
||||
}
|
||||
}
|
||||
|
||||
String[] facetFs = params.getParams(FacetParams.FACET_FIELD);
|
||||
if (facetFs != null) {
|
||||
|
||||
for (String field : facetFs) {
|
||||
DistribFieldFacet ff = new DistribFieldFacet(rb, field);
|
||||
ff.fillParams(params, field);
|
||||
facets.put(field, ff);
|
||||
facets.put(ff.getKey(), ff);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
LinkedHashMap<String,Long> queryFacets;
|
||||
LinkedHashMap<String,DistribFieldFacet> facets;
|
||||
}
|
||||
|
||||
class FacetBase {
|
||||
String facetType; // facet.field, facet.query, etc (make enum?)
|
||||
String facetStr; // original parameter value of facetStr
|
||||
String facetOn; // the field or query, absent localParams if appropriate
|
||||
private String key; // label in the response for the result... "foo" for {!key=foo}myfield
|
||||
SolrParams localParams; // any local params for the facet
|
||||
|
||||
class FieldFacet {
|
||||
String field;
|
||||
public FacetBase(ResponseBuilder rb, String facetType, String facetStr) {
|
||||
this.facetType = facetType;
|
||||
this.facetStr = facetStr;
|
||||
try {
|
||||
this.localParams = QueryParsing.getLocalParams(facetStr, rb.req.getParams());
|
||||
} catch (ParseException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
|
||||
}
|
||||
this.facetOn = facetStr;
|
||||
this.key = facetStr;
|
||||
|
||||
if (localParams != null) {
|
||||
// remove local params unless it's a query
|
||||
if (!facetType.equals(FacetParams.FACET_QUERY)) {
|
||||
facetOn = localParams.get(CommonParams.VALUE);
|
||||
key = facetOn;
|
||||
}
|
||||
|
||||
key = localParams.get(CommonParams.OUTPUT_KEY, key);
|
||||
}
|
||||
}
|
||||
|
||||
/** returns the key in the response that this facet will be under */
|
||||
String getKey() { return key; }
|
||||
String getType() { return facetType; }
|
||||
}
|
||||
|
||||
class QueryFacet extends FacetBase {
|
||||
long count;
|
||||
|
||||
public QueryFacet(ResponseBuilder rb, String facetStr) {
|
||||
super(rb, FacetParams.FACET_QUERY, facetStr);
|
||||
}
|
||||
}
|
||||
|
||||
class FieldFacet extends FacetBase {
|
||||
String field; // the field to facet on... "myfield" for {!key=foo}myfield
|
||||
int offset;
|
||||
int limit;
|
||||
int minCount;
|
||||
|
|
@ -471,7 +521,12 @@ class FieldFacet {
|
|||
String prefix;
|
||||
long missingCount;
|
||||
|
||||
void fillParams(SolrParams params, String field) {
|
||||
public FieldFacet(ResponseBuilder rb, String facetStr) {
|
||||
super(rb, FacetParams.FACET_FIELD, facetStr);
|
||||
fillParams(rb.req.getParams(), facetOn);
|
||||
}
|
||||
|
||||
private void fillParams(SolrParams params, String field) {
|
||||
this.field = field;
|
||||
this.offset = params.getFieldInt(field, FacetParams.FACET_OFFSET, 0);
|
||||
this.limit = params.getFieldInt(field, FacetParams.FACET_LIMIT, 100);
|
||||
|
|
@ -496,7 +551,9 @@ class FieldFacet {
|
|||
}
|
||||
|
||||
class DistribFieldFacet extends FieldFacet {
|
||||
SchemaField sf;
|
||||
List<String>[] _toRefine; // a List<String> of refinements needed, one for each shard.
|
||||
|
||||
// SchemaField sf; // currently unneeded
|
||||
|
||||
// the max possible count for a term appearing on no list
|
||||
long missingMaxPossible;
|
||||
|
|
@ -505,17 +562,16 @@ class DistribFieldFacet extends FieldFacet {
|
|||
OpenBitSet[] counted; // a bitset for each shard, keeping track of which terms seen
|
||||
HashMap<String,ShardFacetCount> counts = new HashMap<String,ShardFacetCount>(128);
|
||||
int termNum;
|
||||
String queryPrefix;
|
||||
|
||||
int initialLimit; // how many terms requested in first phase
|
||||
boolean needRefinements;
|
||||
ShardFacetCount[] countSorted;
|
||||
|
||||
DistribFieldFacet(ResponseBuilder rb, String field) {
|
||||
sf = rb.req.getSchema().getField(field);
|
||||
DistribFieldFacet(ResponseBuilder rb, String facetStr) {
|
||||
super(rb, facetStr);
|
||||
// sf = rb.req.getSchema().getField(field);
|
||||
missingMax = new long[rb.shards.length];
|
||||
counted = new OpenBitSet[rb.shards.length];
|
||||
queryPrefix = "{!field f=" + field + '}';
|
||||
}
|
||||
|
||||
void add(int shardNum, NamedList shardCounts, int numRequested) {
|
||||
|
|
@ -582,10 +638,6 @@ class DistribFieldFacet extends FieldFacet {
|
|||
return arr;
|
||||
}
|
||||
|
||||
String makeQuery(ShardFacetCount sfc) {
|
||||
return queryPrefix + sfc.name;
|
||||
}
|
||||
|
||||
// returns the max possible value this ShardFacetCount could have for this shard
|
||||
// (assumes the shard did not report a count for this value)
|
||||
long maxPossible(ShardFacetCount sfc, int shardNum) {
|
||||
|
|
@ -593,7 +645,6 @@ class DistribFieldFacet extends FieldFacet {
|
|||
// TODO: could store the last term in the shard to tell if this term
|
||||
// comes before or after it. If it comes before, we could subtract 1
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -27,9 +27,11 @@ import org.apache.solr.common.SolrException;
|
|||
import org.apache.solr.common.params.FacetParams;
|
||||
import org.apache.solr.common.params.RequiredSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.FacetParams.FacetDateOther;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.schema.IndexSchema;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
|
|
@ -39,14 +41,10 @@ import org.apache.solr.schema.DateField;
|
|||
import org.apache.solr.search.*;
|
||||
import org.apache.solr.util.BoundedTreeSet;
|
||||
import org.apache.solr.util.DateMathParser;
|
||||
import org.apache.solr.handler.component.ResponseBuilder;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.EnumSet;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* A class that generates simple Facet information for a request.
|
||||
|
|
@ -63,16 +61,92 @@ public class SimpleFacets {
|
|||
/** Searcher to use for all calculations */
|
||||
protected SolrIndexSearcher searcher;
|
||||
protected SolrQueryRequest req;
|
||||
protected ResponseBuilder rb;
|
||||
|
||||
// per-facet values
|
||||
SolrParams localParams; // localParams on this particular facet command
|
||||
String facetValue; // the field to or query to facet on (minus local params)
|
||||
DocSet base; // the base docset for this particular facet
|
||||
String key; // what name should the results be stored under
|
||||
|
||||
public SimpleFacets(SolrQueryRequest req,
|
||||
DocSet docs,
|
||||
SolrParams params) {
|
||||
this(req,docs,params,null);
|
||||
}
|
||||
|
||||
public SimpleFacets(SolrQueryRequest req,
|
||||
DocSet docs,
|
||||
SolrParams params,
|
||||
ResponseBuilder rb) {
|
||||
this.req = req;
|
||||
this.searcher = req.getSearcher();
|
||||
this.docs = docs;
|
||||
this.base = this.docs = docs;
|
||||
this.params = params;
|
||||
this.rb = rb;
|
||||
}
|
||||
|
||||
|
||||
void parseParams(String type, String param) throws ParseException, IOException {
|
||||
localParams = QueryParsing.getLocalParams(param, req.getParams());
|
||||
base = docs;
|
||||
facetValue = param;
|
||||
key = param;
|
||||
|
||||
if (localParams == null) return;
|
||||
|
||||
// remove local params unless it's a query
|
||||
if (type != FacetParams.FACET_QUERY) {
|
||||
facetValue = localParams.get(CommonParams.VALUE);
|
||||
}
|
||||
|
||||
// reset set the default key now that localParams have been removed
|
||||
key = facetValue;
|
||||
|
||||
// allow explicit set of the key
|
||||
key = localParams.get(CommonParams.OUTPUT_KEY, key);
|
||||
|
||||
// figure out if we need a new base DocSet
|
||||
String excludeStr = localParams.get(CommonParams.EXCLUDE);
|
||||
if (excludeStr == null) return;
|
||||
|
||||
Map tagMap = (Map)req.getContext().get("tags");
|
||||
if (tagMap != null && rb != null) {
|
||||
List<String> excludeTagList = StrUtils.splitSmart(excludeStr,',');
|
||||
|
||||
IdentityHashMap<Query,Boolean> excludeSet = new IdentityHashMap<Query,Boolean>();
|
||||
for (String excludeTag : excludeTagList) {
|
||||
Object olst = tagMap.get(excludeTag);
|
||||
// tagMap has entries of List<String,List<QParser>>, but subject to change in the future
|
||||
if (!(olst instanceof Collection)) continue;
|
||||
for (Object o : (Collection)olst) {
|
||||
if (!(o instanceof QParser)) continue;
|
||||
QParser qp = (QParser)o;
|
||||
excludeSet.put(qp.getQuery(), Boolean.TRUE);
|
||||
}
|
||||
}
|
||||
if (excludeSet.size() == 0) return;
|
||||
|
||||
List<Query> qlist = new ArrayList<Query>();
|
||||
|
||||
// add the base query
|
||||
qlist.add(rb.getQuery());
|
||||
|
||||
// add the filters
|
||||
for (Query q : rb.getFilters()) {
|
||||
if (!excludeSet.containsKey(q)) {
|
||||
qlist.add(q);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// get the new base docset for this facet
|
||||
base = searcher.getDocSet(qlist);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Looks at various Params to determing if any simple Facet Constraint count
|
||||
* computations are desired.
|
||||
|
|
@ -123,8 +197,11 @@ public class SimpleFacets {
|
|||
String[] facetQs = params.getParams(FacetParams.FACET_QUERY);
|
||||
if (null != facetQs && 0 != facetQs.length) {
|
||||
for (String q : facetQs) {
|
||||
parseParams(FacetParams.FACET_QUERY, q);
|
||||
|
||||
// TODO: slight optimization would prevent double-parsing of any localParams
|
||||
Query qobj = QParser.getParser(q, null, req).getQuery();
|
||||
res.add(q, searcher.numDocs(qobj, docs));
|
||||
res.add(key, searcher.numDocs(qobj, base));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -164,15 +241,15 @@ public class SimpleFacets {
|
|||
|
||||
// unless the enum method is explicitly specified, use a counting method.
|
||||
if (enumMethod) {
|
||||
counts = getFacetTermEnumCounts(searcher, docs, field, offset, limit, mincount,missing,sort,prefix);
|
||||
counts = getFacetTermEnumCounts(searcher, base, field, offset, limit, mincount,missing,sort,prefix);
|
||||
} else {
|
||||
if (multiToken) {
|
||||
UnInvertedField uif = UnInvertedField.getUnInvertedField(field, searcher);
|
||||
counts = uif.getCounts(searcher, docs, offset, limit, mincount,missing,sort,prefix);
|
||||
counts = uif.getCounts(searcher, base, offset, limit, mincount,missing,sort,prefix);
|
||||
} else {
|
||||
// TODO: future logic could use filters instead of the fieldcache if
|
||||
// the number of terms in the field is small enough.
|
||||
counts = getFieldCacheCounts(searcher, docs, field, offset,limit, mincount, missing, sort, prefix);
|
||||
counts = getFieldCacheCounts(searcher, base, field, offset,limit, mincount, missing, sort, prefix);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -189,18 +266,39 @@ public class SimpleFacets {
|
|||
* @see #getFacetTermEnumCounts
|
||||
*/
|
||||
public NamedList getFacetFieldCounts()
|
||||
throws IOException {
|
||||
throws IOException, ParseException {
|
||||
|
||||
NamedList res = new SimpleOrderedMap();
|
||||
String[] facetFs = params.getParams(FacetParams.FACET_FIELD);
|
||||
if (null != facetFs) {
|
||||
for (String f : facetFs) {
|
||||
res.add(f, getTermCounts(f));
|
||||
parseParams(FacetParams.FACET_FIELD, f);
|
||||
String termList = localParams == null ? null : localParams.get(CommonParams.TERMS);
|
||||
if (termList != null) {
|
||||
res.add(key, getListedTermCounts(facetValue, termList));
|
||||
} else {
|
||||
res.add(key, getTermCounts(facetValue));
|
||||
}
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
private NamedList getListedTermCounts(String field, String termList) throws IOException {
|
||||
FieldType ft = searcher.getSchema().getFieldType(field);
|
||||
List<String> terms = StrUtils.splitSmart(termList, ",", true);
|
||||
NamedList res = new NamedList();
|
||||
Term t = new Term(field);
|
||||
for (String term : terms) {
|
||||
String internal = ft.toInternal(term);
|
||||
int count = searcher.numDocs(new TermQuery(t.createTerm(internal)), base);
|
||||
res.add(term, count);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns a count of the documents in the set which do not have any
|
||||
* terms for for the specified field.
|
||||
|
|
@ -441,7 +539,7 @@ public class SimpleFacets {
|
|||
* @see FacetParams#FACET_DATE
|
||||
*/
|
||||
public NamedList getFacetDateCounts()
|
||||
throws IOException {
|
||||
throws IOException, ParseException {
|
||||
|
||||
final SolrParams required = new RequiredSolrParams(params);
|
||||
final NamedList resOuter = new SimpleOrderedMap();
|
||||
|
|
@ -452,8 +550,12 @@ public class SimpleFacets {
|
|||
|
||||
final IndexSchema schema = searcher.getSchema();
|
||||
for (String f : fields) {
|
||||
parseParams(FacetParams.FACET_DATE, f);
|
||||
f = facetValue;
|
||||
|
||||
|
||||
final NamedList resInner = new SimpleOrderedMap();
|
||||
resOuter.add(f, resInner);
|
||||
resOuter.add(key, resInner);
|
||||
final FieldType trash = schema.getFieldType(f);
|
||||
if (! (trash instanceof DateField)) {
|
||||
throw new SolrException
|
||||
|
|
@ -571,7 +673,7 @@ public class SimpleFacets {
|
|||
boolean iLow, boolean iHigh) throws IOException {
|
||||
return searcher.numDocs(new ConstantScoreRangeQuery(field,low,high,
|
||||
iLow,iHigh),
|
||||
docs);
|
||||
base);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -22,8 +22,11 @@ import org.apache.lucene.search.Sort;
|
|||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public abstract class QParser {
|
||||
String qstr;
|
||||
SolrParams params;
|
||||
|
|
@ -33,13 +36,48 @@ public abstract class QParser {
|
|||
|
||||
Query query;
|
||||
|
||||
|
||||
public QParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
|
||||
this.qstr = qstr;
|
||||
this.localParams = localParams;
|
||||
|
||||
// insert tags into tagmap.
|
||||
// WARNING: the internal representation of tagged objects in the request context is
|
||||
// experimental and subject to change!
|
||||
if (localParams != null) {
|
||||
String tagStr = localParams.get("tag");
|
||||
if (tagStr != null) {
|
||||
Map context = req.getContext();
|
||||
Map<String,Collection<Object>> tagMap = (Map<String, Collection<Object>>)req.getContext().get("tags");
|
||||
if (tagMap == null) {
|
||||
tagMap = new HashMap<String,Collection<Object>>();
|
||||
context.put("tags", tagMap);
|
||||
}
|
||||
if (tagStr.indexOf(',') >= 0) {
|
||||
List<String> tags = StrUtils.splitSmart(tagStr, ',');
|
||||
for (String tag : tags) {
|
||||
addTag(tagMap, tag, this);
|
||||
}
|
||||
} else {
|
||||
addTag(tagMap, tagStr, this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.params = params;
|
||||
this.req = req;
|
||||
}
|
||||
|
||||
|
||||
private static void addTag(Map tagMap, Object key, Object val) {
|
||||
Collection lst = (Collection)tagMap.get(key);
|
||||
if (lst == null) {
|
||||
lst = new ArrayList(2);
|
||||
tagMap.put(key, lst);
|
||||
}
|
||||
lst.add(val);
|
||||
}
|
||||
|
||||
/** Create and return the <code>Query</code> object represented by <code>qstr</code>
|
||||
* @see #getQuery()
|
||||
**/
|
||||
|
|
|
|||
|
|
@ -548,6 +548,15 @@ public class TestDistributedSearch extends TestCase {
|
|||
query("q","*:*", "rows",100, "facet","true", "facet.query","quick", "facet.query","all", "facet.query","*:*"
|
||||
,"facet.field",t1);
|
||||
|
||||
// test filter tagging, facet exclusion, and naming (multi-select facet support)
|
||||
query("q","*:*", "rows",100, "facet","true", "facet.query","{!key=myquick}quick", "facet.query","{!key=myall ex=a}all", "facet.query","*:*"
|
||||
,"facet.field","{!key=mykey ex=a}"+t1
|
||||
,"facet.field","{!key=other ex=b}"+t1
|
||||
,"facet.field","{!key=again ex=a,b}"+t1
|
||||
,"facet.field",t1
|
||||
,"fq","{!tag=a}id:[1 TO 7]", "fq","{!tag=b}id:[3 TO 9]"
|
||||
);
|
||||
|
||||
// test field that is valid in schema but missing in all shards
|
||||
query("q","*:*", "rows",100, "facet","true", "facet.field",missingField, "facet.mincount",2);
|
||||
// test field that is valid in schema and missing in some shards
|
||||
|
|
|
|||
|
|
@ -86,7 +86,31 @@ public class SimpleFacetsTest extends AbstractSolrTestCase {
|
|||
,"//lst[@name='trait_s']/int[@name='Obnoxious'][.='2']"
|
||||
,"//lst[@name='trait_s']/int[@name='Pig'][.='1']"
|
||||
);
|
||||
|
||||
|
||||
assertQ("check multi-select facets with naming",
|
||||
req("q", "id:[42 TO 47]"
|
||||
,"facet", "true"
|
||||
,"facet.query", "{!ex=1}trait_s:Obnoxious"
|
||||
,"facet.query", "{!ex=2 key=foo}id:[42 TO 45]" // tag=2 same as 1
|
||||
,"facet.query", "{!ex=3,4 key=bar}id:[43 TO 47]" // tag=3,4 don't exist
|
||||
,"facet.field", "{!ex=3,1}trait_s" // 3,1 same as 1
|
||||
,"fq", "{!tag=1,2}id:47" // tagged as 1 and 2
|
||||
)
|
||||
,"*[count(//doc)=1]"
|
||||
|
||||
,"//lst[@name='facet_counts']/lst[@name='facet_queries']"
|
||||
,"//lst[@name='facet_queries']/int[@name='{!ex=1}trait_s:Obnoxious'][.='2']"
|
||||
,"//lst[@name='facet_queries']/int[@name='foo'][.='4']"
|
||||
,"//lst[@name='facet_queries']/int[@name='bar'][.='1']"
|
||||
|
||||
,"//lst[@name='facet_counts']/lst[@name='facet_fields']"
|
||||
,"//lst[@name='facet_fields']/lst[@name='trait_s']"
|
||||
,"*[count(//lst[@name='trait_s']/int)=4]"
|
||||
,"//lst[@name='trait_s']/int[@name='Tool'][.='2']"
|
||||
,"//lst[@name='trait_s']/int[@name='Obnoxious'][.='2']"
|
||||
,"//lst[@name='trait_s']/int[@name='Pig'][.='1']"
|
||||
);
|
||||
|
||||
assertQ("check counts for applied facet queries using filtering (fq)",
|
||||
req("q", "id:[42 TO 47]"
|
||||
,"facet", "true"
|
||||
|
|
|
|||
Loading…
Reference in New Issue