Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-4212: SOLR-6353: Let facet queries and facet ranges hang off of pivots 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1689802 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shalin Shekhar Mangar 2015-07-08 07:46:09 +00:00
parent 1f2b51cd96
commit 32c740005a
17 changed files with 3437 additions and 1132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -155,6 +155,9 @@ New Features
* SOLR-7651: New response format added wt=smile (noble)
* SOLR-4212: SOLR-6353: Let facet queries and facet ranges hang off of pivots. Example:
facet.range={!tag=r1}price&facet.query={!tag=q1}somequery&facet.pivot={!range=r1 query=q1}category,manufacturer
Bug Fixes
----------------------

6
solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
View File

@ -37,6 +37,10 @@ import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.FacetComponent;
import org.apache.solr.handler.component.SpatialHeatmapFacets;
import org.apache.solr.handler.component.DateFacetProcessor;
import org.apache.solr.handler.component.RangeFacetProcessor;
import org.apache.solr.request.SimpleFacets;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@ -231,7 +235,7 @@ public class MoreLikeThisHandler extends RequestHandlerBase
rsp.add("facet_counts", null);
} else {
SimpleFacets f = new SimpleFacets(req, mltDocs.docSet, params);
rsp.add("facet_counts", f.getFacetCounts());
rsp.add("facet_counts", FacetComponent.getFacetCounts(f));
}
}
boolean dbg = req.getParams().getBool(CommonParams.DEBUG_QUERY, false);

253
solr/core/src/java/org/apache/solr/handler/component/DateFacetProcessor.java Normal file
View File

@ -0,0 +1,253 @@
package org.apache.solr.handler.component;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.Date;
import java.util.EnumSet;
import java.util.Set;
import org.apache.lucene.search.Query;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.request.SimpleFacets;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TrieDateField;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SyntaxError;
import org.apache.solr.util.DateMathParser;
/**
* Process date facets
*
* @deprecated the whole date faceting feature is deprecated. Use range facets instead which can
* already work with dates.
*/
@Deprecated
public class DateFacetProcessor extends SimpleFacets {
public DateFacetProcessor(SolrQueryRequest req, DocSet docs, SolrParams params, ResponseBuilder rb) {
super(req, docs, params, rb);
}
/**
* @deprecated Use getFacetRangeCounts which is more generalized
*/
@Deprecated
public void getFacetDateCounts(String dateFacet, NamedList<Object> resOuter)
throws IOException {
final IndexSchema schema = searcher.getSchema();
ParsedParams parsed = null;
try {
parsed = parseParams(FacetParams.FACET_DATE, dateFacet);
} catch (SyntaxError syntaxError) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, syntaxError);
}
final SolrParams params = parsed.params;
final SolrParams required = parsed.required;
final String key = parsed.key;
final String f = parsed.facetValue;
final NamedList<Object> resInner = new SimpleOrderedMap<>();
resOuter.add(key, resInner);
final SchemaField sf = schema.getField(f);
if (!(sf.getType() instanceof TrieDateField)) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"Can not date facet on a field which is not a TrieDateField: " + f);
}
final TrieDateField ft = (TrieDateField) sf.getType();
final String startS
= required.getFieldParam(f, FacetParams.FACET_DATE_START);
final Date start;
try {
start = ft.parseMath(null, startS);
} catch (SolrException e) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"date facet 'start' is not a valid Date string: " + startS, e);
}
final String endS
= required.getFieldParam(f, FacetParams.FACET_DATE_END);
Date end; // not final, hardend may change this
try {
end = ft.parseMath(null, endS);
} catch (SolrException e) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"date facet 'end' is not a valid Date string: " + endS, e);
}
if (end.before(start)) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"date facet 'end' comes before 'start': " + endS + " < " + startS);
}
final String gap = required.getFieldParam(f, FacetParams.FACET_DATE_GAP);
final DateMathParser dmp = new DateMathParser();
final int minCount = params.getFieldInt(f, FacetParams.FACET_MINCOUNT, 0);
String[] iStrs = params.getFieldParams(f, FacetParams.FACET_DATE_INCLUDE);
// Legacy support for default of [lower,upper,edge] for date faceting
// this is not handled by FacetRangeInclude.parseParam because
// range faceting has differnet defaults
final EnumSet<FacetParams.FacetRangeInclude> include =
(null == iStrs || 0 == iStrs.length) ?
EnumSet.of(FacetParams.FacetRangeInclude.LOWER,
FacetParams.FacetRangeInclude.UPPER,
FacetParams.FacetRangeInclude.EDGE)
: FacetParams.FacetRangeInclude.parseParam(iStrs);
try {
Date low = start;
while (low.before(end)) {
dmp.setNow(low);
String label = ft.toExternal(low);
Date high = dmp.parseMath(gap);
if (end.before(high)) {
if (params.getFieldBool(f, FacetParams.FACET_DATE_HARD_END, false)) {
high = end;
} else {
end = high;
}
}
if (high.before(low)) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"date facet infinite loop (is gap negative?)");
}
if (high.equals(low)) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"date facet infinite loop: gap is effectively zero");
}
final boolean includeLower =
(include.contains(FacetParams.FacetRangeInclude.LOWER) ||
(include.contains(FacetParams.FacetRangeInclude.EDGE) && low.equals(start)));
final boolean includeUpper =
(include.contains(FacetParams.FacetRangeInclude.UPPER) ||
(include.contains(FacetParams.FacetRangeInclude.EDGE) && high.equals(end)));
final int count = rangeCount(parsed, sf, low, high, includeLower, includeUpper);
if (count >= minCount) {
resInner.add(label, count);
}
low = high;
}
} catch (java.text.ParseException e) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"date facet 'gap' is not a valid Date Math string: " + gap, e);
}
// explicitly return the gap and end so all the counts
// (including before/after/between) are meaningful - even if mincount
// has removed the neighboring ranges
resInner.add("gap", gap);
resInner.add("start", start);
resInner.add("end", end);
final String[] othersP =
params.getFieldParams(f, FacetParams.FACET_DATE_OTHER);
if (null != othersP && 0 < othersP.length) {
final Set<FacetParams.FacetRangeOther> others = EnumSet.noneOf(FacetParams.FacetRangeOther.class);
for (final String o : othersP) {
others.add(FacetParams.FacetRangeOther.get(o));
}
// no matter what other values are listed, we don't do
// anything if "none" is specified.
if (!others.contains(FacetParams.FacetRangeOther.NONE)) {
boolean all = others.contains(FacetParams.FacetRangeOther.ALL);
if (all || others.contains(FacetParams.FacetRangeOther.BEFORE)) {
// include upper bound if "outer" or if first gap doesn't already include it
resInner.add(FacetParams.FacetRangeOther.BEFORE.toString(),
rangeCount(parsed, sf, null, start,
false,
(include.contains(FacetParams.FacetRangeInclude.OUTER) ||
(!(include.contains(FacetParams.FacetRangeInclude.LOWER) ||
include.contains(FacetParams.FacetRangeInclude.EDGE))))));
}
if (all || others.contains(FacetParams.FacetRangeOther.AFTER)) {
// include lower bound if "outer" or if last gap doesn't already include it
resInner.add(FacetParams.FacetRangeOther.AFTER.toString(),
rangeCount(parsed, sf, end, null,
(include.contains(FacetParams.FacetRangeInclude.OUTER) ||
(!(include.contains(FacetParams.FacetRangeInclude.UPPER) ||
include.contains(FacetParams.FacetRangeInclude.EDGE)))),
false));
}
if (all || others.contains(FacetParams.FacetRangeOther.BETWEEN)) {
resInner.add(FacetParams.FacetRangeOther.BETWEEN.toString(),
rangeCount(parsed, sf, start, end,
(include.contains(FacetParams.FacetRangeInclude.LOWER) ||
include.contains(FacetParams.FacetRangeInclude.EDGE)),
(include.contains(FacetParams.FacetRangeInclude.UPPER) ||
include.contains(FacetParams.FacetRangeInclude.EDGE))));
}
}
}
}
/**
* Returns a list of value constraints and the associated facet counts
* for each facet date field, range, and interval specified in the
* SolrParams
*
* @see FacetParams#FACET_DATE
* @deprecated Use getFacetRangeCounts which is more generalized
*/
@Deprecated
public NamedList<Object> getFacetDateCounts()
throws IOException {
final NamedList<Object> resOuter = new SimpleOrderedMap<>();
final String[] fields = global.getParams(FacetParams.FACET_DATE);
if (null == fields || 0 == fields.length) return resOuter;
for (String f : fields) {
getFacetDateCounts(f, resOuter);
}
return resOuter;
}
/**
* @deprecated Use rangeCount(SchemaField,String,String,boolean,boolean) which is more generalized
*/
@Deprecated
protected int rangeCount(ParsedParams parsed, SchemaField sf, Date low, Date high,
boolean iLow, boolean iHigh) throws IOException {
Query rangeQ = ((TrieDateField) (sf.getType())).getRangeQuery(null, sf, low, high, iLow, iHigh);
return searcher.numDocs(rangeQ, parsed.docs);
}
}

397
solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
View File

@ -32,12 +32,13 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.FacetParams.FacetRangeOther;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
@ -45,6 +46,7 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.request.SimpleFacets;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.FieldType;
import org.apache.solr.search.QueryParsing;
import org.apache.solr.search.SyntaxError;
@ -61,17 +63,183 @@ public class FacetComponent extends SearchComponent {
public static Logger log = LoggerFactory.getLogger(FacetComponent.class);
public static final String COMPONENT_NAME = "facet";
public static final String FACET_QUERY_KEY = "facet_queries";
public static final String FACET_FIELD_KEY = "facet_fields";
public static final String FACET_DATE_KEY = "facet_dates";
public static final String FACET_RANGES_KEY = "facet_ranges";
public static final String FACET_INTERVALS_KEY = "facet_intervals";
private static final String PIVOT_KEY = "facet_pivot";
private static final String PIVOT_REFINE_PREFIX = "{!"+PivotFacet.REFINE_PARAM+"=";
@Override
public void prepare(ResponseBuilder rb) throws IOException {
if (rb.req.getParams().getBool(FacetParams.FACET, false)) {
rb.setNeedDocSet(true);
rb.doFacets = true;
// Deduplicate facet params
ModifiableSolrParams params = new ModifiableSolrParams();
SolrParams origParams = rb.req.getParams();
Iterator<String> iter = origParams.getParameterNamesIterator();
while (iter.hasNext()) {
String paramName = iter.next();
// Deduplicate the list with LinkedHashSet, but _only_ for facet params.
if (!paramName.startsWith(FacetParams.FACET)) {
params.add(paramName, origParams.getParams(paramName));
continue;
}
HashSet<String> deDupe = new LinkedHashSet<>(Arrays.asList(origParams.getParams(paramName)));
params.add(paramName, deDupe.toArray(new String[deDupe.size()]));
}
rb.req.setParams(params);
// Initialize context
FacetContext.initContext(rb);
}
}
/**
* Encapsulates facet ranges and facet queries such that their parameters
* are parsed and cached for efficient re-use.
* <p>
* An instance of this class is initialized and kept in the request context via the static
* method {@link org.apache.solr.handler.component.FacetComponent.FacetContext#initContext(ResponseBuilder)} and
* can be retrieved via {@link org.apache.solr.handler.component.FacetComponent.FacetContext#getFacetContext(SolrQueryRequest)}
* <p>
* This class is used exclusively in a single-node context (i.e. non distributed requests or an individual shard
* request). Also see {@link org.apache.solr.handler.component.FacetComponent.FacetInfo} which is
* dedicated exclusively for merging responses from multiple shards and plays no role during computation of facet
* counts in a single node request.
*
* <b>This API is experimental and subject to change</b>
*
* @see org.apache.solr.handler.component.FacetComponent.FacetInfo
*/
public static class FacetContext {
private static final String FACET_CONTEXT_KEY = "_facet.context";
private final List<RangeFacetRequest> allRangeFacets; // init in constructor
private final List<FacetBase> allQueryFacets; // init in constructor
private final Map<String, List<RangeFacetRequest>> taggedRangeFacets;
private final Map<String, List<FacetBase>> taggedQueryFacets;
/**
* Initializes FacetContext using request parameters and saves it in the request
* context which can be retrieved via {@link #getFacetContext(SolrQueryRequest)}
*
* @param rb the ResponseBuilder object from which the request parameters are read
* and to which the FacetContext object is saved.
*/
public static void initContext(ResponseBuilder rb) {
// Parse facet queries and ranges and put them in the request
// context so that they can be hung under pivots if needed without re-parsing
List<RangeFacetRequest> facetRanges = null;
List<FacetBase> facetQueries = null;
String[] ranges = rb.req.getParams().getParams(FacetParams.FACET_RANGE);
if (ranges != null) {
facetRanges = new ArrayList<>(ranges.length);
for (String range : ranges) {
RangeFacetRequest rangeFacetRequest = new RangeFacetRequest(rb, range);
facetRanges.add(rangeFacetRequest);
}
}
String[] queries = rb.req.getParams().getParams(FacetParams.FACET_QUERY);
if (queries != null) {
facetQueries = new ArrayList<>();
for (String query : queries) {
facetQueries.add(new FacetBase(rb, FacetParams.FACET_QUERY, query));
}
}
rb.req.getContext().put(FACET_CONTEXT_KEY, new FacetContext(facetRanges, facetQueries));
}
private FacetContext(List<RangeFacetRequest> allRangeFacets, List<FacetBase> allQueryFacets) {
// avoid NPEs, set to empty list if parameters are null
this.allRangeFacets = allRangeFacets == null ? Collections.emptyList() : allRangeFacets;
this.allQueryFacets = allQueryFacets == null ? Collections.emptyList() : allQueryFacets;
taggedRangeFacets = new HashMap<>();
for (RangeFacetRequest rf : this.allRangeFacets) {
for (String tag : rf.getTags()) {
List<RangeFacetRequest> list = taggedRangeFacets.get(tag);
if (list == null) {
list = new ArrayList<>(1); // typically just one object
taggedRangeFacets.put(tag, list);
}
list.add(rf);
}
}
taggedQueryFacets = new HashMap<>();
for (FacetBase qf : this.allQueryFacets) {
for (String tag : qf.getTags()) {
List<FacetBase> list = taggedQueryFacets.get(tag);
if (list == null) {
list = new ArrayList<>(1);
taggedQueryFacets.put(tag, list);
}
list.add(qf);
}
}
}
/**
* Return the {@link org.apache.solr.handler.component.FacetComponent.FacetContext} instance
* cached in the request context.
*
* @param req the {@link SolrQueryRequest}
* @return the cached FacetContext instance
* @throws IllegalStateException if no cached FacetContext instance is found in the request context
*/
public static FacetContext getFacetContext(SolrQueryRequest req) throws IllegalStateException {
FacetContext result = (FacetContext) req.getContext().get(FACET_CONTEXT_KEY);
if (null == result) {
throw new IllegalStateException("FacetContext can't be accessed before it's initialized in request context");
}
return result;
}
/**
* @return a {@link List} of {@link RangeFacetRequest} objects each representing a facet.range to be
* computed. Returns an empty list if no facet.range were requested.
*/
public List<RangeFacetRequest> getAllRangeFacetRequests() {
return allRangeFacets;
}
/**
* @return a {@link List} of {@link org.apache.solr.handler.component.FacetComponent.FacetBase} objects
* each representing a facet.query to be computed. Returns an empty list of no facet.query were requested.
*/
public List<FacetBase> getAllQueryFacets() {
return allQueryFacets;
}
/**
* @param tag a String tag usually specified via local param on a facet.pivot
* @return a list of {@link RangeFacetRequest} objects which have been tagged with the given tag.
* Returns an empty list if none found.
*/
public List<RangeFacetRequest> getRangeFacetRequestsForTag(String tag) {
List<RangeFacetRequest> list = taggedRangeFacets.get(tag);
return list == null ? Collections.emptyList() : list;
}
/**
* @param tag a String tag usually specified via local param on a facet.pivot
* @return a list of {@link org.apache.solr.handler.component.FacetComponent.FacetBase} objects which have been
* tagged with the given tag. Returns and empty List if none found.
*/
public List<FacetBase> getQueryFacetsForTag(String tag) {
List<FacetBase> list = taggedQueryFacets.get(tag);
return list == null ? Collections.emptyList() : list;
}
}
@ -81,27 +249,13 @@ public class FacetComponent extends SearchComponent {
@Override
public void process(ResponseBuilder rb) throws IOException {
//SolrParams params = rb.req.getParams();
if (rb.doFacets) {
ModifiableSolrParams params = new ModifiableSolrParams();
SolrParams origParams = rb.req.getParams();
Iterator<String> iter = origParams.getParameterNamesIterator();
while (iter.hasNext()) {
String paramName = iter.next();
// Deduplicate the list with LinkedHashSet, but _only_ for facet params.
if (paramName.startsWith(FacetParams.FACET) == false) {
params.add(paramName, origParams.getParams(paramName));
continue;
}
HashSet<String> deDupe = new LinkedHashSet<>(Arrays.asList(origParams.getParams(paramName)));
params.add(paramName, deDupe.toArray(new String[deDupe.size()]));
}
SolrParams params = rb.req.getParams();
SimpleFacets f = new SimpleFacets(rb.req, rb.getResults().docSet, params, rb);
NamedList<Object> counts = f.getFacetCounts();
NamedList<Object> counts = FacetComponent.getFacetCounts(f);
String[] pivots = params.getParams(FacetParams.FACET_PIVOT);
if (pivots != null && pivots.length > 0) {
if (!ArrayUtils.isEmpty(pivots)) {
PivotFacetProcessor pivotProcessor
= new PivotFacetProcessor(rb.req, rb.getResults().docSet, params, rb);
SimpleOrderedMap<List<NamedList<Object>>> v
@ -110,11 +264,46 @@ public class FacetComponent extends SearchComponent {
counts.add(PIVOT_KEY, v);
}
}
rb.rsp.add("facet_counts", counts);
}
}
/**
* Looks at various Params to determining if any simple Facet Constraint count
* computations are desired.
*
* @see SimpleFacets#getFacetQueryCounts
* @see SimpleFacets#getFacetFieldCounts
* @see DateFacetProcessor#getFacetDateCounts
* @see RangeFacetProcessor#getFacetRangeCounts
* @see RangeFacetProcessor#getFacetIntervalCounts
* @see FacetParams#FACET
* @return a NamedList of Facet Count info or null
*/
public static NamedList<Object> getFacetCounts(SimpleFacets simpleFacets) {
// if someone called this method, benefit of the doubt: assume true
if (!simpleFacets.getGlobalParams().getBool(FacetParams.FACET, true))
return null;
DateFacetProcessor dateFacetProcessor = new DateFacetProcessor(simpleFacets.getRequest(), simpleFacets.getDocsOrig(), simpleFacets.getGlobalParams(), simpleFacets.getResponseBuilder());
RangeFacetProcessor rangeFacetProcessor = new RangeFacetProcessor(simpleFacets.getRequest(), simpleFacets.getDocsOrig(), simpleFacets.getGlobalParams(), simpleFacets.getResponseBuilder());
NamedList<Object> counts = new SimpleOrderedMap<>();
try {
counts.add(FACET_QUERY_KEY, simpleFacets.getFacetQueryCounts());
counts.add(FACET_FIELD_KEY, simpleFacets.getFacetFieldCounts());
counts.add(FACET_DATE_KEY, dateFacetProcessor.getFacetDateCounts());
counts.add(FACET_RANGES_KEY, rangeFacetProcessor.getFacetRangeCounts());
counts.add(FACET_INTERVALS_KEY, simpleFacets.getFacetIntervalCounts());
counts.add(SpatialHeatmapFacets.RESPONSE_KEY, simpleFacets.getHeatmapCounts());
} catch (IOException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
} catch (SyntaxError e) {
throw new SolrException(ErrorCode.BAD_REQUEST, e);
}
return counts;
}
private static final String commandPrefix = "{!" + CommonParams.TERMS + "=$";
@Override
@ -514,7 +703,7 @@ public class FacetComponent extends SearchComponent {
qf.count += count;
}
}
// step through each facet.field, adding results from this shard
NamedList facet_fields = (NamedList) facet_counts.get("facet_fields");
@ -528,7 +717,12 @@ public class FacetComponent extends SearchComponent {
doDistribDates(fi, facet_counts);
// Distributed facet_ranges
doDistribRanges(fi, facet_counts);
@SuppressWarnings("unchecked")
SimpleOrderedMap<SimpleOrderedMap<Object>> rangesFromShard = (SimpleOrderedMap<SimpleOrderedMap<Object>>)
facet_counts.get("facet_ranges");
if (rangesFromShard != null) {
RangeFacetRequest.DistribRangeFacet.mergeFacetRangesFromShardResponse(fi.rangeFacets, rangesFromShard);
}
// Distributed facet_intervals
doDistribIntervals(fi, facet_counts);
@ -655,43 +849,19 @@ public class FacetComponent extends SearchComponent {
}
FacetInfo fi = rb._facetInfo;
@SuppressWarnings("unchecked")
SimpleOrderedMap<SimpleOrderedMap<Object>> facet_ranges =
(SimpleOrderedMap<SimpleOrderedMap<Object>>)
fi.rangeFacets;
if (facet_ranges == null) {
return;
}
// go through each facet_range
for (Map.Entry<String, SimpleOrderedMap<Object>> entry : facet_ranges) {
boolean replace = false;
for (Map.Entry<String, RangeFacetRequest.DistribRangeFacet> entry : fi.rangeFacets.entrySet()) {
final String field = entry.getKey();
final RangeFacetRequest.DistribRangeFacet rangeFacet = entry.getValue();
int minCount = rb.req.getParams().getFieldInt(field, FacetParams.FACET_MINCOUNT, 0);
if (minCount == 0) {
continue;
}
@SuppressWarnings("unchecked")
NamedList<Integer> vals
= (NamedList<Integer>) facet_ranges.get(field).get("counts");
NamedList newList = new NamedList();
for (Map.Entry<String, Integer> pair : vals) {
if (pair.getValue() >= minCount) {
newList.add(pair.getKey(), pair.getValue());
} else {
log.trace("Removing facet/key: " + pair.getKey() + "/" + pair.getValue().toString() + " mincount=" + minCount);
replace = true;
}
}
if (replace) {
vals.clear();
vals.addAll(newList);
}
rangeFacet.removeRangeFacetsUnderLimits(minCount);
}
}
private void removeFieldFacetsUnderLimits(ResponseBuilder rb) {
if (rb.stage != ResponseBuilder.STAGE_DONE) {
return;
@ -759,62 +929,6 @@ public class FacetComponent extends SearchComponent {
}
}
private final static String[] OTHER_KEYS = new String[]{FacetRangeOther.BEFORE.toString(), FacetRangeOther.BETWEEN.toString(), FacetRangeOther.AFTER.toString()};
// The implementation below uses the first encountered shard's
// facet_ranges as the basis for subsequent shards' data to be merged.
private void doDistribRanges(FacetInfo fi, NamedList facet_counts) {
@SuppressWarnings("unchecked")
SimpleOrderedMap<SimpleOrderedMap<Object>> facet_ranges =
(SimpleOrderedMap<SimpleOrderedMap<Object>>)
facet_counts.get("facet_ranges");
if (facet_ranges != null) {
// go through each facet_range
for (Map.Entry<String,SimpleOrderedMap<Object>> entry : facet_ranges) {
final String field = entry.getKey();
SimpleOrderedMap<Object> fieldMap = fi.rangeFacets.get(field);
if (fieldMap == null) {
// first time we've seen this field, no merging
fi.rangeFacets.add(field, entry.getValue());
} else {
// not the first time, merge current field counts
@SuppressWarnings("unchecked")
NamedList<Integer> shardFieldValues
= (NamedList<Integer>) entry.getValue().get("counts");
@SuppressWarnings("unchecked")
NamedList<Integer> existFieldValues
= (NamedList<Integer>) fieldMap.get("counts");
for (Map.Entry<String,Integer> existPair : existFieldValues) {
final String key = existPair.getKey();
// can be null if inconsistencies in shards responses
Integer newValue = shardFieldValues.get(key);
if (null != newValue) {
Integer oldValue = existPair.getValue();
existPair.setValue(oldValue + newValue);
}
}
// merge before/between/after if they exist
for (String otherKey:OTHER_KEYS) {
Integer shardValue = (Integer)entry.getValue().get(otherKey);
if (shardValue != null && shardValue > 0) {
Integer existingValue = (Integer)fieldMap.get(otherKey);
// shouldn't be null
int idx = fieldMap.indexOf(otherKey, 0);
fieldMap.setVal(idx, existingValue + shardValue);
}
}
}
}
}
}
//
// The implementation below uses the first encountered shard's
// facet_dates as the basis for subsequent shards' data to be merged.
@ -1046,7 +1160,15 @@ public class FacetComponent extends SearchComponent {
}
facet_counts.add("facet_dates", fi.dateFacets);
facet_counts.add("facet_ranges", fi.rangeFacets);
SimpleOrderedMap<SimpleOrderedMap<Object>> rangeFacetOutput = new SimpleOrderedMap<>();
for (Map.Entry<String, RangeFacetRequest.DistribRangeFacet> entry : fi.rangeFacets.entrySet()) {
String key = entry.getKey();
RangeFacetRequest.DistribRangeFacet value = entry.getValue();
rangeFacetOutput.add(key, value.rangeFacet);
}
facet_counts.add("facet_ranges", rangeFacetOutput);
facet_counts.add("facet_intervals", fi.intervalFacets);
facet_counts.add(SpatialHeatmapFacets.RESPONSE_KEY,
SpatialHeatmapFacets.distribFinish(fi.heatmapFacets, rb));
@ -1077,11 +1199,23 @@ public class FacetComponent extends SearchComponent {
}
// use <int> tags for smaller facet counts (better back compatibility)
private Number num(long val) {
/**
* @param val a primitive long value
* @return an {@link Integer} if the value of the argument is less than {@link Integer#MAX_VALUE}
* else a @{link java.lang.Long}
*/
static Number num(long val) {
if (val < Integer.MAX_VALUE) return (int)val;
else return val;
}
private Number num(Long val) {
/**
* @param val a {@link java.lang.Long} value
* @return an {@link Integer} if the value of the argument is less than {@link Integer#MAX_VALUE}
* else a @{link java.lang.Long}
*/
static Number num(Long val) {
if (val.longValue() < Integer.MAX_VALUE) return val.intValue();
else return val;
}
@ -1102,7 +1236,16 @@ public class FacetComponent extends SearchComponent {
}
/**
* This class is used exclusively for merging results from each shard
* in a distributed facet request. It plays no role in the computation
* of facet counts inside a single node.
*
* A related class {@link org.apache.solr.handler.component.FacetComponent.FacetContext}
* exists for assisting computation inside a single node.
*
* <b>This API is experimental and subject to change</b>
*
* @see org.apache.solr.handler.component.FacetComponent.FacetContext
*/
public static class FacetInfo {
/**
@ -1116,8 +1259,8 @@ public class FacetComponent extends SearchComponent {
public LinkedHashMap<String,DistribFieldFacet> facets;
public SimpleOrderedMap<SimpleOrderedMap<Object>> dateFacets
= new SimpleOrderedMap<>();
public SimpleOrderedMap<SimpleOrderedMap<Object>> rangeFacets
= new SimpleOrderedMap<>();
public LinkedHashMap<String, RangeFacetRequest.DistribRangeFacet> rangeFacets
= new LinkedHashMap<>();
public SimpleOrderedMap<SimpleOrderedMap<Integer>> intervalFacets
= new SimpleOrderedMap<>();
public SimpleOrderedMap<PivotFacet> pivotFacets
@ -1157,7 +1300,7 @@ public class FacetComponent extends SearchComponent {
heatmapFacets = SpatialHeatmapFacets.distribParse(params, rb);
}
}
/**
* <b>This API is experimental and subject to change</b>
*/
@ -1168,6 +1311,9 @@ public class FacetComponent extends SearchComponent {
private String key; // label in the response for the result...
// "foo" for {!key=foo}myfield
SolrParams localParams; // any local params for the facet
private List<String> tags = Collections.emptyList();
private List<String> excludeTags = Collections.emptyList();
private int threadCount = -1;
public FacetBase(ResponseBuilder rb, String facetType, String facetStr) {
this.facetType = facetType;
@ -1189,12 +1335,28 @@ public class FacetComponent extends SearchComponent {
}
key = localParams.get(CommonParams.OUTPUT_KEY, key);
String tagStr = localParams.get(CommonParams.TAG);
this.tags = tagStr == null ? Collections.<String>emptyList() : StrUtils.splitSmart(tagStr,',');
String threadStr = localParams.get(CommonParams.THREADS);
this.threadCount = threadStr != null ? Integer.parseInt(threadStr) : -1;
String excludeStr = localParams.get(CommonParams.EXCLUDE);
if (StringUtils.isEmpty(excludeStr)) {
this.excludeTags = Collections.emptyList();
} else {
this.excludeTags = StrUtils.splitSmart(excludeStr,',');
}
}
}
/** returns the key in the response that this facet will be under */
public String getKey() { return key; }
public String getType() { return facetType; }
public List<String> getTags() { return tags; }
public List<String> getExcludeTags() { return excludeTags; }
public int getThreadCount() { return threadCount; }
}
/**
@ -1407,4 +1569,5 @@ public class FacetComponent extends SearchComponent {
return "{term=" + name + ",termNum=" + termNum + ",count=" + count + "}";
}
}
}

59
solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java
View File

@ -17,22 +17,18 @@
package org.apache.solr.handler.component;
import org.apache.solr.util.PivotListEntry;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Collections;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.solr.util.PivotListEntry;
public class PivotFacetHelper {
@ -52,8 +48,9 @@ public class PivotFacetHelper {
assert null != values;
// special case: empty list => empty string
if (values.isEmpty()) { return ""; }
if (values.isEmpty()) {
return "";
}
StringBuilder out = new StringBuilder();
for (String val : values) {
@ -76,7 +73,7 @@ public class PivotFacetHelper {
* @see #encodeRefinementValuePath
*/
public static List<String> decodeRefinementValuePath(String valuePath) {
List <String> rawvals = StrUtils.splitSmart(valuePath, ",", true);
List<String> rawvals = StrUtils.splitSmart(valuePath, ",", true);
// special case: empty list => empty string
if (rawvals.isEmpty()) return rawvals;
@ -120,6 +117,16 @@ public class PivotFacetHelper {
return (NamedList<NamedList<NamedList<?>>>) PivotListEntry.STATS.extract(pivotList);
}
/** @see PivotListEntry#QUERIES */
public static NamedList<Number> getQueryCounts(NamedList<Object> pivotList) {
return (NamedList<Number>) PivotListEntry.QUERIES.extract(pivotList);
}
/** @see PivotListEntry#RANGES */
public static SimpleOrderedMap<SimpleOrderedMap<Object>> getRanges(NamedList<Object> pivotList) {
return (SimpleOrderedMap<SimpleOrderedMap<Object>>) PivotListEntry.RANGES.extract(pivotList);
}
/**
* Given a mapping of keys to {@link StatsValues} representing the currently
* known "merged" stats (which may be null if none exist yet), and a
@ -156,4 +163,28 @@ public class PivotFacetHelper {
return merged;
}
/**
* Merges query counts returned by a shard into global query counts.
* Entries found only in shard's query counts will be added to global counts.
* Entries found in both shard and global query counts will be summed.
*
* @param globalQueryCounts The global query counts (across all shards) in which to merge the shard query counts
* @param shardQueryCounts Named list from a shard response to be merged into the global counts.
* @return NamedList containing merged values
*/
static NamedList<Number> mergeQueryCounts(
NamedList<Number> globalQueryCounts, NamedList<Number> shardQueryCounts) {
if (globalQueryCounts == null) {
return shardQueryCounts;
}
for (Entry<String, Number> entry : shardQueryCounts) {
int idx = globalQueryCounts.indexOf(entry.getKey(), 0);
if (idx == -1) {
globalQueryCounts.add(entry.getKey(), entry.getValue());
} else {
globalQueryCounts.setVal(idx, FacetComponent.num(globalQueryCounts.getVal(idx).longValue() + entry.getValue().longValue()));
}
}
return globalQueryCounts;
}
}

136
solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java
View File

@ -18,6 +18,8 @@ package org.apache.solr.handler.component;
*/
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.solr.common.StringUtils;
import org.apache.solr.common.params.RequiredSolrParams;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.search.SolrIndexSearcher;
@ -41,7 +43,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
@ -52,6 +53,8 @@ import java.util.Map;
*/
public class PivotFacetProcessor extends SimpleFacets
{
public static final String QUERY = "query";
public static final String RANGE = "range";
protected SolrParams params;
public PivotFacetProcessor(SolrQueryRequest req, DocSet docs, SolrParams params, ResponseBuilder rb) {
@ -101,7 +104,8 @@ public class PivotFacetProcessor extends SimpleFacets
String refineKey = null; // no local => no refinement
List<StatsField> statsFields = Collections.emptyList(); // no local => no stats
List<FacetComponent.FacetBase> facetQueries = Collections.emptyList();
List<RangeFacetRequest> facetRanges = Collections.emptyList();
if (null != parsed.localParams) {
// we might be refining..
refineKey = parsed.localParams.get(PivotFacet.REFINE_PARAM);
@ -117,6 +121,42 @@ public class PivotFacetProcessor extends SimpleFacets
statsInfo = new StatsInfo(rb);
}
statsFields = getTaggedStatsFields(statsInfo, statsLocalParam);
try {
FacetComponent.FacetContext facetContext = FacetComponent.FacetContext.getFacetContext(req);
String taggedQueries = parsed.localParams.get(QUERY);
if (StringUtils.isEmpty(taggedQueries)) {
facetQueries = Collections.emptyList();
} else {
List<String> localParamValue = StrUtils.splitSmart(taggedQueries, ',');
if (localParamValue.size() > 1) {
String msg = QUERY + " local param of " + FacetParams.FACET_PIVOT +
"may not include tags separated by a comma - please use a common tag on all " +
FacetParams.FACET_QUERY + " params you wish to compute under this pivot";
throw new SolrException(ErrorCode.BAD_REQUEST, msg);
}
taggedQueries = localParamValue.get(0);
facetQueries = facetContext.getQueryFacetsForTag(taggedQueries);
}
String taggedRanges = parsed.localParams.get(RANGE);
if (StringUtils.isEmpty(taggedRanges)) {
facetRanges = Collections.emptyList();
} else {
List<String> localParamValue = StrUtils.splitSmart(taggedRanges, ',');
if (localParamValue.size() > 1) {
String msg = RANGE + " local param of " + FacetParams.FACET_PIVOT +
"may not include tags separated by a comma - please use a common tag on all " +
FacetParams.FACET_RANGE + " params you wish to compute under this pivot";
throw new SolrException(ErrorCode.BAD_REQUEST, msg);
}
taggedRanges = localParamValue.get(0);
facetRanges = facetContext.getRangeFacetRequestsForTag(taggedRanges);
}
} catch (IllegalStateException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Faceting context not set, cannot calculate pivot values");
}
}
if (null != refineKey) {
@ -124,10 +164,10 @@ public class PivotFacetProcessor extends SimpleFacets
= params.getParams(PivotFacet.REFINE_PARAM + refineKey);
for(String refinements : refinementValuesByField){
pivotResponse.addAll(processSingle(pivotFields, refinements, statsFields, parsed));
pivotResponse.addAll(processSingle(pivotFields, refinements, statsFields, parsed, facetQueries, facetRanges));
}
} else{
pivotResponse.addAll(processSingle(pivotFields, null, statsFields, parsed));
pivotResponse.addAll(processSingle(pivotFields, null, statsFields, parsed, facetQueries, facetRanges));
}
}
return pivotResponse;
@ -138,12 +178,16 @@ public class PivotFacetProcessor extends SimpleFacets
* @param pivotFields the ordered list of fields in this pivot
* @param refinements the comma separate list of refinement values corresponding to each field in the pivot, or null if there are no refinements
* @param statsFields List of {@link StatsField} instances to compute for each pivot value
* @param facetQueries the list of facet queries hung under this pivot
* @param facetRanges the list of facet ranges hung under this pivot
*/
private SimpleOrderedMap<List<NamedList<Object>>> processSingle
(List<String> pivotFields,
String refinements,
List<StatsField> statsFields,
final ParsedParams parsed) throws IOException {
(List<String> pivotFields,
String refinements,
List<StatsField> statsFields,
final ParsedParams parsed,
List<FacetComponent.FacetBase> facetQueries,
List<RangeFacetRequest> facetRanges) throws IOException {
SolrIndexSearcher searcher = rb.req.getSearcher();
SimpleOrderedMap<List<NamedList<Object>>> pivotResponse = new SimpleOrderedMap<>();
@ -181,9 +225,9 @@ public class PivotFacetProcessor extends SimpleFacets
if(pivotFields.size() > 1) {
String subField = pivotFields.get(1);
pivotResponse.add(parsed.key,
doPivots(facetCounts, field, subField, fnames, vnames, parsed, statsFields));
doPivots(facetCounts, field, subField, fnames, vnames, parsed, statsFields, facetQueries, facetRanges));
} else {
pivotResponse.add(parsed.key, doPivots(facetCounts, field, null, fnames, vnames, parsed, statsFields));
pivotResponse.add(parsed.key, doPivots(facetCounts, field, null, fnames, vnames, parsed, statsFields, facetQueries, facetRanges));
}
return pivotResponse;
}
@ -223,10 +267,11 @@ public class PivotFacetProcessor extends SimpleFacets
* Recursive function to compute all the pivot counts for the values under the specified field
*/
protected List<NamedList<Object>> doPivots(NamedList<Integer> superFacets,
String field, String subField,
Deque<String> fnames, Deque<String> vnames,
ParsedParams parsed, List<StatsField> statsFields)
throws IOException {
String field, String subField,
Deque<String> fnames, Deque<String> vnames,
ParsedParams parsed, List<StatsField> statsFields,
List<FacetComponent.FacetBase> facetQueries, List<RangeFacetRequest> facetRanges)
throws IOException {
boolean isShard = rb.req.getParams().getBool(ShardParams.IS_SHARD, false);
@ -259,6 +304,8 @@ public class PivotFacetProcessor extends SimpleFacets
final DocSet subset = getSubset(parsed.docs, sfield, fieldValue);
addPivotQueriesAndRanges(pivot, params, subset, facetQueries, facetRanges);
if( subField != null ) {
NamedList<Integer> facetCounts;
if(!vnames.isEmpty()){
@ -272,7 +319,7 @@ public class PivotFacetProcessor extends SimpleFacets
}
if (facetCounts.size() >= 1) {
pivot.add( "pivot", doPivots( facetCounts, subField, nextField, fnames, vnames, parsed.withDocs(subset), statsFields ) );
pivot.add( "pivot", doPivots( facetCounts, subField, nextField, fnames, vnames, parsed.withDocs(subset), statsFields, facetQueries, facetRanges) );
}
}
if ((isShard || 0 < pivotCount) && ! statsFields.isEmpty()) {
@ -329,6 +376,65 @@ public class PivotFacetProcessor extends SimpleFacets
}
}
/**
* Add facet.queries and facet.ranges to the pivot response if needed
*
* @param pivot
* Pivot in which to inject additional data
* @param params
* Query parameters.
* @param docs
* DocSet of the current pivot to use for computing sub-counts
* @param facetQueries
* Tagged facet queries should have to be included, must not be null
* @param facetRanges
* Taged facet ranges should have to be included, must not be null
* @throws IOException
* If searcher has issues finding numDocs.
*/
protected void addPivotQueriesAndRanges(NamedList<Object> pivot, SolrParams params, DocSet docs,
List<FacetComponent.FacetBase> facetQueries,
List<RangeFacetRequest> facetRanges) throws IOException {
assert null != facetQueries;
assert null != facetRanges;
if ( ! facetQueries.isEmpty()) {
SimpleFacets facets = new SimpleFacets(req, docs, params);
NamedList<Integer> res = new SimpleOrderedMap<>();
for (FacetComponent.FacetBase facetQuery : facetQueries) {
try {
ParsedParams parsed = getParsedParams(params, docs, facetQuery);
facets.getFacetQueryCount(parsed, res);
} catch (SyntaxError e) {
throw new SolrException(ErrorCode.BAD_REQUEST,
"Invalid " + FacetParams.FACET_QUERY + " (" + facetQuery.facetStr +
") cause: " + e.getMessage(), e);
}
}
pivot.add(PivotListEntry.QUERIES.getName(), res);
}
if ( ! facetRanges.isEmpty()) {
RangeFacetProcessor rangeFacetProcessor = new RangeFacetProcessor(req, docs, params, null);
NamedList<Object> resOuter = new SimpleOrderedMap<>();
for (RangeFacetRequest rangeFacet : facetRanges) {
try {
rangeFacetProcessor.getFacetRangeCounts(rangeFacet, resOuter);
} catch (SyntaxError e) {
throw new SolrException(ErrorCode.BAD_REQUEST,
"Invalid " + FacetParams.FACET_RANGE + " (" + rangeFacet.facetStr +
") cause: " + e.getMessage(), e);
}
}
pivot.add(PivotListEntry.RANGES.getName(), resOuter);
}
}
private ParsedParams getParsedParams(SolrParams params, DocSet docs, FacetComponent.FacetBase facet) {
SolrParams wrapped = SolrParams.wrapDefaults(facet.localParams, global);
SolrParams required = new RequiredSolrParams(params);
return new ParsedParams(facet.localParams, wrapped, required, facet.facetOn, docs, facet.getKey(), facet.getTags(), -1);
}
private int getMinCountForField(String fieldname){
return params.getFieldInt(fieldname, FacetParams.FACET_PIVOT_MINCOUNT, 1);
}

45
solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.handler.component;
import java.util.BitSet;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@ -27,7 +28,6 @@ import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.TrieDateField;
import org.apache.solr.search.QueryParsing;
import org.apache.solr.util.PivotListEntry;
/**
@ -48,7 +48,10 @@ public class PivotFacetValue {
private PivotFacetField childPivot = null;
private int count; // mutable
private Map<String, StatsValues> statsValues = null;
// named list with objects because depending on how big the counts are we may get either a long or an int
private NamedList<Number> queryCounts;
private LinkedHashMap<String, RangeFacetRequest.DistribRangeFacet> rangeCounts;
private PivotFacetValue(PivotFacetField parent, Comparable val) {
this.parentPivot = parent;
this.value = val;
@ -118,6 +121,8 @@ public class PivotFacetValue {
int pivotCount = 0;
List<NamedList<Object>> childPivotData = null;
NamedList<NamedList<NamedList<?>>> statsValues = null;
NamedList<Number> queryCounts = null;
SimpleOrderedMap<SimpleOrderedMap<Object>> ranges = null;
for (int i = 0; i < pivotData.size(); i++) {
String key = pivotData.getName(i);
@ -142,6 +147,12 @@ public class PivotFacetValue {
case STATS:
statsValues = (NamedList<NamedList<NamedList<?>>>) value;
break;
case QUERIES:
queryCounts = (NamedList<Number>) value;
break;
case RANGES:
ranges = (SimpleOrderedMap<SimpleOrderedMap<Object>>) value;
break;
default:
throw new RuntimeException("PivotListEntry contains unaccounted for item: " + entry);
}
@ -153,6 +164,13 @@ public class PivotFacetValue {
if(statsValues != null) {
newPivotFacet.statsValues = PivotFacetHelper.mergeStats(null, statsValues, rb._statsInfo);
}
if(queryCounts != null) {
newPivotFacet.queryCounts = PivotFacetHelper.mergeQueryCounts(null, queryCounts);
}
if(ranges != null) {
newPivotFacet.rangeCounts = new LinkedHashMap<>();
RangeFacetRequest.DistribRangeFacet.mergeFacetRangesFromShardResponse(newPivotFacet.rangeCounts, ranges);
}
newPivotFacet.childPivot = PivotFacetField.createFromListOfNamedLists(shardNumber, rb, newPivotFacet, childPivotData);
@ -178,6 +196,18 @@ public class PivotFacetValue {
newList.add(PivotListEntry.FIELD.getName(), parentPivot.field);
newList.add(PivotListEntry.VALUE.getName(), value);
newList.add(PivotListEntry.COUNT.getName(), count);
if(queryCounts != null) {
newList.add(PivotListEntry.QUERIES.getName(), queryCounts);
}
if(rangeCounts != null) {
SimpleOrderedMap<SimpleOrderedMap<Object>> rangeFacetOutput = new SimpleOrderedMap<>();
for (Map.Entry<String, RangeFacetRequest.DistribRangeFacet> entry : rangeCounts.entrySet()) {
String key = entry.getKey();
RangeFacetRequest.DistribRangeFacet value = entry.getValue();
rangeFacetOutput.add(key, value.rangeFacet);
}
newList.add(PivotListEntry.RANGES.getName(), rangeFacetOutput);
}
if (childPivot != null && childPivot.convertToListOfNamedLists() != null) {
newList.add(PivotListEntry.PIVOT.getName(), childPivot.convertToListOfNamedLists());
}
@ -205,6 +235,17 @@ public class PivotFacetValue {
if (stats != null) {
statsValues = PivotFacetHelper.mergeStats(statsValues, stats, rb._statsInfo);
}
NamedList<Number> shardQueryCounts = PivotFacetHelper.getQueryCounts(value);
if(shardQueryCounts != null) {
queryCounts = PivotFacetHelper.mergeQueryCounts(queryCounts, shardQueryCounts);
}
SimpleOrderedMap<SimpleOrderedMap<Object>> shardRanges = PivotFacetHelper.getRanges(value);
if (shardRanges != null) {
if (rangeCounts == null) {
rangeCounts = new LinkedHashMap<>(shardRanges.size() / 2);
}
RangeFacetRequest.DistribRangeFacet.mergeFacetRangesFromShardResponse(rangeCounts, shardRanges);
}
}
List<NamedList<Object>> shardChildPivots = PivotFacetHelper.getPivots(value);

277
solr/core/src/java/org/apache/solr/handler/component/RangeFacetProcessor.java Normal file
View File

@ -0,0 +1,277 @@
package org.apache.solr.handler.component;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.apache.log4j.Logger;
import org.apache.lucene.search.Query;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.FacetParams.FacetRangeMethod;
import org.apache.solr.common.params.FacetParams.FacetRangeOther;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.request.IntervalFacets;
import org.apache.solr.request.SimpleFacets;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TrieField;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SyntaxError;
/**
* Processor for Range Facets
*/
public class RangeFacetProcessor extends SimpleFacets {
private final static Logger log = Logger.getLogger(RangeFacetProcessor.class);
public RangeFacetProcessor(SolrQueryRequest req, DocSet docs, SolrParams params, ResponseBuilder rb) {
super(req, docs, params, rb);
}
/**
* Returns a list of value constraints and the associated facet
* counts for each facet numerical field, range, and interval
* specified in the SolrParams
*
* @see org.apache.solr.common.params.FacetParams#FACET_RANGE
*/
@SuppressWarnings("unchecked")
public NamedList<Object> getFacetRangeCounts() throws IOException, SyntaxError {
final NamedList<Object> resOuter = new SimpleOrderedMap<>();
List<RangeFacetRequest> rangeFacetRequests = Collections.emptyList();
try {
FacetComponent.FacetContext facetContext = FacetComponent.FacetContext.getFacetContext(req);
rangeFacetRequests = facetContext.getAllRangeFacetRequests();
} catch (IllegalStateException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to compute facet ranges, facet context is not set");
}
if (rangeFacetRequests.isEmpty()) return resOuter;
for (RangeFacetRequest rangeFacetRequest : rangeFacetRequests) {
getFacetRangeCounts(rangeFacetRequest, resOuter);
}
return resOuter;
}
/**
* Returns a list of value constraints and the associated facet counts
* for each facet range specified by the given {@link RangeFacetRequest}
*/
public void getFacetRangeCounts(RangeFacetRequest rangeFacetRequest, NamedList<Object> resOuter)
throws IOException, SyntaxError {
final IndexSchema schema = searcher.getSchema();
final String key = rangeFacetRequest.getKey();
final String f = rangeFacetRequest.facetOn;
FacetRangeMethod method = rangeFacetRequest.getMethod();
final SchemaField sf = schema.getField(f);
final FieldType ft = sf.getType();
if (method.equals(FacetRangeMethod.DV)) {
assert ft instanceof TrieField;
resOuter.add(key, getFacetRangeCountsDocValues(rangeFacetRequest));
} else {
resOuter.add(key, getFacetRangeCounts(rangeFacetRequest));
}
}
private <T extends Comparable<T>> NamedList getFacetRangeCounts(final RangeFacetRequest rfr)
throws IOException, SyntaxError {
final NamedList<Object> res = new SimpleOrderedMap<>();
final NamedList<Integer> counts = new NamedList<>();
res.add("counts", counts);
// explicitly return the gap.
res.add("gap", rfr.getGapObj());
DocSet docSet = computeDocSet(docsOrig, rfr.getExcludeTags());
for (RangeFacetRequest.FacetRange range : rfr.getFacetRanges()) {
if (range.other != null) {
// these are added to top-level NamedList
// and we always include them regardless of mincount
res.add(range.other.toString(), rangeCount(docSet, rfr, range));
} else {
final int count = rangeCount(docSet, rfr, range);
if (count >= rfr.getMinCount()) {
counts.add(range.lower, count);
}
}
}
// explicitly return the start and end so all the counts
// (including before/after/between) are meaningful - even if mincount
// has removed the neighboring ranges
res.add("start", rfr.getStartObj());
res.add("end", rfr.getEndObj());
return res;
}
private <T extends Comparable<T>> NamedList<Object> getFacetRangeCountsDocValues(RangeFacetRequest rfr)
throws IOException, SyntaxError {
SchemaField sf = rfr.getSchemaField();
final NamedList<Object> res = new SimpleOrderedMap<>();
final NamedList<Integer> counts = new NamedList<>();
res.add("counts", counts);
ArrayList<IntervalFacets.FacetInterval> intervals = new ArrayList<>();
// explicitly return the gap. compute this early so we are more
// likely to catch parse errors before attempting math
res.add("gap", rfr.getGapObj());
final int minCount = rfr.getMinCount();
boolean includeBefore = false;
boolean includeBetween = false;
boolean includeAfter = false;
Set<FacetRangeOther> others = rfr.getOthers();
// Intervals must be in order (see IntervalFacets.getSortedIntervals), if "BEFORE" or
// "BETWEEN" are set, they must be added first
// no matter what other values are listed, we don't do
// anything if "none" is specified.
if (!others.contains(FacetRangeOther.NONE)) {
if (others.contains(FacetRangeOther.ALL) || others.contains(FacetRangeOther.BEFORE)) {
// We'll add an interval later in this position
intervals.add(null);
includeBefore = true;
}
if (others.contains(FacetRangeOther.ALL) || others.contains(FacetRangeOther.BETWEEN)) {
// We'll add an interval later in this position
intervals.add(null);
includeBetween = true;
}
if (others.contains(FacetRangeOther.ALL) || others.contains(FacetRangeOther.AFTER)) {
includeAfter = true;
}
}
IntervalFacets.FacetInterval after = null;
for (RangeFacetRequest.FacetRange range : rfr.getFacetRanges()) {
try {
FacetRangeOther other = FacetRangeOther.get(range.name);
if (other != null) {
switch (other) {
case BEFORE:
assert range.lower == null;
intervals.set(0, new IntervalFacets.FacetInterval(sf, "*", range.upper, range.includeLower,
range.includeUpper, FacetRangeOther.BEFORE.toString()));
break;
case AFTER:
assert range.upper == null;
after = new IntervalFacets.FacetInterval(sf, range.lower, "*",
range.includeLower, range.includeUpper, FacetRangeOther.AFTER.toString());
break;
case BETWEEN:
intervals.set(includeBefore ? 1 : 0, new IntervalFacets.FacetInterval(sf, range.lower, range.upper,
range.includeLower, range.includeUpper, FacetRangeOther.BETWEEN.toString()));
break;
}
}
continue;
} catch (SolrException e) {
// safe to ignore
}
intervals.add(new IntervalFacets.FacetInterval(sf, range.lower, range.upper, range.includeLower, range.includeUpper, range.lower));
}
if (includeAfter) {
assert after != null;
intervals.add(after);
}
IntervalFacets.FacetInterval[] intervalsArray = intervals.toArray(new IntervalFacets.FacetInterval[intervals.size()]);
// don't use the ArrayList anymore
intervals = null;
new IntervalFacets(sf, searcher, computeDocSet(docsOrig, rfr.getExcludeTags()), intervalsArray);
int intervalIndex = 0;
int lastIntervalIndex = intervalsArray.length - 1;
// if the user requested "BEFORE", it will be the first of the intervals. Needs to be added to the
// response named list instead of with the counts
if (includeBefore) {
res.add(intervalsArray[intervalIndex].getKey(), intervalsArray[intervalIndex].getCount());
intervalIndex++;
}
// if the user requested "BETWEEN", it will be the first or second of the intervals (depending on if
// "BEFORE" was also requested). Needs to be added to the response named list instead of with the counts
if (includeBetween) {
res.add(intervalsArray[intervalIndex].getKey(), intervalsArray[intervalIndex].getCount());
intervalIndex++;
}
// if the user requested "AFTER", it will be the last of the intervals.
// Needs to be added to the response named list instead of with the counts
if (includeAfter) {
res.add(intervalsArray[lastIntervalIndex].getKey(), intervalsArray[lastIntervalIndex].getCount());
lastIntervalIndex--;
}
// now add all other intervals to the counts NL
while (intervalIndex <= lastIntervalIndex) {
IntervalFacets.FacetInterval interval = intervalsArray[intervalIndex];
if (interval.getCount() >= minCount) {
counts.add(interval.getKey(), interval.getCount());
}
intervalIndex++;
}
res.add("start", rfr.getStartObj());
res.add("end", rfr.getEndObj());
return res;
}
/**
* Macro for getting the numDocs of range over docs
*
* @see org.apache.solr.search.SolrIndexSearcher#numDocs
* @see org.apache.lucene.search.TermRangeQuery
*/
protected int rangeCount(DocSet subset, RangeFacetRequest rfr, RangeFacetRequest.FacetRange fr) throws IOException, SyntaxError {
SchemaField schemaField = rfr.getSchemaField();
Query rangeQ = schemaField.getType().getRangeQuery(null, schemaField, fr.lower, fr.upper, fr.includeLower, fr.includeUpper);
if (rfr.isGroupFacet()) {
return getGroupedFacetQueryCount(rangeQ, subset);
} else {
return searcher.numDocs(rangeQ, subset);
}
}
}

802
solr/core/src/java/org/apache/solr/handler/component/RangeFacetRequest.java Normal file
View File

@ -0,0 +1,802 @@
package org.apache.solr.handler.component;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.Date;
import java.util.EnumSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.GroupParams;
import org.apache.solr.common.params.RequiredSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.DateRangeField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TrieDateField;
import org.apache.solr.schema.TrieField;
import org.apache.solr.util.DateMathParser;
/**
* Encapsulates a single facet.range request along with all its parameters. This class
* calculates all the ranges (gaps) required to be counted.
*/
public class RangeFacetRequest extends FacetComponent.FacetBase {
private final static Logger log = Logger.getLogger(RangeFacetRequest.class);
protected final SchemaField schemaField;
protected final String start;
protected final String end;
protected final String gap;
protected final boolean hardEnd;
protected final EnumSet<FacetParams.FacetRangeInclude> include;
protected final EnumSet<FacetParams.FacetRangeOther> others;
protected final FacetParams.FacetRangeMethod method;
protected final int minCount;
protected final boolean groupFacet;
protected final List<FacetRange> facetRanges;
/**
* The computed start value of this range
*/
protected final Object startObj;
/**
* The computed end value of this range taking into account facet.range.hardend
*/
protected final Object endObj;
/**
* The computed gap between each range
*/
protected final Object gapObj;
public RangeFacetRequest(ResponseBuilder rb, String f) {
super(rb, FacetParams.FACET_RANGE, f);
IndexSchema schema = rb.req.getSchema();
this.schemaField = schema.getField(facetOn);
SolrParams params = SolrParams.wrapDefaults(localParams, rb.req.getParams());
SolrParams required = new RequiredSolrParams(params);
String methodStr = params.get(FacetParams.FACET_RANGE_METHOD);
FacetParams.FacetRangeMethod method = (methodStr == null ? FacetParams.FacetRangeMethod.getDefault() : FacetParams.FacetRangeMethod.get(methodStr));
if ((schemaField.getType() instanceof DateRangeField) && method.equals(FacetParams.FacetRangeMethod.DV)) {
// the user has explicitly selected the FacetRangeMethod.DV method
log.warn("Range facet method '" + FacetParams.FacetRangeMethod.DV + "' is not supported together with field type '" +
DateRangeField.class + "'. Will use method '" + FacetParams.FacetRangeMethod.FILTER + "' instead");
method = FacetParams.FacetRangeMethod.FILTER;
}
this.start = required.getFieldParam(facetOn, FacetParams.FACET_RANGE_START);
this.end = required.getFieldParam(facetOn, FacetParams.FACET_RANGE_END);
this.gap = required.getFieldParam(facetOn, FacetParams.FACET_RANGE_GAP);
this.minCount = params.getFieldInt(facetOn, FacetParams.FACET_MINCOUNT, 0);
this.include = FacetParams.FacetRangeInclude.parseParam
(params.getFieldParams(facetOn, FacetParams.FACET_RANGE_INCLUDE));
this.hardEnd = params.getFieldBool(facetOn, FacetParams.FACET_RANGE_HARD_END, false);
this.others = EnumSet.noneOf(FacetParams.FacetRangeOther.class);
final String[] othersP = params.getFieldParams(facetOn, FacetParams.FACET_RANGE_OTHER);
if (othersP != null && othersP.length > 0) {
for (final String o : othersP) {
others.add(FacetParams.FacetRangeOther.get(o));
}
}
this.groupFacet = params.getBool(GroupParams.GROUP_FACET, false);
if (groupFacet && method.equals(FacetParams.FacetRangeMethod.DV)) {
// the user has explicitly selected the FacetRangeMethod.DV method
log.warn("Range facet method '" + FacetParams.FacetRangeMethod.DV + "' is not supported together with '" +
GroupParams.GROUP_FACET + "'. Will use method '" + FacetParams.FacetRangeMethod.FILTER + "' instead");
method = FacetParams.FacetRangeMethod.FILTER;
}
this.method = method;
RangeEndpointCalculator<? extends Comparable<?>> calculator = createCalculator();
this.facetRanges = calculator.computeRanges();
this.gapObj = calculator.getGap();
this.startObj = calculator.getStart();
this.endObj = calculator.getComputedEnd();
}
/**
* Creates the right instance of {@link org.apache.solr.handler.component.RangeFacetRequest.RangeEndpointCalculator}
* depending on the field type of the schema field
*/
private RangeEndpointCalculator<? extends Comparable<?>> createCalculator() {
RangeEndpointCalculator<?> calc;
FieldType ft = schemaField.getType();
if (ft instanceof TrieField) {
final TrieField trie = (TrieField) ft;
switch (trie.getType()) {
case FLOAT:
calc = new FloatRangeEndpointCalculator(this);
break;
case DOUBLE:
calc = new DoubleRangeEndpointCalculator(this);
break;
case INTEGER:
calc = new IntegerRangeEndpointCalculator(this);
break;
case LONG:
calc = new LongRangeEndpointCalculator(this);
break;
case DATE:
calc = new DateRangeEndpointCalculator(this, null);
break;
default:
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"Unable to range facet on tried field of unexpected type:" + this.facetOn);
}
} else if (ft instanceof DateRangeField) {
calc = new DateRangeFieldEndpointCalculator(this, null);
} else {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"Unable to range facet on field:" + schemaField);
}
return calc;
}
/**
* @return the start of this range as specified by {@link FacetParams#FACET_RANGE_START} parameter
*/
public String getStart() {
return start;
}
/**
* The end of this facet.range as specified by {@link FacetParams#FACET_RANGE_END} parameter
* <p>
* Note that the actual computed end value can be different depending on the
* {@link FacetParams#FACET_RANGE_HARD_END} parameter. See {@link #endObj}
*/
public String getEnd() {
return end;
}
/**
* @return an {@link EnumSet} containing all the values specified via
* {@link FacetParams#FACET_RANGE_INCLUDE} parameter. Defaults to
* {@link org.apache.solr.common.params.FacetParams.FacetRangeInclude#LOWER} if no parameter
* is supplied. Includes all values from {@link org.apache.solr.common.params.FacetParams.FacetRangeInclude} enum
* if {@link FacetParams#FACET_RANGE_INCLUDE} includes
* {@link org.apache.solr.common.params.FacetParams.FacetRangeInclude#ALL}
*/
public EnumSet<FacetParams.FacetRangeInclude> getInclude() {
return include;
}
/**
* @return the gap as specified by {@link FacetParams#FACET_RANGE_GAP} parameter
*/
public String getGap() {
return gap;
}
/**
* @return the computed gap object
*/
public Object getGapObj() {
return gapObj;
}
/**
* @return the boolean value of {@link FacetParams#FACET_RANGE_HARD_END} parameter
*/
public boolean isHardEnd() {
return hardEnd;
}
/**
* @return an {@link EnumSet} of {@link org.apache.solr.common.params.FacetParams.FacetRangeOther} values
* specified by {@link FacetParams#FACET_RANGE_OTHER} parameter
*/
public EnumSet<FacetParams.FacetRangeOther> getOthers() {
return others;
}
/**
* @return the {@link org.apache.solr.common.params.FacetParams.FacetRangeMethod} to be used for computing
* ranges determined either by the value of {@link FacetParams#FACET_RANGE_METHOD} parameter
* or other internal constraints.
*/
public FacetParams.FacetRangeMethod getMethod() {
return method;
}
/**
* @return the minimum allowed count for facet ranges as specified by {@link FacetParams#FACET_MINCOUNT}
*/
public int getMinCount() {
return minCount;
}
/**
* @return the {@link SchemaField} instance representing the field on which ranges have to be calculated
*/
public SchemaField getSchemaField() {
return schemaField;
}
/**
* @return the boolean value specified by {@link GroupParams#GROUP_FACET} parameter
*/
public boolean isGroupFacet() {
return groupFacet;
}
/**
* @return a {@link List} of {@link org.apache.solr.handler.component.RangeFacetRequest.FacetRange} objects
* representing the ranges (gaps) for which range counts are to be calculated.
*/
public List<FacetRange> getFacetRanges() {
return facetRanges;
}
/**
* @return The computed start value of this range
*/
public Object getStartObj() {
return startObj;
}
/**
* The end of this facet.range as calculated using the value of facet.range.end
* parameter and facet.range.hardend. This can be different from the
* value specified in facet.range.end if facet.range.hardend=true
*/
public Object getEndObj() {
return endObj;
}
/**
* Represents a range facet response combined from all shards.
* Provides helper methods to merge facet_ranges response from a shard.
* See {@link #mergeFacetRangesFromShardResponse(LinkedHashMap, SimpleOrderedMap)}
* and {@link #mergeContributionFromShard(SimpleOrderedMap)}
*/
static class DistribRangeFacet {
public SimpleOrderedMap<Object> rangeFacet;
public DistribRangeFacet(SimpleOrderedMap<Object> rangeFacet) {
this.rangeFacet = rangeFacet;
}
/**
* Helper method to merge range facet values from a shard's response to already accumulated
* values for each range.
*
* @param rangeCounts a {@link LinkedHashMap} containing the accumulated values for each range
* keyed by the 'key' of the facet.range. Must not be null.
* @param shardRanges the facet_ranges response from a shard. Must not be null.
*/
public static void mergeFacetRangesFromShardResponse(LinkedHashMap<String, DistribRangeFacet> rangeCounts,
SimpleOrderedMap<SimpleOrderedMap<Object>> shardRanges) {
assert shardRanges != null;
assert rangeCounts != null;
for (Map.Entry<String, SimpleOrderedMap<Object>> entry : shardRanges) {
String rangeKey = entry.getKey();
RangeFacetRequest.DistribRangeFacet existing = rangeCounts.get(rangeKey);
if (existing == null) {
rangeCounts.put(rangeKey, new RangeFacetRequest.DistribRangeFacet(entry.getValue()));
} else {
existing.mergeContributionFromShard(entry.getValue());
}
}
}
/**
* Accumulates an individual facet_ranges count from a shard into global counts.
* <p>
* The implementation below uses the first encountered shard's
* facet_ranges as the basis for subsequent shards' data to be merged.
*
* @param rangeFromShard the facet_ranges response from a shard
*/
public void mergeContributionFromShard(SimpleOrderedMap<Object> rangeFromShard) {
if (rangeFacet == null) {
rangeFacet = rangeFromShard;
return;
}
@SuppressWarnings("unchecked")
NamedList<Integer> shardFieldValues
= (NamedList<Integer>) rangeFromShard.get("counts");
@SuppressWarnings("unchecked")
NamedList<Integer> existFieldValues
= (NamedList<Integer>) rangeFacet.get("counts");
for (Map.Entry<String, Integer> existPair : existFieldValues) {
final String key = existPair.getKey();
// can be null if inconsistencies in shards responses
Integer newValue = shardFieldValues.get(key);
if (null != newValue) {
Integer oldValue = existPair.getValue();
existPair.setValue(oldValue + newValue);
}
}
// merge facet.other=before/between/after/all if they exist
for (FacetParams.FacetRangeOther otherKey : FacetParams.FacetRangeOther.values()) {
if (otherKey == FacetParams.FacetRangeOther.NONE) continue;
String name = otherKey.toString();
Integer shardValue = (Integer) rangeFromShard.get(name);
if (shardValue != null && shardValue > 0) {
Integer existingValue = (Integer) rangeFacet.get(name);
// shouldn't be null
int idx = rangeFacet.indexOf(name, 0);
rangeFacet.setVal(idx, existingValue + shardValue);
}
}
}
/**
* Removes all counts under the given minCount from the accumulated facet_ranges.
* <p>
* Note: this method should only be called after all shard responses have been
* accumulated using {@link #mergeContributionFromShard(SimpleOrderedMap)}
*
* @param minCount the minimum allowed count for any range
*/
public void removeRangeFacetsUnderLimits(int minCount) {
boolean replace = false;
@SuppressWarnings("unchecked")
NamedList<Number> vals = (NamedList<Number>) rangeFacet.get("counts");
NamedList<Number> newList = new NamedList<>();
for (Map.Entry<String, Number> pair : vals) {
if (pair.getValue().longValue() >= minCount) {
newList.add(pair.getKey(), pair.getValue());
} else {
replace = true;
}
}
if (replace) {
vals.clear();
vals.addAll(newList);
}
}
}
/**
* Perhaps someday instead of having a giant "instanceof" case
* statement to pick an impl, we can add a "RangeFacetable" marker
* interface to FieldTypes and they can return instances of these
* directly from some method -- but until then, keep this locked down
* and private.
*/
private static abstract class RangeEndpointCalculator<T extends Comparable<T>> {
protected final RangeFacetRequest rfr;
protected final SchemaField field;
/**
* The end of the facet.range as determined by this calculator.
* This can be different from the facet.range.end depending on the
* facet.range.hardend parameter
*/
protected T computedEnd;
protected T start;
protected Object gap;
protected boolean computed = false;
public RangeEndpointCalculator(RangeFacetRequest rfr) {
this.rfr = rfr;
this.field = rfr.getSchemaField();
}
public T getComputedEnd() {
assert computed;
return computedEnd;
}
public T getStart() {
assert computed;
return start;
}
/**
* @return the parsed value of {@link FacetParams#FACET_RANGE_GAP} parameter. This type
* of the returned object is the boxed type of the schema field type's primitive counterpart
* except in the case of Dates in which case the returned type is just a string (because in
* case of dates the gap can either be a date or a DateMath string).
*/
public Object getGap() {
assert computed;
return gap;
}
/**
* Formats a Range endpoint for use as a range label name in the response.
* Default Impl just uses toString()
*/
public String formatValue(final T val) {
return val.toString();
}
/**
* Parses a String param into an Range endpoint value throwing
* a useful exception if not possible
*/
public final T getValue(final String rawval) {
try {
return parseVal(rawval);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Can't parse value " + rawval + " for field: " +
field.getName(), e);
}
}
/**
* Parses a String param into an Range endpoint.
* Can throw a low level format exception as needed.
*/
protected abstract T parseVal(final String rawval)
throws java.text.ParseException;
/**
* Parses a String param into a value that represents the gap and
* can be included in the response, throwing
* a useful exception if not possible.
* <p>
* Note: uses Object as the return type instead of T for things like
* Date where gap is just a DateMathParser string
*/
protected final Object getGap(final String gap) {
try {
return parseGap(gap);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Can't parse gap " + gap + " for field: " +
field.getName(), e);
}
}
/**
* Parses a String param into a value that represents the gap and
* can be included in the response.
* Can throw a low level format exception as needed.
* <p>
* Default Impl calls parseVal
*/
protected Object parseGap(final String rawval)
throws java.text.ParseException {
return parseVal(rawval);
}
/**
* Adds the String gap param to a low Range endpoint value to determine
* the corresponding high Range endpoint value, throwing
* a useful exception if not possible.
*/
public final T addGap(T value, String gap) {
try {
return parseAndAddGap(value, gap);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Can't add gap " + gap + " to value " + value +
" for field: " + field.getName(), e);
}
}
/**
* Adds the String gap param to a low Range endpoint value to determine
* the corrisponding high Range endpoint value.
* Can throw a low level format exception as needed.
*/
protected abstract T parseAndAddGap(T value, String gap)
throws java.text.ParseException;
public List<FacetRange> computeRanges() {
List<FacetRange> ranges = new ArrayList<>();
this.gap = getGap(rfr.getGap());
this.start = getValue(rfr.getStart());
// not final, hardend may change this
T end = getValue(rfr.getEnd());
if (end.compareTo(start) < 0) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"range facet 'end' comes before 'start': " + end + " < " + start);
}
final EnumSet<FacetParams.FacetRangeInclude> include = rfr.getInclude();
T low = start;
while (low.compareTo(end) < 0) {
T high = addGap(low, rfr.getGap());
if (end.compareTo(high) < 0) {
if (rfr.isHardEnd()) {
high = end;
} else {
end = high;
}
}
if (high.compareTo(low) < 0) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"range facet infinite loop (is gap negative? did the math overflow?)");
}
if (high.compareTo(low) == 0) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"range facet infinite loop: gap is either zero, or too small relative start/end and caused underflow: " + low + " + " + rfr.getGap() + " = " + high);
}
final boolean includeLower =
(include.contains(FacetParams.FacetRangeInclude.LOWER) ||
(include.contains(FacetParams.FacetRangeInclude.EDGE) &&
0 == low.compareTo(start)));
final boolean includeUpper =
(include.contains(FacetParams.FacetRangeInclude.UPPER) ||
(include.contains(FacetParams.FacetRangeInclude.EDGE) &&
0 == high.compareTo(end)));
final String lowS = formatValue(low);
final String highS = formatValue(high);
ranges.add(new FacetRange(lowS, lowS, highS, includeLower, includeUpper));
low = high;
}
// we must update the end value in RangeFacetRequest because the end is returned
// as a separate element in the range facet response
this.computedEnd = end;
this.computed = true;
// no matter what other values are listed, we don't do
// anything if "none" is specified.
if (!rfr.getOthers().contains(FacetParams.FacetRangeOther.NONE)) {
boolean all = rfr.getOthers().contains(FacetParams.FacetRangeOther.ALL);
final String startS = formatValue(start);
final String endS = formatValue(end);
if (all || rfr.getOthers().contains(FacetParams.FacetRangeOther.BEFORE)) {
// include upper bound if "outer" or if first gap doesn't already include it
ranges.add(new FacetRange(FacetParams.FacetRangeOther.BEFORE,
null, startS, false, include.contains(FacetParams.FacetRangeInclude.OUTER) || include.contains(FacetParams.FacetRangeInclude.ALL) ||
!(include.contains(FacetParams.FacetRangeInclude.LOWER) || include.contains(FacetParams.FacetRangeInclude.EDGE))));
}
if (all || rfr.getOthers().contains(FacetParams.FacetRangeOther.AFTER)) {
// include lower bound if "outer" or if last gap doesn't already include it
ranges.add(new FacetRange(FacetParams.FacetRangeOther.AFTER,
endS, null, include.contains(FacetParams.FacetRangeInclude.OUTER) || include.contains(FacetParams.FacetRangeInclude.ALL) ||
!(include.contains(FacetParams.FacetRangeInclude.UPPER) || include.contains(FacetParams.FacetRangeInclude.EDGE)), false));
}
if (all || rfr.getOthers().contains(FacetParams.FacetRangeOther.BETWEEN)) {
ranges.add(new FacetRange(FacetParams.FacetRangeOther.BETWEEN, startS, endS,
include.contains(FacetParams.FacetRangeInclude.LOWER) || include.contains(FacetParams.FacetRangeInclude.EDGE) || include.contains(FacetParams.FacetRangeInclude.ALL),
include.contains(FacetParams.FacetRangeInclude.UPPER) || include.contains(FacetParams.FacetRangeInclude.EDGE) || include.contains(FacetParams.FacetRangeInclude.ALL)));
}
}
return ranges;
}
}
private static class FloatRangeEndpointCalculator
extends RangeEndpointCalculator<Float> {
public FloatRangeEndpointCalculator(final RangeFacetRequest rangeFacetRequest) {
super(rangeFacetRequest);
}
@Override
protected Float parseVal(String rawval) {
return Float.valueOf(rawval);
}
@Override
public Float parseAndAddGap(Float value, String gap) {
return new Float(value.floatValue() + Float.valueOf(gap).floatValue());
}
}
private static class DoubleRangeEndpointCalculator
extends RangeEndpointCalculator<Double> {
public DoubleRangeEndpointCalculator(final RangeFacetRequest rangeFacetRequest) {
super(rangeFacetRequest);
}
@Override
protected Double parseVal(String rawval) {
return Double.valueOf(rawval);
}
@Override
public Double parseAndAddGap(Double value, String gap) {
return new Double(value.doubleValue() + Double.valueOf(gap).doubleValue());
}
}
private static class IntegerRangeEndpointCalculator
extends RangeEndpointCalculator<Integer> {
public IntegerRangeEndpointCalculator(final RangeFacetRequest rangeFacetRequest) {
super(rangeFacetRequest);
}
@Override
protected Integer parseVal(String rawval) {
return Integer.valueOf(rawval);
}
@Override
public Integer parseAndAddGap(Integer value, String gap) {
return new Integer(value.intValue() + Integer.valueOf(gap).intValue());
}
}
private static class LongRangeEndpointCalculator
extends RangeEndpointCalculator<Long> {
public LongRangeEndpointCalculator(final RangeFacetRequest rangeFacetRequest) {
super(rangeFacetRequest);
}
@Override
protected Long parseVal(String rawval) {
return Long.valueOf(rawval);
}
@Override
public Long parseAndAddGap(Long value, String gap) {
return new Long(value.longValue() + Long.valueOf(gap).longValue());
}
}
private static class DateRangeEndpointCalculator
extends RangeEndpointCalculator<Date> {
private static final String TYPE_ERR_MSG = "SchemaField must use field type extending TrieDateField or DateRangeField";
private final Date now;
public DateRangeEndpointCalculator(final RangeFacetRequest rangeFacetRequest,
final Date now) {
super(rangeFacetRequest);
this.now = now;
if (!(field.getType() instanceof TrieDateField)) {
throw new IllegalArgumentException
(TYPE_ERR_MSG);
}
}
@Override
public String formatValue(Date val) {
return ((TrieDateField) field.getType()).toExternal(val);
}
@Override
protected Date parseVal(String rawval) {
return ((TrieDateField) field.getType()).parseMath(now, rawval);
}
@Override
protected Object parseGap(final String rawval) {
return rawval;
}
@Override
public Date parseAndAddGap(Date value, String gap) throws java.text.ParseException {
final DateMathParser dmp = new DateMathParser();
dmp.setNow(value);
return dmp.parseMath(gap);
}
}
private static class DateRangeFieldEndpointCalculator
extends RangeEndpointCalculator<Date> {
private final Date now;
public DateRangeFieldEndpointCalculator(final RangeFacetRequest rangeFacetRequest,
final Date now) {
super(rangeFacetRequest);
this.now = now;
if (!(field.getType() instanceof DateRangeField)) {
throw new IllegalArgumentException(DateRangeEndpointCalculator.TYPE_ERR_MSG);
}
}
@Override
public String formatValue(Date val) {
return TrieDateField.formatExternal(val);
}
@Override
protected Date parseVal(String rawval) {
return ((DateRangeField) field.getType()).parseMath(now, rawval);
}
@Override
protected Object parseGap(final String rawval) {
return rawval;
}
@Override
public Date parseAndAddGap(Date value, String gap) throws java.text.ParseException {
final DateMathParser dmp = new DateMathParser();
dmp.setNow(value);
return dmp.parseMath(gap);
}
}
/**
* Represents a single facet range (or gap) for which the count is to be calculated
*/
public static class FacetRange {
public final FacetParams.FacetRangeOther other;
public final String name;
public final String lower;
public final String upper;
public final boolean includeLower;
public final boolean includeUpper;
private FacetRange(FacetParams.FacetRangeOther other, String name, String lower, String upper, boolean includeLower, boolean includeUpper) {
this.other = other;
this.name = name;
this.lower = lower;
this.upper = upper;
this.includeLower = includeLower;
this.includeUpper = includeUpper;
}
/**
* Construct a facet range for a {@link org.apache.solr.common.params.FacetParams.FacetRangeOther} instance
*/
public FacetRange(FacetParams.FacetRangeOther other, String lower, String upper, boolean includeLower, boolean includeUpper) {
this(other, other.toString(), lower, upper, includeLower, includeUpper);
}
/**
* Construct a facet range for the give name
*/
public FacetRange(String name, String lower, String upper, boolean includeLower, boolean includeUpper) {
this(null, name, lower, upper, includeLower, includeUpper);
}
}
}

6
solr/core/src/java/org/apache/solr/request/IntervalFacets.java
View File

@ -120,7 +120,7 @@ public class IntervalFacets implements Iterable<FacetInterval> {
* Constructor that accepts an already constructed array of {@link FacetInterval} objects. This array needs to be sorted
* by start value in weakly ascending order. null values are not allowed in the array.
*/
IntervalFacets(SchemaField schemaField, SolrIndexSearcher searcher, DocSet docs, FacetInterval[] intervals) throws IOException {
public IntervalFacets(SchemaField schemaField, SolrIndexSearcher searcher, DocSet docs, FacetInterval[] intervals) throws IOException {
this.schemaField = schemaField;
this.searcher = searcher;
this.docs = docs;
@ -360,7 +360,7 @@ public class IntervalFacets implements Iterable<FacetInterval> {
/**
* Helper class to match and count of documents in specified intervals
*/
static class FacetInterval {
public static class FacetInterval {
/**
* Key to represent this interval
@ -519,7 +519,7 @@ public class IntervalFacets implements Iterable<FacetInterval> {
* @param includeUpper Indicates weather this interval should include values equal to end
* @param key String key of this interval
*/
FacetInterval(SchemaField schemaField, String startStr, String endStr,
public FacetInterval(SchemaField schemaField, String startStr, String endStr,
boolean includeLower, boolean includeUpper, String key) {
assert schemaField.getType().getNumericType() != null: "Only numeric fields supported with this constructor";
this.key = key;

1030
solr/core/src/java/org/apache/solr/request/SimpleFacets.java
View File

File diff suppressed because it is too large Load Diff

4
solr/core/src/java/org/apache/solr/util/PivotListEntry.java
View File

@ -34,7 +34,9 @@ public enum PivotListEntry {
COUNT(2),
// optional entries
PIVOT,
STATS;
STATS,
QUERIES,
RANGES;
private static final int MIN_INDEX_OF_OPTIONAL = 3;

114
solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java
View File

@ -17,21 +17,21 @@ package org.apache.solr.handler.component;
* limitations under the License.
*/
import java.io.IOException;
import java.util.Date;
import java.util.List;
import java.io.IOException;
import junit.framework.AssertionFailedError;
import org.apache.solr.BaseDistributedSearchTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.RangeFacet;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.SolrParams;
import junit.framework.AssertionFailedError;
import org.junit.Test;
public class DistributedFacetPivotLargeTest extends BaseDistributedSearchTestCase {
@ -665,6 +665,7 @@ public class DistributedFacetPivotLargeTest extends BaseDistributedSearchTestCas
FacetParams.FACET_OVERREQUEST_COUNT, "0");
doTestDeepPivotStats();
doTestPivotRanges();
}
private void doTestDeepPivotStats() throws Exception {
@ -739,6 +740,101 @@ public class DistributedFacetPivotLargeTest extends BaseDistributedSearchTestCas
assertEquals(1741.742422595, microsoftPlaceholder0PivotFieldStatsInfo.getStddev(), 0.1E-7);
}
/**
* spot checks some pivot values and the ranges hanging on them
*/
private void doTestPivotRanges() throws Exception {
// note: 'p0' is only a top level range, not included in per-pivot ranges
for (SolrParams p : new SolrParams[]{
// results should be identical for all of these
params("facet.range", "{!key=p0 facet.range.gap=500}pay_i",
"facet.range", "{!key=p1 tag=t1 facet.range.gap=100}pay_i",
"facet.range", "{!key=p2 tag=t1 facet.range.gap=200}pay_i",
"facet.range.start", "0",
"facet.range.end", "1000"),
params("facet.range", "{!key=p0 facet.range.gap=500}pay_i",
"facet.range", "{!key=p1 tag=t1 facet.range.gap=100}pay_i",
"facet.range", "{!key=p2 tag=t1 facet.range.gap=200}pay_i",
"f.pay_i.facet.range.start", "0",
"facet.range.end", "1000"),
params("facet.range", "{!key=p0 facet.range.gap=500 facet.range.start=0}pay_i",
"facet.range", "{!key=p1 tag=t1 facet.range.gap=100 facet.range.start=0}pay_i",
"facet.range", "{!key=p2 tag=t1 facet.range.gap=200 facet.range.start=0}pay_i",
"facet.range.end", "1000")}) {
QueryResponse rsp
= query(SolrParams.wrapDefaults(p, params("q", "*:*",
"rows", "0",
"facet", "true",
"facet.pivot", "{!range=t1}place_s,company_t")));
List<PivotField> pivots = rsp.getFacetPivot().get("place_s,company_t");
PivotField pf = null; // changes as we spot check
List<RangeFacet.Count> rfc = null; // changes as we spot check
// 1st sanity check top level ranges
assertEquals(3, rsp.getFacetRanges().size());
assertRange("p0", 0, 500, 1000, 2, rsp.getFacetRanges().get(0));
assertRange("p1", 0, 100, 1000, 10, rsp.getFacetRanges().get(1));
assertRange("p2", 0, 200, 1000, 5, rsp.getFacetRanges().get(2));
// check pivots...
// first top level pivot value
pf = pivots.get(0);
assertPivot("place_s", "cardiff", 257, pf);
assertRange("p1", 0, 100, 1000, 10, pf.getFacetRanges().get(0));
assertRange("p2", 0, 200, 1000, 5, pf.getFacetRanges().get(1));
rfc = pf.getFacetRanges().get(0).getCounts();
assertEquals("200", rfc.get(2).getValue());
assertEquals(14, rfc.get(2).getCount());
assertEquals("300", rfc.get(3).getValue());
assertEquals(15, rfc.get(3).getCount());
rfc = pf.getFacetRanges().get(1).getCounts();
assertEquals("200", rfc.get(1).getValue());
assertEquals(29, rfc.get(1).getCount());
// drill down one level of the pivot
pf = pf.getPivot().get(0);
assertPivot("company_t", "bbc", 101, pf);
assertRange("p1", 0, 100, 1000, 10, pf.getFacetRanges().get(0));
assertRange("p2", 0, 200, 1000, 5, pf.getFacetRanges().get(1));
rfc = pf.getFacetRanges().get(0).getCounts();
for (RangeFacet.Count c : rfc) {
assertEquals(0, c.getCount()); // no docs in our ranges for this pivot drill down
}
// pop back up and spot check a different top level pivot value
pf = pivots.get(53);
assertPivot("place_s", "placeholder0", 1, pf);
assertRange("p1", 0, 100, 1000, 10, pf.getFacetRanges().get(0));
assertRange("p2", 0, 200, 1000, 5, pf.getFacetRanges().get(1));
rfc = pf.getFacetRanges().get(0).getCounts();
assertEquals("0", rfc.get(0).getValue());
assertEquals(1, rfc.get(0).getCount());
assertEquals("100", rfc.get(1).getValue());
assertEquals(0, rfc.get(1).getCount());
// drill down one level of the pivot
pf = pf.getPivot().get(0);
assertPivot("company_t", "compholder0", 1, pf);
assertRange("p1", 0, 100, 1000, 10, pf.getFacetRanges().get(0));
assertRange("p2", 0, 200, 1000, 5, pf.getFacetRanges().get(1));
rfc = pf.getFacetRanges().get(0).getCounts();
assertEquals("0", rfc.get(0).getValue());
assertEquals(1, rfc.get(0).getCount());
assertEquals("100", rfc.get(1).getValue());
assertEquals(0, rfc.get(1).getCount());
}
}
/**
* asserts that the actual PivotField matches the expected criteria
*/
@ -751,8 +847,18 @@ public class DistributedFacetPivotLargeTest extends BaseDistributedSearchTestCas
//assertEquals("#KIDS: " + actual.toString(), numKids, actual.getPivot().size());
}
/**
* asserts that the actual RangeFacet matches the expected criteria
*/
private void assertRange(String name, Object start, Object gap, Object end, int numCount,
RangeFacet actual) {
assertEquals("NAME: " + actual.toString(), name, actual.getName());
assertEquals("START: " + actual.toString(), start, actual.getStart());
assertEquals("GAP: " + actual.toString(), gap, actual.getGap());
assertEquals("END: " + actual.toString(), end, actual.getEnd());
assertEquals("#COUNT: " + actual.toString(), numCount, actual.getCounts().size());
}
private void setupDistributedPivotFacetDocuments() throws Exception{
//Clear docs

1097
solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java
View File

File diff suppressed because it is too large Load Diff

26
solr/solrj/src/java/org/apache/solr/client/solrj/response/PivotField.java
View File

@ -22,6 +22,8 @@ import java.io.Serializable;
import java.util.List;
import java.util.Map;
import org.apache.solr.common.util.NamedList;
public class PivotField implements Serializable
{
final String _field;
@ -29,22 +31,26 @@ public class PivotField implements Serializable
final int _count;
final List<PivotField> _pivot;
final Map<String,FieldStatsInfo> _statsInfo;
final Map<String,Integer> _querycounts;
final List<RangeFacet> _ranges;
/**
* @deprecated Use {@link #PivotField(String,Object,int,List,Map)} with a null <code>statsInfo</code>
* @deprecated Use {@link #PivotField(String,Object,int,List,Map,Map,List)} with null <code>statsInfo</code>, queryCounts and ranges
*/
@Deprecated
public PivotField( String f, Object v, int count, List<PivotField> pivot) {
this(f, v, count, pivot, null);
this(f, v, count, pivot, null, null, null);
}
public PivotField( String f, Object v, int count, List<PivotField> pivot, Map<String,FieldStatsInfo> statsInfo)
public PivotField( String f, Object v, int count, List<PivotField> pivot, Map<String,FieldStatsInfo> statsInfo, Map<String,Integer> queryCounts, List<RangeFacet> ranges)
{
_field = f;
_value = v;
_count = count;
_pivot = pivot;
_statsInfo = statsInfo;
_querycounts= queryCounts;
_ranges= ranges;
}
public String getField() {
@ -67,6 +73,14 @@ public class PivotField implements Serializable
return _statsInfo;
}
public Map<String,Integer> getFacetQuery() {
return _querycounts;
}
public List<RangeFacet> getFacetRanges() {
return _ranges;
}
@Override
public String toString()
{
@ -88,6 +102,12 @@ public class PivotField implements Serializable
out.print("]");
}
out.println();
if(_querycounts != null) {
out.println(_querycounts.toString());
}
if(_ranges != null) {
out.println(_ranges.toString());
}
if( _pivot != null ) {
for( PivotField p : _pivot ) {
p.write( out, indent+1 );

97
solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java
View File

@ -348,41 +348,7 @@ public class QueryResponse extends SolrResponseBase
//Parse range facets
NamedList<NamedList<Object>> rf = (NamedList<NamedList<Object>>) info.get("facet_ranges");
if (rf != null) {
_facetRanges = new ArrayList<>( rf.size() );
for (Map.Entry<String, NamedList<Object>> facet : rf) {
NamedList<Object> values = facet.getValue();
Object rawGap = values.get("gap");
RangeFacet rangeFacet;
if (rawGap instanceof Number) {
Number gap = (Number) rawGap;
Number start = (Number) values.get("start");
Number end = (Number) values.get("end");
Number before = (Number) values.get("before");
Number after = (Number) values.get("after");
Number between = (Number) values.get("between");
rangeFacet = new RangeFacet.Numeric(facet.getKey(), start, end, gap, before, after, between);
} else {
String gap = (String) rawGap;
Date start = (Date) values.get("start");
Date end = (Date) values.get("end");
Number before = (Number) values.get("before");
Number after = (Number) values.get("after");
Number between = (Number) values.get("between");
rangeFacet = new RangeFacet.Date(facet.getKey(), start, end, gap, before, after, between);
}
NamedList<Integer> counts = (NamedList<Integer>) values.get("counts");
for (Map.Entry<String, Integer> entry : counts) {
rangeFacet.addCount(entry.getKey(), entry.getValue());
}
_facetRanges.add(rangeFacet);
}
_facetRanges = extractRangeFacets(rf);
}
//Parse pivot facets
@ -408,7 +374,47 @@ public class QueryResponse extends SolrResponseBase
}
}
}
private List<RangeFacet> extractRangeFacets(NamedList<NamedList<Object>> rf) {
List<RangeFacet> facetRanges = new ArrayList<>( rf.size() );
for (Map.Entry<String, NamedList<Object>> facet : rf) {
NamedList<Object> values = facet.getValue();
Object rawGap = values.get("gap");
RangeFacet rangeFacet;
if (rawGap instanceof Number) {
Number gap = (Number) rawGap;
Number start = (Number) values.get("start");
Number end = (Number) values.get("end");
Number before = (Number) values.get("before");
Number after = (Number) values.get("after");
Number between = (Number) values.get("between");
rangeFacet = new RangeFacet.Numeric(facet.getKey(), start, end, gap, before, after, between);
} else {
String gap = (String) rawGap;
Date start = (Date) values.get("start");
Date end = (Date) values.get("end");
Number before = (Number) values.get("before");
Number after = (Number) values.get("after");
Number between = (Number) values.get("between");
rangeFacet = new RangeFacet.Date(facet.getKey(), start, end, gap, before, after, between);
}
NamedList<Integer> counts = (NamedList<Integer>) values.get("counts");
for (Map.Entry<String, Integer> entry : counts) {
rangeFacet.addCount(entry.getKey(), entry.getValue());
}
facetRanges.add(rangeFacet);
}
return facetRanges;
}
protected List<PivotField> readPivots( List<NamedList> list )
{
ArrayList<PivotField> values = new ArrayList<>( list.size() );
@ -423,6 +429,8 @@ public class QueryResponse extends SolrResponseBase
List<PivotField> subPivots = null;
Map<String,FieldStatsInfo> fieldStatsInfos = null;
Map<String,Integer> queryCounts = null;
List<RangeFacet> ranges = null;
if (4 <= nl.size()) {
for(int index = 3; index < nl.size(); index++) {
@ -444,6 +452,21 @@ public class QueryResponse extends SolrResponseBase
fieldStatsInfos = extractFieldStatsInfo((NamedList<Object>) val);
break;
}
case "queries": {
// Parse the queries
queryCounts = new LinkedHashMap<>();
NamedList<Integer> fq = (NamedList<Integer>) val;
if (fq != null) {
for( Map.Entry<String, Integer> entry : fq ) {
queryCounts.put( entry.getKey(), entry.getValue() );
}
}
break;
}
case "ranges": {
ranges = extractRangeFacets((NamedList<NamedList<Object>>) val);
break;
}
default:
throw new RuntimeException( "unknown key in pivot: "+ key+ " ["+val+"]");
@ -451,7 +474,7 @@ public class QueryResponse extends SolrResponseBase
}
}
values.add( new PivotField( f, v, cnt, subPivots, fieldStatsInfos ) );
values.add( new PivotField( f, v, cnt, subPivots, fieldStatsInfos, queryCounts, ranges ) );
}
return values;
}

217
solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
View File

@ -42,7 +42,9 @@ import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.LukeResponse;
import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.RangeFacet;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.client.solrj.response.RangeFacet.Count;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
@ -65,6 +67,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Random;
@ -1094,6 +1097,218 @@ abstract public class SolrExampleTests extends SolrExampleTestsBase
}
@Test
public void testPivotFacetsQueries() throws Exception {
SolrClient client = getSolrClient();
// Empty the database...
client.deleteByQuery("*:*");// delete everything!
client.commit();
assertNumFound("*:*", 0); // make sure it got in
int id = 1;
ArrayList<SolrInputDocument> docs = new ArrayList<>();
docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "a", "inStock", true, "popularity", 12, "price", .017));
docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "a", "inStock", false, "popularity", 13, "price", 16.04));
docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "a", "inStock", true, "popularity", 14, "price", 12.34));
docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "b", "inStock", false, "popularity", 24, "price", 51.39));
docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "b", "inStock", true, "popularity", 28, "price", 131.39));
docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "a", "inStock", false, "popularity", 32));
docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "a", "inStock", true, "popularity", 31, "price", 131.39));
docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", false, "popularity", 36));
docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", true, "popularity", 37, "price", 1.39));
docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", false, "popularity", 38, "price", 47.98));
docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", true, "popularity", -38));
docs.add(makeTestDoc("id", id++, "cat", "b")); // something not matching all fields
client.add(docs);
client.commit();
SolrQuery query = new SolrQuery("*:*");
query.addFacetPivotField("{!query=s1}features,manu");
query.addFacetQuery("{!key=highPrice tag=s1}price:[100 TO *]");
query.addFacetQuery("{!tag=s1 key=lowPrice}price:[0 TO 50]");
query.setFacetMinCount(0);
query.setRows(0);
QueryResponse rsp = client.query(query);
Map<String,Integer> map = rsp.getFacetQuery();
assertEquals(2, map.get("highPrice").intValue());
assertEquals(5, map.get("lowPrice").intValue());
NamedList<List<PivotField>> pivots = rsp.getFacetPivot();
List<PivotField> pivotValues = pivots.get("features,manu");
PivotField featuresBBBPivot = pivotValues.get(0);
assertEquals("features", featuresBBBPivot.getField());
assertEquals("bbb", featuresBBBPivot.getValue());
assertNotNull(featuresBBBPivot.getFacetQuery());
assertEquals(2, featuresBBBPivot.getFacetQuery().size());
assertEquals(1, featuresBBBPivot.getFacetQuery().get("highPrice").intValue());
assertEquals(2, featuresBBBPivot.getFacetQuery().get("lowPrice").intValue());
PivotField featuresAAAPivot = pivotValues.get(1);
assertEquals("features", featuresAAAPivot.getField());
assertEquals("aaa", featuresAAAPivot.getValue());
assertNotNull(featuresAAAPivot.getFacetQuery());
assertEquals(2, featuresAAAPivot.getFacetQuery().size());
assertEquals(1, featuresAAAPivot.getFacetQuery().get("highPrice").intValue());
assertEquals(3, featuresAAAPivot.getFacetQuery().get("lowPrice").intValue());
}
@Test
public void testPivotFacetsRanges() throws Exception {
SolrClient client = getSolrClient();
// Empty the database...
client.deleteByQuery("*:*");// delete everything!
client.commit();
assertNumFound("*:*", 0); // make sure it got in
int id = 1;
ArrayList<SolrInputDocument> docs = new ArrayList<>();
docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "a", "inStock", true, "popularity", 12, "price", .017));
docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "a", "inStock", false, "popularity", 13, "price", 16.04));
docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "a", "inStock", true, "popularity", 14, "price", 12.34));
docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "b", "inStock", false, "popularity", 24, "price", 51.39));
docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "b", "inStock", true, "popularity", 28, "price", 131.39));
docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "a", "inStock", false, "popularity", 32));
docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "a", "inStock", true, "popularity", 31, "price", 131.39));
docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", false, "popularity", 36));
docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", true, "popularity", 37, "price", 1.39));
docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", false, "popularity", 38, "price", 47.98));
docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", true, "popularity", -38));
docs.add(makeTestDoc("id", id++, "cat", "b")); // something not matching all fields
client.add(docs);
client.commit();
SolrQuery query = new SolrQuery("*:*");
query.addFacetPivotField("{!range=s1}features,manu");
query.add(FacetParams.FACET_RANGE, "{!key=price1 tag=s1}price");
query.add(String.format(Locale.ROOT, "f.%s.%s", "price", FacetParams.FACET_RANGE_START), "0");
query.add(String.format(Locale.ROOT, "f.%s.%s", "price", FacetParams.FACET_RANGE_END), "200");
query.add(String.format(Locale.ROOT, "f.%s.%s", "price", FacetParams.FACET_RANGE_GAP), "50");
query.set(FacetParams.FACET, true);
query.add(FacetParams.FACET_RANGE, "{!key=price2 tag=s1}price");
query.setFacetMinCount(0);
query.setRows(0);
QueryResponse rsp = client.query(query);
List<RangeFacet> list = rsp.getFacetRanges();
assertEquals(2, list.size());
@SuppressWarnings("unchecked")
RangeFacet<Float, Float> range1 = list.get(0);
assertEquals("price1", range1.getName());
assertEquals(0, range1.getStart().intValue());
assertEquals(200, range1.getEnd().intValue());
assertEquals(50, range1.getGap().intValue());
List<Count> counts1 = range1.getCounts();
assertEquals(4, counts1.size());
assertEquals(5, counts1.get(0).getCount());
assertEquals("0.0", counts1.get(0).getValue());
assertEquals(1, counts1.get(1).getCount());
assertEquals("50.0", counts1.get(1).getValue());
assertEquals(2, counts1.get(2).getCount());
assertEquals("100.0", counts1.get(2).getValue());
assertEquals(0, counts1.get(3).getCount());
assertEquals("150.0", counts1.get(3).getValue());
@SuppressWarnings("unchecked")
RangeFacet<Float, Float> range2 = list.get(1);
assertEquals("price2", range2.getName());
assertEquals(0, range2.getStart().intValue());
assertEquals(200, range2.getEnd().intValue());
assertEquals(50, range2.getGap().intValue());
List<Count> counts2 = range2.getCounts();
assertEquals(4, counts2.size());
assertEquals(5, counts2.get(0).getCount());
assertEquals("0.0", counts2.get(0).getValue());
assertEquals(1, counts2.get(1).getCount());
assertEquals("50.0", counts2.get(1).getValue());
assertEquals(2, counts2.get(2).getCount());
assertEquals("100.0", counts2.get(2).getValue());
assertEquals(0, counts2.get(3).getCount());
assertEquals("150.0", counts2.get(3).getValue());
NamedList<List<PivotField>> pivots = rsp.getFacetPivot();
List<PivotField> pivotValues = pivots.get("features,manu");
PivotField featuresBBBPivot = pivotValues.get(0);
assertEquals("features", featuresBBBPivot.getField());
assertEquals("bbb", featuresBBBPivot.getValue());
List<RangeFacet> featuresBBBRanges = featuresBBBPivot.getFacetRanges();
for (RangeFacet range : featuresBBBRanges) {
if (range.getName().equals("price1")) {
assertNotNull(range);
assertEquals(0, ((Float)range.getStart()).intValue());
assertEquals(200, ((Float)range.getEnd()).intValue());
assertEquals(50, ((Float)range.getGap()).intValue());
List<Count> counts = range.getCounts();
assertEquals(4, counts.size());
for (Count count : counts) {
switch (count.getValue()) {
case "0.0": assertEquals(2, count.getCount()); break;
case "50.0": assertEquals(0, count.getCount()); break;
case "100.0": assertEquals(1, count.getCount()); break;
case "150.0": assertEquals(0, count.getCount()); break;
}
}
} else if (range.getName().equals("price2")) {
assertNotNull(range);
assertEquals(0, ((Float) range.getStart()).intValue());
assertEquals(200, ((Float) range.getEnd()).intValue());
assertEquals(50, ((Float) range.getGap()).intValue());
List<Count> counts = range.getCounts();
assertEquals(4, counts.size());
for (Count count : counts) {
switch (count.getValue()) {
case "0.0": assertEquals(2, count.getCount()); break;
case "50.0": assertEquals(0, count.getCount()); break;
case "100.0": assertEquals(1, count.getCount()); break;
case "150.0": assertEquals(0, count.getCount()); break;
}
}
}
}
PivotField featuresAAAPivot = pivotValues.get(1);
assertEquals("features", featuresAAAPivot.getField());
assertEquals("aaa", featuresAAAPivot.getValue());
List<RangeFacet> facetRanges = featuresAAAPivot.getFacetRanges();
for (RangeFacet range : facetRanges) {
if (range.getName().equals("price1")) {
assertNotNull(range);
assertEquals(0, ((Float)range.getStart()).intValue());
assertEquals(200, ((Float)range.getEnd()).intValue());
assertEquals(50, ((Float)range.getGap()).intValue());
List<Count> counts = range.getCounts();
assertEquals(4, counts.size());
for (Count count : counts) {
switch (count.getValue()) {
case "0.0": assertEquals(3, count.getCount()); break;
case "50.0": assertEquals(1, count.getCount()); break;
case "100.0": assertEquals(1, count.getCount()); break;
case "150.0": assertEquals(0, count.getCount()); break;
}
}
} else if (range.getName().equals("price2")) {
assertNotNull(range);
assertEquals(0, ((Float)range.getStart()).intValue());
assertEquals(200, ((Float)range.getEnd()).intValue());
assertEquals(50, ((Float)range.getGap()).intValue());
List<Count> counts = range.getCounts();
assertEquals(4, counts.size());
for (Count count : counts) {
switch (count.getValue()) {
case "0.0": assertEquals(3, count.getCount()); break;
case "50.0": assertEquals(1, count.getCount()); break;
case "100.0": assertEquals(1, count.getCount()); break;
case "150.0": assertEquals(0, count.getCount()); break;
}
}
}
}
}
public void testPivotFacetsMissing() throws Exception {
doPivotFacetTest(true);
}
@ -1870,4 +2085,4 @@ abstract public class SolrExampleTests extends SolrExampleTestsBase
}
return sdoc;
}
}
}