diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 0d1f9c50d8a..020bc266ccd 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -155,6 +155,9 @@ New Features * SOLR-7651: New response format added wt=smile (noble) +* SOLR-4212: SOLR-6353: Let facet queries and facet ranges hang off of pivots. Example: + facet.range={!tag=r1}price&facet.query={!tag=q1}somequery&facet.pivot={!range=r1 query=q1}category,manufacturer + Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index da744bb3ec5..6df2de5221f 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -37,6 +37,10 @@ import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.component.FacetComponent; +import org.apache.solr.handler.component.SpatialHeatmapFacets; +import org.apache.solr.handler.component.DateFacetProcessor; +import org.apache.solr.handler.component.RangeFacetProcessor; import org.apache.solr.request.SimpleFacets; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -231,7 +235,7 @@ public class MoreLikeThisHandler extends RequestHandlerBase rsp.add("facet_counts", null); } else { SimpleFacets f = new SimpleFacets(req, mltDocs.docSet, params); - rsp.add("facet_counts", f.getFacetCounts()); + rsp.add("facet_counts", FacetComponent.getFacetCounts(f)); } } boolean dbg = req.getParams().getBool(CommonParams.DEBUG_QUERY, false); diff --git a/solr/core/src/java/org/apache/solr/handler/component/DateFacetProcessor.java b/solr/core/src/java/org/apache/solr/handler/component/DateFacetProcessor.java new file mode 100644 index 00000000000..65d92ad4f21 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/component/DateFacetProcessor.java @@ -0,0 +1,253 @@ +package org.apache.solr.handler.component; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Date; +import java.util.EnumSet; +import java.util.Set; + +import org.apache.lucene.search.Query; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.FacetParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.request.SimpleFacets; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.TrieDateField; +import org.apache.solr.search.DocSet; +import org.apache.solr.search.SyntaxError; +import org.apache.solr.util.DateMathParser; + +/** + * Process date facets + * + * @deprecated the whole date faceting feature is deprecated. Use range facets instead which can + * already work with dates. + */ +@Deprecated +public class DateFacetProcessor extends SimpleFacets { + public DateFacetProcessor(SolrQueryRequest req, DocSet docs, SolrParams params, ResponseBuilder rb) { + super(req, docs, params, rb); + } + + /** + * @deprecated Use getFacetRangeCounts which is more generalized + */ + @Deprecated + public void getFacetDateCounts(String dateFacet, NamedList resOuter) + throws IOException { + + final IndexSchema schema = searcher.getSchema(); + + ParsedParams parsed = null; + try { + parsed = parseParams(FacetParams.FACET_DATE, dateFacet); + } catch (SyntaxError syntaxError) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, syntaxError); + } + + final SolrParams params = parsed.params; + final SolrParams required = parsed.required; + final String key = parsed.key; + final String f = parsed.facetValue; + + final NamedList resInner = new SimpleOrderedMap<>(); + resOuter.add(key, resInner); + final SchemaField sf = schema.getField(f); + if (!(sf.getType() instanceof TrieDateField)) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "Can not date facet on a field which is not a TrieDateField: " + f); + } + final TrieDateField ft = (TrieDateField) sf.getType(); + final String startS + = required.getFieldParam(f, FacetParams.FACET_DATE_START); + final Date start; + try { + start = ft.parseMath(null, startS); + } catch (SolrException e) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "date facet 'start' is not a valid Date string: " + startS, e); + } + final String endS + = required.getFieldParam(f, FacetParams.FACET_DATE_END); + Date end; // not final, hardend may change this + try { + end = ft.parseMath(null, endS); + } catch (SolrException e) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "date facet 'end' is not a valid Date string: " + endS, e); + } + + if (end.before(start)) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "date facet 'end' comes before 'start': " + endS + " < " + startS); + } + + final String gap = required.getFieldParam(f, FacetParams.FACET_DATE_GAP); + final DateMathParser dmp = new DateMathParser(); + + final int minCount = params.getFieldInt(f, FacetParams.FACET_MINCOUNT, 0); + + String[] iStrs = params.getFieldParams(f, FacetParams.FACET_DATE_INCLUDE); + // Legacy support for default of [lower,upper,edge] for date faceting + // this is not handled by FacetRangeInclude.parseParam because + // range faceting has differnet defaults + final EnumSet include = + (null == iStrs || 0 == iStrs.length) ? + EnumSet.of(FacetParams.FacetRangeInclude.LOWER, + FacetParams.FacetRangeInclude.UPPER, + FacetParams.FacetRangeInclude.EDGE) + : FacetParams.FacetRangeInclude.parseParam(iStrs); + + try { + Date low = start; + while (low.before(end)) { + dmp.setNow(low); + String label = ft.toExternal(low); + + Date high = dmp.parseMath(gap); + if (end.before(high)) { + if (params.getFieldBool(f, FacetParams.FACET_DATE_HARD_END, false)) { + high = end; + } else { + end = high; + } + } + if (high.before(low)) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "date facet infinite loop (is gap negative?)"); + } + if (high.equals(low)) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "date facet infinite loop: gap is effectively zero"); + } + final boolean includeLower = + (include.contains(FacetParams.FacetRangeInclude.LOWER) || + (include.contains(FacetParams.FacetRangeInclude.EDGE) && low.equals(start))); + final boolean includeUpper = + (include.contains(FacetParams.FacetRangeInclude.UPPER) || + (include.contains(FacetParams.FacetRangeInclude.EDGE) && high.equals(end))); + + final int count = rangeCount(parsed, sf, low, high, includeLower, includeUpper); + if (count >= minCount) { + resInner.add(label, count); + } + low = high; + } + } catch (java.text.ParseException e) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "date facet 'gap' is not a valid Date Math string: " + gap, e); + } + + // explicitly return the gap and end so all the counts + // (including before/after/between) are meaningful - even if mincount + // has removed the neighboring ranges + resInner.add("gap", gap); + resInner.add("start", start); + resInner.add("end", end); + + final String[] othersP = + params.getFieldParams(f, FacetParams.FACET_DATE_OTHER); + if (null != othersP && 0 < othersP.length) { + final Set others = EnumSet.noneOf(FacetParams.FacetRangeOther.class); + + for (final String o : othersP) { + others.add(FacetParams.FacetRangeOther.get(o)); + } + + // no matter what other values are listed, we don't do + // anything if "none" is specified. + if (!others.contains(FacetParams.FacetRangeOther.NONE)) { + boolean all = others.contains(FacetParams.FacetRangeOther.ALL); + + if (all || others.contains(FacetParams.FacetRangeOther.BEFORE)) { + // include upper bound if "outer" or if first gap doesn't already include it + resInner.add(FacetParams.FacetRangeOther.BEFORE.toString(), + rangeCount(parsed, sf, null, start, + false, + (include.contains(FacetParams.FacetRangeInclude.OUTER) || + (!(include.contains(FacetParams.FacetRangeInclude.LOWER) || + include.contains(FacetParams.FacetRangeInclude.EDGE)))))); + } + if (all || others.contains(FacetParams.FacetRangeOther.AFTER)) { + // include lower bound if "outer" or if last gap doesn't already include it + resInner.add(FacetParams.FacetRangeOther.AFTER.toString(), + rangeCount(parsed, sf, end, null, + (include.contains(FacetParams.FacetRangeInclude.OUTER) || + (!(include.contains(FacetParams.FacetRangeInclude.UPPER) || + include.contains(FacetParams.FacetRangeInclude.EDGE)))), + false)); + } + if (all || others.contains(FacetParams.FacetRangeOther.BETWEEN)) { + resInner.add(FacetParams.FacetRangeOther.BETWEEN.toString(), + rangeCount(parsed, sf, start, end, + (include.contains(FacetParams.FacetRangeInclude.LOWER) || + include.contains(FacetParams.FacetRangeInclude.EDGE)), + (include.contains(FacetParams.FacetRangeInclude.UPPER) || + include.contains(FacetParams.FacetRangeInclude.EDGE)))); + } + } + } + } + + /** + * Returns a list of value constraints and the associated facet counts + * for each facet date field, range, and interval specified in the + * SolrParams + * + * @see FacetParams#FACET_DATE + * @deprecated Use getFacetRangeCounts which is more generalized + */ + @Deprecated + public NamedList getFacetDateCounts() + throws IOException { + + final NamedList resOuter = new SimpleOrderedMap<>(); + final String[] fields = global.getParams(FacetParams.FACET_DATE); + + if (null == fields || 0 == fields.length) return resOuter; + + for (String f : fields) { + getFacetDateCounts(f, resOuter); + } + + return resOuter; + } + + /** + * @deprecated Use rangeCount(SchemaField,String,String,boolean,boolean) which is more generalized + */ + @Deprecated + protected int rangeCount(ParsedParams parsed, SchemaField sf, Date low, Date high, + boolean iLow, boolean iHigh) throws IOException { + Query rangeQ = ((TrieDateField) (sf.getType())).getRangeQuery(null, sf, low, high, iLow, iHigh); + return searcher.numDocs(rangeQ, parsed.docs); + } +} + diff --git a/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java index fded4ac3f78..aa10cc37ac3 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java @@ -32,12 +32,13 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; import org.apache.lucene.util.FixedBitSet; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.FacetParams; -import org.apache.solr.common.params.FacetParams.FacetRangeOther; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; @@ -45,6 +46,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; import org.apache.solr.request.SimpleFacets; +import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.FieldType; import org.apache.solr.search.QueryParsing; import org.apache.solr.search.SyntaxError; @@ -61,17 +63,183 @@ public class FacetComponent extends SearchComponent { public static Logger log = LoggerFactory.getLogger(FacetComponent.class); public static final String COMPONENT_NAME = "facet"; - + + public static final String FACET_QUERY_KEY = "facet_queries"; + public static final String FACET_FIELD_KEY = "facet_fields"; + public static final String FACET_DATE_KEY = "facet_dates"; + public static final String FACET_RANGES_KEY = "facet_ranges"; + public static final String FACET_INTERVALS_KEY = "facet_intervals"; + private static final String PIVOT_KEY = "facet_pivot"; private static final String PIVOT_REFINE_PREFIX = "{!"+PivotFacet.REFINE_PARAM+"="; - - @Override public void prepare(ResponseBuilder rb) throws IOException { if (rb.req.getParams().getBool(FacetParams.FACET, false)) { rb.setNeedDocSet(true); rb.doFacets = true; + + // Deduplicate facet params + ModifiableSolrParams params = new ModifiableSolrParams(); + SolrParams origParams = rb.req.getParams(); + Iterator iter = origParams.getParameterNamesIterator(); + while (iter.hasNext()) { + String paramName = iter.next(); + // Deduplicate the list with LinkedHashSet, but _only_ for facet params. + if (!paramName.startsWith(FacetParams.FACET)) { + params.add(paramName, origParams.getParams(paramName)); + continue; + } + HashSet deDupe = new LinkedHashSet<>(Arrays.asList(origParams.getParams(paramName))); + params.add(paramName, deDupe.toArray(new String[deDupe.size()])); + + } + rb.req.setParams(params); + + // Initialize context + FacetContext.initContext(rb); + } + } + + /** + * Encapsulates facet ranges and facet queries such that their parameters + * are parsed and cached for efficient re-use. + *

+ * An instance of this class is initialized and kept in the request context via the static + * method {@link org.apache.solr.handler.component.FacetComponent.FacetContext#initContext(ResponseBuilder)} and + * can be retrieved via {@link org.apache.solr.handler.component.FacetComponent.FacetContext#getFacetContext(SolrQueryRequest)} + *

+ * This class is used exclusively in a single-node context (i.e. non distributed requests or an individual shard + * request). Also see {@link org.apache.solr.handler.component.FacetComponent.FacetInfo} which is + * dedicated exclusively for merging responses from multiple shards and plays no role during computation of facet + * counts in a single node request. + * + * This API is experimental and subject to change + * + * @see org.apache.solr.handler.component.FacetComponent.FacetInfo + */ + public static class FacetContext { + private static final String FACET_CONTEXT_KEY = "_facet.context"; + + private final List allRangeFacets; // init in constructor + private final List allQueryFacets; // init in constructor + + private final Map> taggedRangeFacets; + private final Map> taggedQueryFacets; + + /** + * Initializes FacetContext using request parameters and saves it in the request + * context which can be retrieved via {@link #getFacetContext(SolrQueryRequest)} + * + * @param rb the ResponseBuilder object from which the request parameters are read + * and to which the FacetContext object is saved. + */ + public static void initContext(ResponseBuilder rb) { + // Parse facet queries and ranges and put them in the request + // context so that they can be hung under pivots if needed without re-parsing + List facetRanges = null; + List facetQueries = null; + + String[] ranges = rb.req.getParams().getParams(FacetParams.FACET_RANGE); + if (ranges != null) { + facetRanges = new ArrayList<>(ranges.length); + for (String range : ranges) { + RangeFacetRequest rangeFacetRequest = new RangeFacetRequest(rb, range); + facetRanges.add(rangeFacetRequest); + } + } + + String[] queries = rb.req.getParams().getParams(FacetParams.FACET_QUERY); + if (queries != null) { + facetQueries = new ArrayList<>(); + for (String query : queries) { + facetQueries.add(new FacetBase(rb, FacetParams.FACET_QUERY, query)); + } + } + + rb.req.getContext().put(FACET_CONTEXT_KEY, new FacetContext(facetRanges, facetQueries)); + } + + private FacetContext(List allRangeFacets, List allQueryFacets) { + // avoid NPEs, set to empty list if parameters are null + this.allRangeFacets = allRangeFacets == null ? Collections.emptyList() : allRangeFacets; + this.allQueryFacets = allQueryFacets == null ? Collections.emptyList() : allQueryFacets; + + taggedRangeFacets = new HashMap<>(); + for (RangeFacetRequest rf : this.allRangeFacets) { + for (String tag : rf.getTags()) { + List list = taggedRangeFacets.get(tag); + if (list == null) { + list = new ArrayList<>(1); // typically just one object + taggedRangeFacets.put(tag, list); + } + list.add(rf); + } + } + + taggedQueryFacets = new HashMap<>(); + for (FacetBase qf : this.allQueryFacets) { + for (String tag : qf.getTags()) { + List list = taggedQueryFacets.get(tag); + if (list == null) { + list = new ArrayList<>(1); + taggedQueryFacets.put(tag, list); + } + list.add(qf); + } + } + } + + /** + * Return the {@link org.apache.solr.handler.component.FacetComponent.FacetContext} instance + * cached in the request context. + * + * @param req the {@link SolrQueryRequest} + * @return the cached FacetContext instance + * @throws IllegalStateException if no cached FacetContext instance is found in the request context + */ + public static FacetContext getFacetContext(SolrQueryRequest req) throws IllegalStateException { + FacetContext result = (FacetContext) req.getContext().get(FACET_CONTEXT_KEY); + if (null == result) { + throw new IllegalStateException("FacetContext can't be accessed before it's initialized in request context"); + } + return result; + } + + /** + * @return a {@link List} of {@link RangeFacetRequest} objects each representing a facet.range to be + * computed. Returns an empty list if no facet.range were requested. + */ + public List getAllRangeFacetRequests() { + return allRangeFacets; + } + + /** + * @return a {@link List} of {@link org.apache.solr.handler.component.FacetComponent.FacetBase} objects + * each representing a facet.query to be computed. Returns an empty list of no facet.query were requested. + */ + public List getAllQueryFacets() { + return allQueryFacets; + } + + /** + * @param tag a String tag usually specified via local param on a facet.pivot + * @return a list of {@link RangeFacetRequest} objects which have been tagged with the given tag. + * Returns an empty list if none found. + */ + public List getRangeFacetRequestsForTag(String tag) { + List list = taggedRangeFacets.get(tag); + return list == null ? Collections.emptyList() : list; + } + + /** + * @param tag a String tag usually specified via local param on a facet.pivot + * @return a list of {@link org.apache.solr.handler.component.FacetComponent.FacetBase} objects which have been + * tagged with the given tag. Returns and empty List if none found. + */ + public List getQueryFacetsForTag(String tag) { + List list = taggedQueryFacets.get(tag); + return list == null ? Collections.emptyList() : list; } } @@ -81,27 +249,13 @@ public class FacetComponent extends SearchComponent { @Override public void process(ResponseBuilder rb) throws IOException { - //SolrParams params = rb.req.getParams(); if (rb.doFacets) { - ModifiableSolrParams params = new ModifiableSolrParams(); - SolrParams origParams = rb.req.getParams(); - Iterator iter = origParams.getParameterNamesIterator(); - while (iter.hasNext()) { - String paramName = iter.next(); - // Deduplicate the list with LinkedHashSet, but _only_ for facet params. - if (paramName.startsWith(FacetParams.FACET) == false) { - params.add(paramName, origParams.getParams(paramName)); - continue; - } - HashSet deDupe = new LinkedHashSet<>(Arrays.asList(origParams.getParams(paramName))); - params.add(paramName, deDupe.toArray(new String[deDupe.size()])); - } - + SolrParams params = rb.req.getParams(); SimpleFacets f = new SimpleFacets(rb.req, rb.getResults().docSet, params, rb); - - NamedList counts = f.getFacetCounts(); + + NamedList counts = FacetComponent.getFacetCounts(f); String[] pivots = params.getParams(FacetParams.FACET_PIVOT); - if (pivots != null && pivots.length > 0) { + if (!ArrayUtils.isEmpty(pivots)) { PivotFacetProcessor pivotProcessor = new PivotFacetProcessor(rb.req, rb.getResults().docSet, params, rb); SimpleOrderedMap>> v @@ -110,11 +264,46 @@ public class FacetComponent extends SearchComponent { counts.add(PIVOT_KEY, v); } } - + rb.rsp.add("facet_counts", counts); } } - + + /** + * Looks at various Params to determining if any simple Facet Constraint count + * computations are desired. + * + * @see SimpleFacets#getFacetQueryCounts + * @see SimpleFacets#getFacetFieldCounts + * @see DateFacetProcessor#getFacetDateCounts + * @see RangeFacetProcessor#getFacetRangeCounts + * @see RangeFacetProcessor#getFacetIntervalCounts + * @see FacetParams#FACET + * @return a NamedList of Facet Count info or null + */ + public static NamedList getFacetCounts(SimpleFacets simpleFacets) { + // if someone called this method, benefit of the doubt: assume true + if (!simpleFacets.getGlobalParams().getBool(FacetParams.FACET, true)) + return null; + + DateFacetProcessor dateFacetProcessor = new DateFacetProcessor(simpleFacets.getRequest(), simpleFacets.getDocsOrig(), simpleFacets.getGlobalParams(), simpleFacets.getResponseBuilder()); + RangeFacetProcessor rangeFacetProcessor = new RangeFacetProcessor(simpleFacets.getRequest(), simpleFacets.getDocsOrig(), simpleFacets.getGlobalParams(), simpleFacets.getResponseBuilder()); + NamedList counts = new SimpleOrderedMap<>(); + try { + counts.add(FACET_QUERY_KEY, simpleFacets.getFacetQueryCounts()); + counts.add(FACET_FIELD_KEY, simpleFacets.getFacetFieldCounts()); + counts.add(FACET_DATE_KEY, dateFacetProcessor.getFacetDateCounts()); + counts.add(FACET_RANGES_KEY, rangeFacetProcessor.getFacetRangeCounts()); + counts.add(FACET_INTERVALS_KEY, simpleFacets.getFacetIntervalCounts()); + counts.add(SpatialHeatmapFacets.RESPONSE_KEY, simpleFacets.getHeatmapCounts()); + } catch (IOException e) { + throw new SolrException(ErrorCode.SERVER_ERROR, e); + } catch (SyntaxError e) { + throw new SolrException(ErrorCode.BAD_REQUEST, e); + } + return counts; + } + private static final String commandPrefix = "{!" + CommonParams.TERMS + "=$"; @Override @@ -514,7 +703,7 @@ public class FacetComponent extends SearchComponent { qf.count += count; } } - + // step through each facet.field, adding results from this shard NamedList facet_fields = (NamedList) facet_counts.get("facet_fields"); @@ -528,7 +717,12 @@ public class FacetComponent extends SearchComponent { doDistribDates(fi, facet_counts); // Distributed facet_ranges - doDistribRanges(fi, facet_counts); + @SuppressWarnings("unchecked") + SimpleOrderedMap> rangesFromShard = (SimpleOrderedMap>) + facet_counts.get("facet_ranges"); + if (rangesFromShard != null) { + RangeFacetRequest.DistribRangeFacet.mergeFacetRangesFromShardResponse(fi.rangeFacets, rangesFromShard); + } // Distributed facet_intervals doDistribIntervals(fi, facet_counts); @@ -655,43 +849,19 @@ public class FacetComponent extends SearchComponent { } FacetInfo fi = rb._facetInfo; - - @SuppressWarnings("unchecked") - SimpleOrderedMap> facet_ranges = - (SimpleOrderedMap>) - fi.rangeFacets; - - if (facet_ranges == null) { - return; - } - - // go through each facet_range - for (Map.Entry> entry : facet_ranges) { - boolean replace = false; + for (Map.Entry entry : fi.rangeFacets.entrySet()) { final String field = entry.getKey(); + final RangeFacetRequest.DistribRangeFacet rangeFacet = entry.getValue(); + int minCount = rb.req.getParams().getFieldInt(field, FacetParams.FACET_MINCOUNT, 0); if (minCount == 0) { continue; } - @SuppressWarnings("unchecked") - NamedList vals - = (NamedList) facet_ranges.get(field).get("counts"); - NamedList newList = new NamedList(); - for (Map.Entry pair : vals) { - if (pair.getValue() >= minCount) { - newList.add(pair.getKey(), pair.getValue()); - } else { - log.trace("Removing facet/key: " + pair.getKey() + "/" + pair.getValue().toString() + " mincount=" + minCount); - replace = true; - } - } - if (replace) { - vals.clear(); - vals.addAll(newList); - } + rangeFacet.removeRangeFacetsUnderLimits(minCount); } } + private void removeFieldFacetsUnderLimits(ResponseBuilder rb) { if (rb.stage != ResponseBuilder.STAGE_DONE) { return; @@ -759,62 +929,6 @@ public class FacetComponent extends SearchComponent { } } - private final static String[] OTHER_KEYS = new String[]{FacetRangeOther.BEFORE.toString(), FacetRangeOther.BETWEEN.toString(), FacetRangeOther.AFTER.toString()}; - // The implementation below uses the first encountered shard's - // facet_ranges as the basis for subsequent shards' data to be merged. - private void doDistribRanges(FacetInfo fi, NamedList facet_counts) { - @SuppressWarnings("unchecked") - SimpleOrderedMap> facet_ranges = - (SimpleOrderedMap>) - facet_counts.get("facet_ranges"); - - if (facet_ranges != null) { - - // go through each facet_range - for (Map.Entry> entry : facet_ranges) { - final String field = entry.getKey(); - SimpleOrderedMap fieldMap = fi.rangeFacets.get(field); - if (fieldMap == null) { - // first time we've seen this field, no merging - fi.rangeFacets.add(field, entry.getValue()); - - } else { - // not the first time, merge current field counts - - @SuppressWarnings("unchecked") - NamedList shardFieldValues - = (NamedList) entry.getValue().get("counts"); - - @SuppressWarnings("unchecked") - NamedList existFieldValues - = (NamedList) fieldMap.get("counts"); - - for (Map.Entry existPair : existFieldValues) { - final String key = existPair.getKey(); - // can be null if inconsistencies in shards responses - Integer newValue = shardFieldValues.get(key); - if (null != newValue) { - Integer oldValue = existPair.getValue(); - existPair.setValue(oldValue + newValue); - } - } - - // merge before/between/after if they exist - for (String otherKey:OTHER_KEYS) { - Integer shardValue = (Integer)entry.getValue().get(otherKey); - if (shardValue != null && shardValue > 0) { - Integer existingValue = (Integer)fieldMap.get(otherKey); - // shouldn't be null - int idx = fieldMap.indexOf(otherKey, 0); - fieldMap.setVal(idx, existingValue + shardValue); - } - } - - } - } - } - } - // // The implementation below uses the first encountered shard's // facet_dates as the basis for subsequent shards' data to be merged. @@ -1046,7 +1160,15 @@ public class FacetComponent extends SearchComponent { } facet_counts.add("facet_dates", fi.dateFacets); - facet_counts.add("facet_ranges", fi.rangeFacets); + + SimpleOrderedMap> rangeFacetOutput = new SimpleOrderedMap<>(); + for (Map.Entry entry : fi.rangeFacets.entrySet()) { + String key = entry.getKey(); + RangeFacetRequest.DistribRangeFacet value = entry.getValue(); + rangeFacetOutput.add(key, value.rangeFacet); + } + facet_counts.add("facet_ranges", rangeFacetOutput); + facet_counts.add("facet_intervals", fi.intervalFacets); facet_counts.add(SpatialHeatmapFacets.RESPONSE_KEY, SpatialHeatmapFacets.distribFinish(fi.heatmapFacets, rb)); @@ -1077,11 +1199,23 @@ public class FacetComponent extends SearchComponent { } // use tags for smaller facet counts (better back compatibility) - private Number num(long val) { + + /** + * @param val a primitive long value + * @return an {@link Integer} if the value of the argument is less than {@link Integer#MAX_VALUE} + * else a @{link java.lang.Long} + */ + static Number num(long val) { if (val < Integer.MAX_VALUE) return (int)val; else return val; } - private Number num(Long val) { + + /** + * @param val a {@link java.lang.Long} value + * @return an {@link Integer} if the value of the argument is less than {@link Integer#MAX_VALUE} + * else a @{link java.lang.Long} + */ + static Number num(Long val) { if (val.longValue() < Integer.MAX_VALUE) return val.intValue(); else return val; } @@ -1102,7 +1236,16 @@ public class FacetComponent extends SearchComponent { } /** + * This class is used exclusively for merging results from each shard + * in a distributed facet request. It plays no role in the computation + * of facet counts inside a single node. + * + * A related class {@link org.apache.solr.handler.component.FacetComponent.FacetContext} + * exists for assisting computation inside a single node. + * * This API is experimental and subject to change + * + * @see org.apache.solr.handler.component.FacetComponent.FacetContext */ public static class FacetInfo { /** @@ -1116,8 +1259,8 @@ public class FacetComponent extends SearchComponent { public LinkedHashMap facets; public SimpleOrderedMap> dateFacets = new SimpleOrderedMap<>(); - public SimpleOrderedMap> rangeFacets - = new SimpleOrderedMap<>(); + public LinkedHashMap rangeFacets + = new LinkedHashMap<>(); public SimpleOrderedMap> intervalFacets = new SimpleOrderedMap<>(); public SimpleOrderedMap pivotFacets @@ -1157,7 +1300,7 @@ public class FacetComponent extends SearchComponent { heatmapFacets = SpatialHeatmapFacets.distribParse(params, rb); } } - + /** * This API is experimental and subject to change */ @@ -1168,6 +1311,9 @@ public class FacetComponent extends SearchComponent { private String key; // label in the response for the result... // "foo" for {!key=foo}myfield SolrParams localParams; // any local params for the facet + private List tags = Collections.emptyList(); + private List excludeTags = Collections.emptyList(); + private int threadCount = -1; public FacetBase(ResponseBuilder rb, String facetType, String facetStr) { this.facetType = facetType; @@ -1189,12 +1335,28 @@ public class FacetComponent extends SearchComponent { } key = localParams.get(CommonParams.OUTPUT_KEY, key); + + String tagStr = localParams.get(CommonParams.TAG); + this.tags = tagStr == null ? Collections.emptyList() : StrUtils.splitSmart(tagStr,','); + + String threadStr = localParams.get(CommonParams.THREADS); + this.threadCount = threadStr != null ? Integer.parseInt(threadStr) : -1; + + String excludeStr = localParams.get(CommonParams.EXCLUDE); + if (StringUtils.isEmpty(excludeStr)) { + this.excludeTags = Collections.emptyList(); + } else { + this.excludeTags = StrUtils.splitSmart(excludeStr,','); + } } } /** returns the key in the response that this facet will be under */ public String getKey() { return key; } public String getType() { return facetType; } + public List getTags() { return tags; } + public List getExcludeTags() { return excludeTags; } + public int getThreadCount() { return threadCount; } } /** @@ -1407,4 +1569,5 @@ public class FacetComponent extends SearchComponent { return "{term=" + name + ",termNum=" + termNum + ",count=" + count + "}"; } } + } diff --git a/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java b/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java index c1730072c0f..9d484391657 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java +++ b/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java @@ -17,22 +17,18 @@ package org.apache.solr.handler.component; -import org.apache.solr.util.PivotListEntry; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.params.FacetParams; -import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Collections; -import java.util.Map; -import java.util.Map.Entry; +import org.apache.solr.util.PivotListEntry; public class PivotFacetHelper { @@ -52,8 +48,9 @@ public class PivotFacetHelper { assert null != values; // special case: empty list => empty string - if (values.isEmpty()) { return ""; } - + if (values.isEmpty()) { + return ""; + } StringBuilder out = new StringBuilder(); for (String val : values) { @@ -76,7 +73,7 @@ public class PivotFacetHelper { * @see #encodeRefinementValuePath */ public static List decodeRefinementValuePath(String valuePath) { - List rawvals = StrUtils.splitSmart(valuePath, ",", true); + List rawvals = StrUtils.splitSmart(valuePath, ",", true); // special case: empty list => empty string if (rawvals.isEmpty()) return rawvals; @@ -120,6 +117,16 @@ public class PivotFacetHelper { return (NamedList>>) PivotListEntry.STATS.extract(pivotList); } + /** @see PivotListEntry#QUERIES */ + public static NamedList getQueryCounts(NamedList pivotList) { + return (NamedList) PivotListEntry.QUERIES.extract(pivotList); + } + + /** @see PivotListEntry#RANGES */ + public static SimpleOrderedMap> getRanges(NamedList pivotList) { + return (SimpleOrderedMap>) PivotListEntry.RANGES.extract(pivotList); + } + /** * Given a mapping of keys to {@link StatsValues} representing the currently * known "merged" stats (which may be null if none exist yet), and a @@ -156,4 +163,28 @@ public class PivotFacetHelper { return merged; } + /** + * Merges query counts returned by a shard into global query counts. + * Entries found only in shard's query counts will be added to global counts. + * Entries found in both shard and global query counts will be summed. + * + * @param globalQueryCounts The global query counts (across all shards) in which to merge the shard query counts + * @param shardQueryCounts Named list from a shard response to be merged into the global counts. + * @return NamedList containing merged values + */ + static NamedList mergeQueryCounts( + NamedList globalQueryCounts, NamedList shardQueryCounts) { + if (globalQueryCounts == null) { + return shardQueryCounts; + } + for (Entry entry : shardQueryCounts) { + int idx = globalQueryCounts.indexOf(entry.getKey(), 0); + if (idx == -1) { + globalQueryCounts.add(entry.getKey(), entry.getValue()); + } else { + globalQueryCounts.setVal(idx, FacetComponent.num(globalQueryCounts.getVal(idx).longValue() + entry.getValue().longValue())); + } + } + return globalQueryCounts; + } } diff --git a/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java b/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java index 0064a22312c..d76d40306f7 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java +++ b/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java @@ -18,6 +18,8 @@ package org.apache.solr.handler.component; */ import org.apache.lucene.util.BytesRefBuilder; +import org.apache.solr.common.StringUtils; +import org.apache.solr.common.params.RequiredSolrParams; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.FieldType; import org.apache.solr.search.SolrIndexSearcher; @@ -41,7 +43,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Deque; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -52,6 +53,8 @@ import java.util.Map; */ public class PivotFacetProcessor extends SimpleFacets { + public static final String QUERY = "query"; + public static final String RANGE = "range"; protected SolrParams params; public PivotFacetProcessor(SolrQueryRequest req, DocSet docs, SolrParams params, ResponseBuilder rb) { @@ -101,7 +104,8 @@ public class PivotFacetProcessor extends SimpleFacets String refineKey = null; // no local => no refinement List statsFields = Collections.emptyList(); // no local => no stats - + List facetQueries = Collections.emptyList(); + List facetRanges = Collections.emptyList(); if (null != parsed.localParams) { // we might be refining.. refineKey = parsed.localParams.get(PivotFacet.REFINE_PARAM); @@ -117,6 +121,42 @@ public class PivotFacetProcessor extends SimpleFacets statsInfo = new StatsInfo(rb); } statsFields = getTaggedStatsFields(statsInfo, statsLocalParam); + + try { + FacetComponent.FacetContext facetContext = FacetComponent.FacetContext.getFacetContext(req); + + String taggedQueries = parsed.localParams.get(QUERY); + if (StringUtils.isEmpty(taggedQueries)) { + facetQueries = Collections.emptyList(); + } else { + List localParamValue = StrUtils.splitSmart(taggedQueries, ','); + if (localParamValue.size() > 1) { + String msg = QUERY + " local param of " + FacetParams.FACET_PIVOT + + "may not include tags separated by a comma - please use a common tag on all " + + FacetParams.FACET_QUERY + " params you wish to compute under this pivot"; + throw new SolrException(ErrorCode.BAD_REQUEST, msg); + } + taggedQueries = localParamValue.get(0); + facetQueries = facetContext.getQueryFacetsForTag(taggedQueries); + } + + String taggedRanges = parsed.localParams.get(RANGE); + if (StringUtils.isEmpty(taggedRanges)) { + facetRanges = Collections.emptyList(); + } else { + List localParamValue = StrUtils.splitSmart(taggedRanges, ','); + if (localParamValue.size() > 1) { + String msg = RANGE + " local param of " + FacetParams.FACET_PIVOT + + "may not include tags separated by a comma - please use a common tag on all " + + FacetParams.FACET_RANGE + " params you wish to compute under this pivot"; + throw new SolrException(ErrorCode.BAD_REQUEST, msg); + } + taggedRanges = localParamValue.get(0); + facetRanges = facetContext.getRangeFacetRequestsForTag(taggedRanges); + } + } catch (IllegalStateException e) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Faceting context not set, cannot calculate pivot values"); + } } if (null != refineKey) { @@ -124,10 +164,10 @@ public class PivotFacetProcessor extends SimpleFacets = params.getParams(PivotFacet.REFINE_PARAM + refineKey); for(String refinements : refinementValuesByField){ - pivotResponse.addAll(processSingle(pivotFields, refinements, statsFields, parsed)); + pivotResponse.addAll(processSingle(pivotFields, refinements, statsFields, parsed, facetQueries, facetRanges)); } } else{ - pivotResponse.addAll(processSingle(pivotFields, null, statsFields, parsed)); + pivotResponse.addAll(processSingle(pivotFields, null, statsFields, parsed, facetQueries, facetRanges)); } } return pivotResponse; @@ -138,12 +178,16 @@ public class PivotFacetProcessor extends SimpleFacets * @param pivotFields the ordered list of fields in this pivot * @param refinements the comma separate list of refinement values corresponding to each field in the pivot, or null if there are no refinements * @param statsFields List of {@link StatsField} instances to compute for each pivot value + * @param facetQueries the list of facet queries hung under this pivot + * @param facetRanges the list of facet ranges hung under this pivot */ private SimpleOrderedMap>> processSingle - (List pivotFields, - String refinements, - List statsFields, - final ParsedParams parsed) throws IOException { + (List pivotFields, + String refinements, + List statsFields, + final ParsedParams parsed, + List facetQueries, + List facetRanges) throws IOException { SolrIndexSearcher searcher = rb.req.getSearcher(); SimpleOrderedMap>> pivotResponse = new SimpleOrderedMap<>(); @@ -181,9 +225,9 @@ public class PivotFacetProcessor extends SimpleFacets if(pivotFields.size() > 1) { String subField = pivotFields.get(1); pivotResponse.add(parsed.key, - doPivots(facetCounts, field, subField, fnames, vnames, parsed, statsFields)); + doPivots(facetCounts, field, subField, fnames, vnames, parsed, statsFields, facetQueries, facetRanges)); } else { - pivotResponse.add(parsed.key, doPivots(facetCounts, field, null, fnames, vnames, parsed, statsFields)); + pivotResponse.add(parsed.key, doPivots(facetCounts, field, null, fnames, vnames, parsed, statsFields, facetQueries, facetRanges)); } return pivotResponse; } @@ -223,10 +267,11 @@ public class PivotFacetProcessor extends SimpleFacets * Recursive function to compute all the pivot counts for the values under the specified field */ protected List> doPivots(NamedList superFacets, - String field, String subField, - Deque fnames, Deque vnames, - ParsedParams parsed, List statsFields) - throws IOException { + String field, String subField, + Deque fnames, Deque vnames, + ParsedParams parsed, List statsFields, + List facetQueries, List facetRanges) + throws IOException { boolean isShard = rb.req.getParams().getBool(ShardParams.IS_SHARD, false); @@ -259,6 +304,8 @@ public class PivotFacetProcessor extends SimpleFacets final DocSet subset = getSubset(parsed.docs, sfield, fieldValue); + addPivotQueriesAndRanges(pivot, params, subset, facetQueries, facetRanges); + if( subField != null ) { NamedList facetCounts; if(!vnames.isEmpty()){ @@ -272,7 +319,7 @@ public class PivotFacetProcessor extends SimpleFacets } if (facetCounts.size() >= 1) { - pivot.add( "pivot", doPivots( facetCounts, subField, nextField, fnames, vnames, parsed.withDocs(subset), statsFields ) ); + pivot.add( "pivot", doPivots( facetCounts, subField, nextField, fnames, vnames, parsed.withDocs(subset), statsFields, facetQueries, facetRanges) ); } } if ((isShard || 0 < pivotCount) && ! statsFields.isEmpty()) { @@ -329,6 +376,65 @@ public class PivotFacetProcessor extends SimpleFacets } } + /** + * Add facet.queries and facet.ranges to the pivot response if needed + * + * @param pivot + * Pivot in which to inject additional data + * @param params + * Query parameters. + * @param docs + * DocSet of the current pivot to use for computing sub-counts + * @param facetQueries + * Tagged facet queries should have to be included, must not be null + * @param facetRanges + * Taged facet ranges should have to be included, must not be null + * @throws IOException + * If searcher has issues finding numDocs. + */ + protected void addPivotQueriesAndRanges(NamedList pivot, SolrParams params, DocSet docs, + List facetQueries, + List facetRanges) throws IOException { + assert null != facetQueries; + assert null != facetRanges; + + if ( ! facetQueries.isEmpty()) { + SimpleFacets facets = new SimpleFacets(req, docs, params); + NamedList res = new SimpleOrderedMap<>(); + for (FacetComponent.FacetBase facetQuery : facetQueries) { + try { + ParsedParams parsed = getParsedParams(params, docs, facetQuery); + facets.getFacetQueryCount(parsed, res); + } catch (SyntaxError e) { + throw new SolrException(ErrorCode.BAD_REQUEST, + "Invalid " + FacetParams.FACET_QUERY + " (" + facetQuery.facetStr + + ") cause: " + e.getMessage(), e); + } + } + pivot.add(PivotListEntry.QUERIES.getName(), res); + } + if ( ! facetRanges.isEmpty()) { + RangeFacetProcessor rangeFacetProcessor = new RangeFacetProcessor(req, docs, params, null); + NamedList resOuter = new SimpleOrderedMap<>(); + for (RangeFacetRequest rangeFacet : facetRanges) { + try { + rangeFacetProcessor.getFacetRangeCounts(rangeFacet, resOuter); + } catch (SyntaxError e) { + throw new SolrException(ErrorCode.BAD_REQUEST, + "Invalid " + FacetParams.FACET_RANGE + " (" + rangeFacet.facetStr + + ") cause: " + e.getMessage(), e); + } + } + pivot.add(PivotListEntry.RANGES.getName(), resOuter); + } + } + + private ParsedParams getParsedParams(SolrParams params, DocSet docs, FacetComponent.FacetBase facet) { + SolrParams wrapped = SolrParams.wrapDefaults(facet.localParams, global); + SolrParams required = new RequiredSolrParams(params); + return new ParsedParams(facet.localParams, wrapped, required, facet.facetOn, docs, facet.getKey(), facet.getTags(), -1); + } + private int getMinCountForField(String fieldname){ return params.getFieldInt(fieldname, FacetParams.FACET_PIVOT_MINCOUNT, 1); } diff --git a/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java b/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java index 55111c04874..26e9800cbb3 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java +++ b/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java @@ -19,6 +19,7 @@ package org.apache.solr.handler.component; import java.util.BitSet; import java.util.Date; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -27,7 +28,6 @@ import org.apache.solr.common.params.FacetParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.schema.TrieDateField; -import org.apache.solr.search.QueryParsing; import org.apache.solr.util.PivotListEntry; /** @@ -48,7 +48,10 @@ public class PivotFacetValue { private PivotFacetField childPivot = null; private int count; // mutable private Map statsValues = null; - + // named list with objects because depending on how big the counts are we may get either a long or an int + private NamedList queryCounts; + private LinkedHashMap rangeCounts; + private PivotFacetValue(PivotFacetField parent, Comparable val) { this.parentPivot = parent; this.value = val; @@ -118,6 +121,8 @@ public class PivotFacetValue { int pivotCount = 0; List> childPivotData = null; NamedList>> statsValues = null; + NamedList queryCounts = null; + SimpleOrderedMap> ranges = null; for (int i = 0; i < pivotData.size(); i++) { String key = pivotData.getName(i); @@ -142,6 +147,12 @@ public class PivotFacetValue { case STATS: statsValues = (NamedList>>) value; break; + case QUERIES: + queryCounts = (NamedList) value; + break; + case RANGES: + ranges = (SimpleOrderedMap>) value; + break; default: throw new RuntimeException("PivotListEntry contains unaccounted for item: " + entry); } @@ -153,6 +164,13 @@ public class PivotFacetValue { if(statsValues != null) { newPivotFacet.statsValues = PivotFacetHelper.mergeStats(null, statsValues, rb._statsInfo); } + if(queryCounts != null) { + newPivotFacet.queryCounts = PivotFacetHelper.mergeQueryCounts(null, queryCounts); + } + if(ranges != null) { + newPivotFacet.rangeCounts = new LinkedHashMap<>(); + RangeFacetRequest.DistribRangeFacet.mergeFacetRangesFromShardResponse(newPivotFacet.rangeCounts, ranges); + } newPivotFacet.childPivot = PivotFacetField.createFromListOfNamedLists(shardNumber, rb, newPivotFacet, childPivotData); @@ -178,6 +196,18 @@ public class PivotFacetValue { newList.add(PivotListEntry.FIELD.getName(), parentPivot.field); newList.add(PivotListEntry.VALUE.getName(), value); newList.add(PivotListEntry.COUNT.getName(), count); + if(queryCounts != null) { + newList.add(PivotListEntry.QUERIES.getName(), queryCounts); + } + if(rangeCounts != null) { + SimpleOrderedMap> rangeFacetOutput = new SimpleOrderedMap<>(); + for (Map.Entry entry : rangeCounts.entrySet()) { + String key = entry.getKey(); + RangeFacetRequest.DistribRangeFacet value = entry.getValue(); + rangeFacetOutput.add(key, value.rangeFacet); + } + newList.add(PivotListEntry.RANGES.getName(), rangeFacetOutput); + } if (childPivot != null && childPivot.convertToListOfNamedLists() != null) { newList.add(PivotListEntry.PIVOT.getName(), childPivot.convertToListOfNamedLists()); } @@ -205,6 +235,17 @@ public class PivotFacetValue { if (stats != null) { statsValues = PivotFacetHelper.mergeStats(statsValues, stats, rb._statsInfo); } + NamedList shardQueryCounts = PivotFacetHelper.getQueryCounts(value); + if(shardQueryCounts != null) { + queryCounts = PivotFacetHelper.mergeQueryCounts(queryCounts, shardQueryCounts); + } + SimpleOrderedMap> shardRanges = PivotFacetHelper.getRanges(value); + if (shardRanges != null) { + if (rangeCounts == null) { + rangeCounts = new LinkedHashMap<>(shardRanges.size() / 2); + } + RangeFacetRequest.DistribRangeFacet.mergeFacetRangesFromShardResponse(rangeCounts, shardRanges); + } } List> shardChildPivots = PivotFacetHelper.getPivots(value); diff --git a/solr/core/src/java/org/apache/solr/handler/component/RangeFacetProcessor.java b/solr/core/src/java/org/apache/solr/handler/component/RangeFacetProcessor.java new file mode 100644 index 00000000000..1564d9bbbda --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/component/RangeFacetProcessor.java @@ -0,0 +1,277 @@ +package org.apache.solr.handler.component; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.apache.log4j.Logger; +import org.apache.lucene.search.Query; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.FacetParams.FacetRangeMethod; +import org.apache.solr.common.params.FacetParams.FacetRangeOther; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.request.IntervalFacets; +import org.apache.solr.request.SimpleFacets; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.TrieField; +import org.apache.solr.search.DocSet; +import org.apache.solr.search.SyntaxError; + +/** + * Processor for Range Facets + */ +public class RangeFacetProcessor extends SimpleFacets { + private final static Logger log = Logger.getLogger(RangeFacetProcessor.class); + + public RangeFacetProcessor(SolrQueryRequest req, DocSet docs, SolrParams params, ResponseBuilder rb) { + super(req, docs, params, rb); + } + + /** + * Returns a list of value constraints and the associated facet + * counts for each facet numerical field, range, and interval + * specified in the SolrParams + * + * @see org.apache.solr.common.params.FacetParams#FACET_RANGE + */ + @SuppressWarnings("unchecked") + public NamedList getFacetRangeCounts() throws IOException, SyntaxError { + final NamedList resOuter = new SimpleOrderedMap<>(); + + List rangeFacetRequests = Collections.emptyList(); + try { + FacetComponent.FacetContext facetContext = FacetComponent.FacetContext.getFacetContext(req); + rangeFacetRequests = facetContext.getAllRangeFacetRequests(); + } catch (IllegalStateException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to compute facet ranges, facet context is not set"); + } + + if (rangeFacetRequests.isEmpty()) return resOuter; + for (RangeFacetRequest rangeFacetRequest : rangeFacetRequests) { + getFacetRangeCounts(rangeFacetRequest, resOuter); + } + + return resOuter; + } + + /** + * Returns a list of value constraints and the associated facet counts + * for each facet range specified by the given {@link RangeFacetRequest} + */ + public void getFacetRangeCounts(RangeFacetRequest rangeFacetRequest, NamedList resOuter) + throws IOException, SyntaxError { + + final IndexSchema schema = searcher.getSchema(); + + final String key = rangeFacetRequest.getKey(); + final String f = rangeFacetRequest.facetOn; + FacetRangeMethod method = rangeFacetRequest.getMethod(); + + final SchemaField sf = schema.getField(f); + final FieldType ft = sf.getType(); + + if (method.equals(FacetRangeMethod.DV)) { + assert ft instanceof TrieField; + resOuter.add(key, getFacetRangeCountsDocValues(rangeFacetRequest)); + } else { + resOuter.add(key, getFacetRangeCounts(rangeFacetRequest)); + } + } + + private > NamedList getFacetRangeCounts(final RangeFacetRequest rfr) + throws IOException, SyntaxError { + + final NamedList res = new SimpleOrderedMap<>(); + final NamedList counts = new NamedList<>(); + res.add("counts", counts); + + // explicitly return the gap. + res.add("gap", rfr.getGapObj()); + + DocSet docSet = computeDocSet(docsOrig, rfr.getExcludeTags()); + + for (RangeFacetRequest.FacetRange range : rfr.getFacetRanges()) { + if (range.other != null) { + // these are added to top-level NamedList + // and we always include them regardless of mincount + res.add(range.other.toString(), rangeCount(docSet, rfr, range)); + } else { + final int count = rangeCount(docSet, rfr, range); + if (count >= rfr.getMinCount()) { + counts.add(range.lower, count); + } + } + } + + // explicitly return the start and end so all the counts + // (including before/after/between) are meaningful - even if mincount + // has removed the neighboring ranges + res.add("start", rfr.getStartObj()); + res.add("end", rfr.getEndObj()); + + return res; + } + + private > NamedList getFacetRangeCountsDocValues(RangeFacetRequest rfr) + throws IOException, SyntaxError { + + SchemaField sf = rfr.getSchemaField(); + final NamedList res = new SimpleOrderedMap<>(); + final NamedList counts = new NamedList<>(); + res.add("counts", counts); + + ArrayList intervals = new ArrayList<>(); + + // explicitly return the gap. compute this early so we are more + // likely to catch parse errors before attempting math + res.add("gap", rfr.getGapObj()); + + final int minCount = rfr.getMinCount(); + + boolean includeBefore = false; + boolean includeBetween = false; + boolean includeAfter = false; + + Set others = rfr.getOthers(); + // Intervals must be in order (see IntervalFacets.getSortedIntervals), if "BEFORE" or + // "BETWEEN" are set, they must be added first + // no matter what other values are listed, we don't do + // anything if "none" is specified. + if (!others.contains(FacetRangeOther.NONE)) { + if (others.contains(FacetRangeOther.ALL) || others.contains(FacetRangeOther.BEFORE)) { + // We'll add an interval later in this position + intervals.add(null); + includeBefore = true; + } + + if (others.contains(FacetRangeOther.ALL) || others.contains(FacetRangeOther.BETWEEN)) { + // We'll add an interval later in this position + intervals.add(null); + includeBetween = true; + } + + if (others.contains(FacetRangeOther.ALL) || others.contains(FacetRangeOther.AFTER)) { + includeAfter = true; + } + } + + IntervalFacets.FacetInterval after = null; + + for (RangeFacetRequest.FacetRange range : rfr.getFacetRanges()) { + try { + FacetRangeOther other = FacetRangeOther.get(range.name); + if (other != null) { + switch (other) { + case BEFORE: + assert range.lower == null; + intervals.set(0, new IntervalFacets.FacetInterval(sf, "*", range.upper, range.includeLower, + range.includeUpper, FacetRangeOther.BEFORE.toString())); + break; + case AFTER: + assert range.upper == null; + after = new IntervalFacets.FacetInterval(sf, range.lower, "*", + range.includeLower, range.includeUpper, FacetRangeOther.AFTER.toString()); + break; + case BETWEEN: + intervals.set(includeBefore ? 1 : 0, new IntervalFacets.FacetInterval(sf, range.lower, range.upper, + range.includeLower, range.includeUpper, FacetRangeOther.BETWEEN.toString())); + break; + } + } + continue; + } catch (SolrException e) { + // safe to ignore + } + + intervals.add(new IntervalFacets.FacetInterval(sf, range.lower, range.upper, range.includeLower, range.includeUpper, range.lower)); + } + + if (includeAfter) { + assert after != null; + intervals.add(after); + } + + IntervalFacets.FacetInterval[] intervalsArray = intervals.toArray(new IntervalFacets.FacetInterval[intervals.size()]); + // don't use the ArrayList anymore + intervals = null; + + new IntervalFacets(sf, searcher, computeDocSet(docsOrig, rfr.getExcludeTags()), intervalsArray); + + int intervalIndex = 0; + int lastIntervalIndex = intervalsArray.length - 1; + // if the user requested "BEFORE", it will be the first of the intervals. Needs to be added to the + // response named list instead of with the counts + if (includeBefore) { + res.add(intervalsArray[intervalIndex].getKey(), intervalsArray[intervalIndex].getCount()); + intervalIndex++; + } + + // if the user requested "BETWEEN", it will be the first or second of the intervals (depending on if + // "BEFORE" was also requested). Needs to be added to the response named list instead of with the counts + if (includeBetween) { + res.add(intervalsArray[intervalIndex].getKey(), intervalsArray[intervalIndex].getCount()); + intervalIndex++; + } + + // if the user requested "AFTER", it will be the last of the intervals. + // Needs to be added to the response named list instead of with the counts + if (includeAfter) { + res.add(intervalsArray[lastIntervalIndex].getKey(), intervalsArray[lastIntervalIndex].getCount()); + lastIntervalIndex--; + } + // now add all other intervals to the counts NL + while (intervalIndex <= lastIntervalIndex) { + IntervalFacets.FacetInterval interval = intervalsArray[intervalIndex]; + if (interval.getCount() >= minCount) { + counts.add(interval.getKey(), interval.getCount()); + } + intervalIndex++; + } + + res.add("start", rfr.getStartObj()); + res.add("end", rfr.getEndObj()); + return res; + } + + /** + * Macro for getting the numDocs of range over docs + * + * @see org.apache.solr.search.SolrIndexSearcher#numDocs + * @see org.apache.lucene.search.TermRangeQuery + */ + protected int rangeCount(DocSet subset, RangeFacetRequest rfr, RangeFacetRequest.FacetRange fr) throws IOException, SyntaxError { + SchemaField schemaField = rfr.getSchemaField(); + Query rangeQ = schemaField.getType().getRangeQuery(null, schemaField, fr.lower, fr.upper, fr.includeLower, fr.includeUpper); + if (rfr.isGroupFacet()) { + return getGroupedFacetQueryCount(rangeQ, subset); + } else { + return searcher.numDocs(rangeQ, subset); + } + } + +} + diff --git a/solr/core/src/java/org/apache/solr/handler/component/RangeFacetRequest.java b/solr/core/src/java/org/apache/solr/handler/component/RangeFacetRequest.java new file mode 100644 index 00000000000..ada0864124a --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/component/RangeFacetRequest.java @@ -0,0 +1,802 @@ +package org.apache.solr.handler.component; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.Date; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.FacetParams; +import org.apache.solr.common.params.GroupParams; +import org.apache.solr.common.params.RequiredSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.schema.DateRangeField; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.TrieDateField; +import org.apache.solr.schema.TrieField; +import org.apache.solr.util.DateMathParser; + +/** + * Encapsulates a single facet.range request along with all its parameters. This class + * calculates all the ranges (gaps) required to be counted. + */ +public class RangeFacetRequest extends FacetComponent.FacetBase { + private final static Logger log = Logger.getLogger(RangeFacetRequest.class); + + protected final SchemaField schemaField; + protected final String start; + protected final String end; + protected final String gap; + protected final boolean hardEnd; + protected final EnumSet include; + protected final EnumSet others; + protected final FacetParams.FacetRangeMethod method; + protected final int minCount; + protected final boolean groupFacet; + protected final List facetRanges; + + /** + * The computed start value of this range + */ + protected final Object startObj; + /** + * The computed end value of this range taking into account facet.range.hardend + */ + protected final Object endObj; + /** + * The computed gap between each range + */ + protected final Object gapObj; + + public RangeFacetRequest(ResponseBuilder rb, String f) { + super(rb, FacetParams.FACET_RANGE, f); + + IndexSchema schema = rb.req.getSchema(); + this.schemaField = schema.getField(facetOn); + + SolrParams params = SolrParams.wrapDefaults(localParams, rb.req.getParams()); + SolrParams required = new RequiredSolrParams(params); + + String methodStr = params.get(FacetParams.FACET_RANGE_METHOD); + FacetParams.FacetRangeMethod method = (methodStr == null ? FacetParams.FacetRangeMethod.getDefault() : FacetParams.FacetRangeMethod.get(methodStr)); + + if ((schemaField.getType() instanceof DateRangeField) && method.equals(FacetParams.FacetRangeMethod.DV)) { + // the user has explicitly selected the FacetRangeMethod.DV method + log.warn("Range facet method '" + FacetParams.FacetRangeMethod.DV + "' is not supported together with field type '" + + DateRangeField.class + "'. Will use method '" + FacetParams.FacetRangeMethod.FILTER + "' instead"); + method = FacetParams.FacetRangeMethod.FILTER; + } + + this.start = required.getFieldParam(facetOn, FacetParams.FACET_RANGE_START); + this.end = required.getFieldParam(facetOn, FacetParams.FACET_RANGE_END); + + + this.gap = required.getFieldParam(facetOn, FacetParams.FACET_RANGE_GAP); + this.minCount = params.getFieldInt(facetOn, FacetParams.FACET_MINCOUNT, 0); + + this.include = FacetParams.FacetRangeInclude.parseParam + (params.getFieldParams(facetOn, FacetParams.FACET_RANGE_INCLUDE)); + + this.hardEnd = params.getFieldBool(facetOn, FacetParams.FACET_RANGE_HARD_END, false); + + this.others = EnumSet.noneOf(FacetParams.FacetRangeOther.class); + final String[] othersP = params.getFieldParams(facetOn, FacetParams.FACET_RANGE_OTHER); + if (othersP != null && othersP.length > 0) { + for (final String o : othersP) { + others.add(FacetParams.FacetRangeOther.get(o)); + } + } + + this.groupFacet = params.getBool(GroupParams.GROUP_FACET, false); + if (groupFacet && method.equals(FacetParams.FacetRangeMethod.DV)) { + // the user has explicitly selected the FacetRangeMethod.DV method + log.warn("Range facet method '" + FacetParams.FacetRangeMethod.DV + "' is not supported together with '" + + GroupParams.GROUP_FACET + "'. Will use method '" + FacetParams.FacetRangeMethod.FILTER + "' instead"); + method = FacetParams.FacetRangeMethod.FILTER; + } + + this.method = method; + + RangeEndpointCalculator> calculator = createCalculator(); + this.facetRanges = calculator.computeRanges(); + this.gapObj = calculator.getGap(); + this.startObj = calculator.getStart(); + this.endObj = calculator.getComputedEnd(); + } + + /** + * Creates the right instance of {@link org.apache.solr.handler.component.RangeFacetRequest.RangeEndpointCalculator} + * depending on the field type of the schema field + */ + private RangeEndpointCalculator> createCalculator() { + RangeEndpointCalculator calc; + FieldType ft = schemaField.getType(); + + if (ft instanceof TrieField) { + final TrieField trie = (TrieField) ft; + + switch (trie.getType()) { + case FLOAT: + calc = new FloatRangeEndpointCalculator(this); + break; + case DOUBLE: + calc = new DoubleRangeEndpointCalculator(this); + break; + case INTEGER: + calc = new IntegerRangeEndpointCalculator(this); + break; + case LONG: + calc = new LongRangeEndpointCalculator(this); + break; + case DATE: + calc = new DateRangeEndpointCalculator(this, null); + break; + default: + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "Unable to range facet on tried field of unexpected type:" + this.facetOn); + } + } else if (ft instanceof DateRangeField) { + calc = new DateRangeFieldEndpointCalculator(this, null); + } else { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "Unable to range facet on field:" + schemaField); + } + + return calc; + } + + /** + * @return the start of this range as specified by {@link FacetParams#FACET_RANGE_START} parameter + */ + public String getStart() { + return start; + } + + /** + * The end of this facet.range as specified by {@link FacetParams#FACET_RANGE_END} parameter + *

+ * Note that the actual computed end value can be different depending on the + * {@link FacetParams#FACET_RANGE_HARD_END} parameter. See {@link #endObj} + */ + public String getEnd() { + return end; + } + + /** + * @return an {@link EnumSet} containing all the values specified via + * {@link FacetParams#FACET_RANGE_INCLUDE} parameter. Defaults to + * {@link org.apache.solr.common.params.FacetParams.FacetRangeInclude#LOWER} if no parameter + * is supplied. Includes all values from {@link org.apache.solr.common.params.FacetParams.FacetRangeInclude} enum + * if {@link FacetParams#FACET_RANGE_INCLUDE} includes + * {@link org.apache.solr.common.params.FacetParams.FacetRangeInclude#ALL} + */ + public EnumSet getInclude() { + return include; + } + + /** + * @return the gap as specified by {@link FacetParams#FACET_RANGE_GAP} parameter + */ + public String getGap() { + return gap; + } + + /** + * @return the computed gap object + */ + public Object getGapObj() { + return gapObj; + } + + /** + * @return the boolean value of {@link FacetParams#FACET_RANGE_HARD_END} parameter + */ + public boolean isHardEnd() { + return hardEnd; + } + + /** + * @return an {@link EnumSet} of {@link org.apache.solr.common.params.FacetParams.FacetRangeOther} values + * specified by {@link FacetParams#FACET_RANGE_OTHER} parameter + */ + public EnumSet getOthers() { + return others; + } + + /** + * @return the {@link org.apache.solr.common.params.FacetParams.FacetRangeMethod} to be used for computing + * ranges determined either by the value of {@link FacetParams#FACET_RANGE_METHOD} parameter + * or other internal constraints. + */ + public FacetParams.FacetRangeMethod getMethod() { + return method; + } + + /** + * @return the minimum allowed count for facet ranges as specified by {@link FacetParams#FACET_MINCOUNT} + */ + public int getMinCount() { + return minCount; + } + + /** + * @return the {@link SchemaField} instance representing the field on which ranges have to be calculated + */ + public SchemaField getSchemaField() { + return schemaField; + } + + /** + * @return the boolean value specified by {@link GroupParams#GROUP_FACET} parameter + */ + public boolean isGroupFacet() { + return groupFacet; + } + + /** + * @return a {@link List} of {@link org.apache.solr.handler.component.RangeFacetRequest.FacetRange} objects + * representing the ranges (gaps) for which range counts are to be calculated. + */ + public List getFacetRanges() { + return facetRanges; + } + + /** + * @return The computed start value of this range + */ + public Object getStartObj() { + return startObj; + } + + /** + * The end of this facet.range as calculated using the value of facet.range.end + * parameter and facet.range.hardend. This can be different from the + * value specified in facet.range.end if facet.range.hardend=true + */ + public Object getEndObj() { + return endObj; + } + + /** + * Represents a range facet response combined from all shards. + * Provides helper methods to merge facet_ranges response from a shard. + * See {@link #mergeFacetRangesFromShardResponse(LinkedHashMap, SimpleOrderedMap)} + * and {@link #mergeContributionFromShard(SimpleOrderedMap)} + */ + static class DistribRangeFacet { + public SimpleOrderedMap rangeFacet; + + public DistribRangeFacet(SimpleOrderedMap rangeFacet) { + this.rangeFacet = rangeFacet; + } + + /** + * Helper method to merge range facet values from a shard's response to already accumulated + * values for each range. + * + * @param rangeCounts a {@link LinkedHashMap} containing the accumulated values for each range + * keyed by the 'key' of the facet.range. Must not be null. + * @param shardRanges the facet_ranges response from a shard. Must not be null. + */ + public static void mergeFacetRangesFromShardResponse(LinkedHashMap rangeCounts, + SimpleOrderedMap> shardRanges) { + assert shardRanges != null; + assert rangeCounts != null; + for (Map.Entry> entry : shardRanges) { + String rangeKey = entry.getKey(); + + RangeFacetRequest.DistribRangeFacet existing = rangeCounts.get(rangeKey); + if (existing == null) { + rangeCounts.put(rangeKey, new RangeFacetRequest.DistribRangeFacet(entry.getValue())); + } else { + existing.mergeContributionFromShard(entry.getValue()); + } + } + } + + /** + * Accumulates an individual facet_ranges count from a shard into global counts. + *

+ * The implementation below uses the first encountered shard's + * facet_ranges as the basis for subsequent shards' data to be merged. + * + * @param rangeFromShard the facet_ranges response from a shard + */ + public void mergeContributionFromShard(SimpleOrderedMap rangeFromShard) { + if (rangeFacet == null) { + rangeFacet = rangeFromShard; + return; + } + + @SuppressWarnings("unchecked") + NamedList shardFieldValues + = (NamedList) rangeFromShard.get("counts"); + + @SuppressWarnings("unchecked") + NamedList existFieldValues + = (NamedList) rangeFacet.get("counts"); + + for (Map.Entry existPair : existFieldValues) { + final String key = existPair.getKey(); + // can be null if inconsistencies in shards responses + Integer newValue = shardFieldValues.get(key); + if (null != newValue) { + Integer oldValue = existPair.getValue(); + existPair.setValue(oldValue + newValue); + } + } + + // merge facet.other=before/between/after/all if they exist + for (FacetParams.FacetRangeOther otherKey : FacetParams.FacetRangeOther.values()) { + if (otherKey == FacetParams.FacetRangeOther.NONE) continue; + + String name = otherKey.toString(); + Integer shardValue = (Integer) rangeFromShard.get(name); + if (shardValue != null && shardValue > 0) { + Integer existingValue = (Integer) rangeFacet.get(name); + // shouldn't be null + int idx = rangeFacet.indexOf(name, 0); + rangeFacet.setVal(idx, existingValue + shardValue); + } + } + } + + /** + * Removes all counts under the given minCount from the accumulated facet_ranges. + *

+ * Note: this method should only be called after all shard responses have been + * accumulated using {@link #mergeContributionFromShard(SimpleOrderedMap)} + * + * @param minCount the minimum allowed count for any range + */ + public void removeRangeFacetsUnderLimits(int minCount) { + boolean replace = false; + + @SuppressWarnings("unchecked") + NamedList vals = (NamedList) rangeFacet.get("counts"); + NamedList newList = new NamedList<>(); + for (Map.Entry pair : vals) { + if (pair.getValue().longValue() >= minCount) { + newList.add(pair.getKey(), pair.getValue()); + } else { + replace = true; + } + } + if (replace) { + vals.clear(); + vals.addAll(newList); + } + } + } + + /** + * Perhaps someday instead of having a giant "instanceof" case + * statement to pick an impl, we can add a "RangeFacetable" marker + * interface to FieldTypes and they can return instances of these + * directly from some method -- but until then, keep this locked down + * and private. + */ + private static abstract class RangeEndpointCalculator> { + protected final RangeFacetRequest rfr; + protected final SchemaField field; + + /** + * The end of the facet.range as determined by this calculator. + * This can be different from the facet.range.end depending on the + * facet.range.hardend parameter + */ + protected T computedEnd; + + protected T start; + + protected Object gap; + + protected boolean computed = false; + + public RangeEndpointCalculator(RangeFacetRequest rfr) { + this.rfr = rfr; + this.field = rfr.getSchemaField(); + } + + public T getComputedEnd() { + assert computed; + return computedEnd; + } + + public T getStart() { + assert computed; + return start; + } + + /** + * @return the parsed value of {@link FacetParams#FACET_RANGE_GAP} parameter. This type + * of the returned object is the boxed type of the schema field type's primitive counterpart + * except in the case of Dates in which case the returned type is just a string (because in + * case of dates the gap can either be a date or a DateMath string). + */ + public Object getGap() { + assert computed; + return gap; + } + + /** + * Formats a Range endpoint for use as a range label name in the response. + * Default Impl just uses toString() + */ + public String formatValue(final T val) { + return val.toString(); + } + + /** + * Parses a String param into an Range endpoint value throwing + * a useful exception if not possible + */ + public final T getValue(final String rawval) { + try { + return parseVal(rawval); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Can't parse value " + rawval + " for field: " + + field.getName(), e); + } + } + + /** + * Parses a String param into an Range endpoint. + * Can throw a low level format exception as needed. + */ + protected abstract T parseVal(final String rawval) + throws java.text.ParseException; + + /** + * Parses a String param into a value that represents the gap and + * can be included in the response, throwing + * a useful exception if not possible. + *

+ * Note: uses Object as the return type instead of T for things like + * Date where gap is just a DateMathParser string + */ + protected final Object getGap(final String gap) { + try { + return parseGap(gap); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Can't parse gap " + gap + " for field: " + + field.getName(), e); + } + } + + /** + * Parses a String param into a value that represents the gap and + * can be included in the response. + * Can throw a low level format exception as needed. + *

+ * Default Impl calls parseVal + */ + protected Object parseGap(final String rawval) + throws java.text.ParseException { + return parseVal(rawval); + } + + /** + * Adds the String gap param to a low Range endpoint value to determine + * the corresponding high Range endpoint value, throwing + * a useful exception if not possible. + */ + public final T addGap(T value, String gap) { + try { + return parseAndAddGap(value, gap); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Can't add gap " + gap + " to value " + value + + " for field: " + field.getName(), e); + } + } + + /** + * Adds the String gap param to a low Range endpoint value to determine + * the corrisponding high Range endpoint value. + * Can throw a low level format exception as needed. + */ + protected abstract T parseAndAddGap(T value, String gap) + throws java.text.ParseException; + + public List computeRanges() { + List ranges = new ArrayList<>(); + + this.gap = getGap(rfr.getGap()); + this.start = getValue(rfr.getStart()); + // not final, hardend may change this + T end = getValue(rfr.getEnd()); + if (end.compareTo(start) < 0) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "range facet 'end' comes before 'start': " + end + " < " + start); + } + + final EnumSet include = rfr.getInclude(); + + T low = start; + + while (low.compareTo(end) < 0) { + T high = addGap(low, rfr.getGap()); + if (end.compareTo(high) < 0) { + if (rfr.isHardEnd()) { + high = end; + } else { + end = high; + } + } + if (high.compareTo(low) < 0) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "range facet infinite loop (is gap negative? did the math overflow?)"); + } + if (high.compareTo(low) == 0) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "range facet infinite loop: gap is either zero, or too small relative start/end and caused underflow: " + low + " + " + rfr.getGap() + " = " + high); + } + + final boolean includeLower = + (include.contains(FacetParams.FacetRangeInclude.LOWER) || + (include.contains(FacetParams.FacetRangeInclude.EDGE) && + 0 == low.compareTo(start))); + final boolean includeUpper = + (include.contains(FacetParams.FacetRangeInclude.UPPER) || + (include.contains(FacetParams.FacetRangeInclude.EDGE) && + 0 == high.compareTo(end))); + + final String lowS = formatValue(low); + final String highS = formatValue(high); + + ranges.add(new FacetRange(lowS, lowS, highS, includeLower, includeUpper)); + + low = high; + } + + // we must update the end value in RangeFacetRequest because the end is returned + // as a separate element in the range facet response + this.computedEnd = end; + this.computed = true; + + // no matter what other values are listed, we don't do + // anything if "none" is specified. + if (!rfr.getOthers().contains(FacetParams.FacetRangeOther.NONE)) { + + boolean all = rfr.getOthers().contains(FacetParams.FacetRangeOther.ALL); + final String startS = formatValue(start); + final String endS = formatValue(end); + + if (all || rfr.getOthers().contains(FacetParams.FacetRangeOther.BEFORE)) { + // include upper bound if "outer" or if first gap doesn't already include it + ranges.add(new FacetRange(FacetParams.FacetRangeOther.BEFORE, + null, startS, false, include.contains(FacetParams.FacetRangeInclude.OUTER) || include.contains(FacetParams.FacetRangeInclude.ALL) || + !(include.contains(FacetParams.FacetRangeInclude.LOWER) || include.contains(FacetParams.FacetRangeInclude.EDGE)))); + } + if (all || rfr.getOthers().contains(FacetParams.FacetRangeOther.AFTER)) { + // include lower bound if "outer" or if last gap doesn't already include it + ranges.add(new FacetRange(FacetParams.FacetRangeOther.AFTER, + endS, null, include.contains(FacetParams.FacetRangeInclude.OUTER) || include.contains(FacetParams.FacetRangeInclude.ALL) || + !(include.contains(FacetParams.FacetRangeInclude.UPPER) || include.contains(FacetParams.FacetRangeInclude.EDGE)), false)); + } + if (all || rfr.getOthers().contains(FacetParams.FacetRangeOther.BETWEEN)) { + ranges.add(new FacetRange(FacetParams.FacetRangeOther.BETWEEN, startS, endS, + include.contains(FacetParams.FacetRangeInclude.LOWER) || include.contains(FacetParams.FacetRangeInclude.EDGE) || include.contains(FacetParams.FacetRangeInclude.ALL), + include.contains(FacetParams.FacetRangeInclude.UPPER) || include.contains(FacetParams.FacetRangeInclude.EDGE) || include.contains(FacetParams.FacetRangeInclude.ALL))); + } + } + + return ranges; + } + + } + + private static class FloatRangeEndpointCalculator + extends RangeEndpointCalculator { + + public FloatRangeEndpointCalculator(final RangeFacetRequest rangeFacetRequest) { + super(rangeFacetRequest); + } + + @Override + protected Float parseVal(String rawval) { + return Float.valueOf(rawval); + } + + @Override + public Float parseAndAddGap(Float value, String gap) { + return new Float(value.floatValue() + Float.valueOf(gap).floatValue()); + } + } + + private static class DoubleRangeEndpointCalculator + extends RangeEndpointCalculator { + + public DoubleRangeEndpointCalculator(final RangeFacetRequest rangeFacetRequest) { + super(rangeFacetRequest); + } + + @Override + protected Double parseVal(String rawval) { + return Double.valueOf(rawval); + } + + @Override + public Double parseAndAddGap(Double value, String gap) { + return new Double(value.doubleValue() + Double.valueOf(gap).doubleValue()); + } + } + + private static class IntegerRangeEndpointCalculator + extends RangeEndpointCalculator { + + public IntegerRangeEndpointCalculator(final RangeFacetRequest rangeFacetRequest) { + super(rangeFacetRequest); + } + + @Override + protected Integer parseVal(String rawval) { + return Integer.valueOf(rawval); + } + + @Override + public Integer parseAndAddGap(Integer value, String gap) { + return new Integer(value.intValue() + Integer.valueOf(gap).intValue()); + } + } + + private static class LongRangeEndpointCalculator + extends RangeEndpointCalculator { + + public LongRangeEndpointCalculator(final RangeFacetRequest rangeFacetRequest) { + super(rangeFacetRequest); + } + + @Override + protected Long parseVal(String rawval) { + return Long.valueOf(rawval); + } + + @Override + public Long parseAndAddGap(Long value, String gap) { + return new Long(value.longValue() + Long.valueOf(gap).longValue()); + } + } + + private static class DateRangeEndpointCalculator + extends RangeEndpointCalculator { + private static final String TYPE_ERR_MSG = "SchemaField must use field type extending TrieDateField or DateRangeField"; + private final Date now; + + public DateRangeEndpointCalculator(final RangeFacetRequest rangeFacetRequest, + final Date now) { + super(rangeFacetRequest); + this.now = now; + if (!(field.getType() instanceof TrieDateField)) { + throw new IllegalArgumentException + (TYPE_ERR_MSG); + } + } + + @Override + public String formatValue(Date val) { + return ((TrieDateField) field.getType()).toExternal(val); + } + + @Override + protected Date parseVal(String rawval) { + return ((TrieDateField) field.getType()).parseMath(now, rawval); + } + + @Override + protected Object parseGap(final String rawval) { + return rawval; + } + + @Override + public Date parseAndAddGap(Date value, String gap) throws java.text.ParseException { + final DateMathParser dmp = new DateMathParser(); + dmp.setNow(value); + return dmp.parseMath(gap); + } + } + + private static class DateRangeFieldEndpointCalculator + extends RangeEndpointCalculator { + private final Date now; + + public DateRangeFieldEndpointCalculator(final RangeFacetRequest rangeFacetRequest, + final Date now) { + super(rangeFacetRequest); + this.now = now; + if (!(field.getType() instanceof DateRangeField)) { + throw new IllegalArgumentException(DateRangeEndpointCalculator.TYPE_ERR_MSG); + } + } + + @Override + public String formatValue(Date val) { + return TrieDateField.formatExternal(val); + } + + @Override + protected Date parseVal(String rawval) { + return ((DateRangeField) field.getType()).parseMath(now, rawval); + } + + @Override + protected Object parseGap(final String rawval) { + return rawval; + } + + @Override + public Date parseAndAddGap(Date value, String gap) throws java.text.ParseException { + final DateMathParser dmp = new DateMathParser(); + dmp.setNow(value); + return dmp.parseMath(gap); + } + } + + /** + * Represents a single facet range (or gap) for which the count is to be calculated + */ + public static class FacetRange { + public final FacetParams.FacetRangeOther other; + public final String name; + public final String lower; + public final String upper; + public final boolean includeLower; + public final boolean includeUpper; + + private FacetRange(FacetParams.FacetRangeOther other, String name, String lower, String upper, boolean includeLower, boolean includeUpper) { + this.other = other; + this.name = name; + this.lower = lower; + this.upper = upper; + this.includeLower = includeLower; + this.includeUpper = includeUpper; + } + + /** + * Construct a facet range for a {@link org.apache.solr.common.params.FacetParams.FacetRangeOther} instance + */ + public FacetRange(FacetParams.FacetRangeOther other, String lower, String upper, boolean includeLower, boolean includeUpper) { + this(other, other.toString(), lower, upper, includeLower, includeUpper); + } + + /** + * Construct a facet range for the give name + */ + public FacetRange(String name, String lower, String upper, boolean includeLower, boolean includeUpper) { + this(null, name, lower, upper, includeLower, includeUpper); + } + } +} + diff --git a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java index 7c7e776634b..fe4cee0cc6b 100644 --- a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java +++ b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java @@ -120,7 +120,7 @@ public class IntervalFacets implements Iterable { * Constructor that accepts an already constructed array of {@link FacetInterval} objects. This array needs to be sorted * by start value in weakly ascending order. null values are not allowed in the array. */ - IntervalFacets(SchemaField schemaField, SolrIndexSearcher searcher, DocSet docs, FacetInterval[] intervals) throws IOException { + public IntervalFacets(SchemaField schemaField, SolrIndexSearcher searcher, DocSet docs, FacetInterval[] intervals) throws IOException { this.schemaField = schemaField; this.searcher = searcher; this.docs = docs; @@ -360,7 +360,7 @@ public class IntervalFacets implements Iterable { /** * Helper class to match and count of documents in specified intervals */ - static class FacetInterval { + public static class FacetInterval { /** * Key to represent this interval @@ -519,7 +519,7 @@ public class IntervalFacets implements Iterable { * @param includeUpper Indicates weather this interval should include values equal to end * @param key String key of this interval */ - FacetInterval(SchemaField schemaField, String startStr, String endStr, + public FacetInterval(SchemaField schemaField, String startStr, String endStr, boolean includeLower, boolean includeUpper, String key) { assert schemaField.getType().getNumericType() != null: "Only numeric fields supported with this constructor"; this.key = key; diff --git a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java index 728ce4ff01e..cebf640eac2 100644 --- a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java +++ b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java @@ -32,10 +32,8 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Filter; import org.apache.lucene.search.FilterCollector; import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector; import org.apache.lucene.search.grouping.term.TermAllGroupsCollector; import org.apache.lucene.search.grouping.term.TermGroupFacetCollector; @@ -46,9 +44,6 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.FacetParams; -import org.apache.solr.common.params.FacetParams.FacetRangeInclude; -import org.apache.solr.common.params.FacetParams.FacetRangeMethod; -import org.apache.solr.common.params.FacetParams.FacetRangeOther; import org.apache.solr.common.params.GroupParams; import org.apache.solr.common.params.RequiredSolrParams; import org.apache.solr.common.params.SolrParams; @@ -56,15 +51,14 @@ import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; +import org.apache.solr.handler.component.FacetComponent; import org.apache.solr.handler.component.ResponseBuilder; import org.apache.solr.handler.component.SpatialHeatmapFacets; import org.apache.solr.request.IntervalFacets.FacetInterval; import org.apache.solr.schema.BoolField; -import org.apache.solr.schema.DateRangeField; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; -import org.apache.solr.schema.TrieDateField; import org.apache.solr.schema.TrieField; import org.apache.solr.search.BitDocSet; import org.apache.solr.search.DocSet; @@ -78,7 +72,6 @@ import org.apache.solr.search.SortedIntDocSet; import org.apache.solr.search.SyntaxError; import org.apache.solr.search.grouping.GroupingSpecification; import org.apache.solr.util.BoundedTreeSet; -import org.apache.solr.util.DateMathParser; import org.apache.solr.util.DefaultSolrThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,12 +79,10 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Date; -import java.util.EnumSet; +import java.util.Collections; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; @@ -121,8 +112,6 @@ public class SimpleFacets { protected final SolrQueryRequest req; protected final ResponseBuilder rb; - protected SimpleOrderedMap facetResponse; - // per-facet values protected final static class ParsedParams { final public SolrParams localParams; // localParams on this particular facet command @@ -131,6 +120,7 @@ public class SimpleFacets { final public String facetValue; // the field to or query to facet on (minus local params) final public DocSet docs; // the base docset for this particular facet final public String key; // what name should the results be stored under + final public List tags; // the tags applied to this facet value final public int threads; public ParsedParams(final SolrParams localParams, // localParams on this particular facet command @@ -139,6 +129,7 @@ public class SimpleFacets { final String facetValue, // the field to or query to facet on (minus local params) final DocSet docs, // the base docset for this particular facet final String key, // what name should the results be stored under + final List tags, final int threads) { this.localParams = localParams; this.params = params; @@ -146,14 +137,15 @@ public class SimpleFacets { this.facetValue = facetValue; this.docs = docs; this.key = key; + this.tags = tags; this.threads = threads; } public ParsedParams withDocs(final DocSet docs) { - return new ParsedParams(localParams, params, required, facetValue, docs, key, threads); + return new ParsedParams(localParams, params, required, facetValue, docs, key, tags, threads); } } - + public SimpleFacets(SolrQueryRequest req, DocSet docs, SolrParams params) { @@ -196,12 +188,13 @@ public class SimpleFacets { DocSet docs = docsOrig; String facetValue = param; String key = param; + List tags = Collections.emptyList(); int threads = -1; if (localParams == null) { SolrParams params = global; SolrParams required = new RequiredSolrParams(params); - return new ParsedParams(localParams, params, required, facetValue, docs, key, threads); + return new ParsedParams(localParams, params, required, facetValue, docs, key, tags, threads); } SolrParams params = SolrParams.wrapDefaults(localParams, global); @@ -218,6 +211,9 @@ public class SimpleFacets { // allow explicit set of the key key = localParams.get(CommonParams.OUTPUT_KEY, key); + String tagStr = localParams.get(CommonParams.TAG); + tags = tagStr == null ? Collections.emptyList() : StrUtils.splitSmart(tagStr,','); + String threadStr = localParams.get(CommonParams.THREADS); if (threadStr != null) { threads = Integer.parseInt(threadStr); @@ -225,102 +221,83 @@ public class SimpleFacets { // figure out if we need a new base DocSet String excludeStr = localParams.get(CommonParams.EXCLUDE); - if (excludeStr == null) return new ParsedParams(localParams, params, required, facetValue, docs, key, threads); + if (excludeStr == null) return new ParsedParams(localParams, params, required, facetValue, docs, key, tags, threads); + List excludeTagList = StrUtils.splitSmart(excludeStr,','); + docs = computeDocSet(docs, excludeTagList); + return new ParsedParams(localParams, params, required, facetValue, docs, key, tags, threads); + } + + protected DocSet computeDocSet(DocSet baseDocSet, List excludeTagList) throws SyntaxError, IOException { Map tagMap = (Map)req.getContext().get("tags"); - if (tagMap != null && rb != null) { - List excludeTagList = StrUtils.splitSmart(excludeStr,','); + // rb can be null if facets are being calculated from a RequestHandler e.g. MoreLikeThisHandler + if (tagMap == null || rb == null) { + return baseDocSet; + } - IdentityHashMap excludeSet = new IdentityHashMap<>(); - for (String excludeTag : excludeTagList) { - Object olst = tagMap.get(excludeTag); - // tagMap has entries of List>, but subject to change in the future - if (!(olst instanceof Collection)) continue; - for (Object o : (Collection)olst) { - if (!(o instanceof QParser)) continue; - QParser qp = (QParser)o; - excludeSet.put(qp.getQuery(), Boolean.TRUE); - } + IdentityHashMap excludeSet = new IdentityHashMap<>(); + for (String excludeTag : excludeTagList) { + Object olst = tagMap.get(excludeTag); + // tagMap has entries of List>, but subject to change in the future + if (!(olst instanceof Collection)) continue; + for (Object o : (Collection)olst) { + if (!(o instanceof QParser)) continue; + QParser qp = (QParser)o; + excludeSet.put(qp.getQuery(), Boolean.TRUE); } - if (excludeSet.size() == 0) return new ParsedParams(localParams, params, required, facetValue, docs, key, threads); + } + if (excludeSet.size() == 0) return baseDocSet; - List qlist = new ArrayList<>(); + List qlist = new ArrayList<>(); - // add the base query - if (!excludeSet.containsKey(rb.getQuery())) { - qlist.add(rb.getQuery()); - } + // add the base query + if (!excludeSet.containsKey(rb.getQuery())) { + qlist.add(rb.getQuery()); + } - // add the filters - if (rb.getFilters() != null) { - for (Query q : rb.getFilters()) { - if (!excludeSet.containsKey(q)) { - qlist.add(q); - } + // add the filters + if (rb.getFilters() != null) { + for (Query q : rb.getFilters()) { + if (!excludeSet.containsKey(q)) { + qlist.add(q); } } - - // get the new base docset for this facet - DocSet base = searcher.getDocSet(qlist); - if (rb.grouping() && rb.getGroupingSpec().isTruncateGroups()) { - Grouping grouping = new Grouping(searcher, null, rb.getQueryCommand(), false, 0, false); - grouping.setGroupSort(rb.getGroupingSpec().getSortWithinGroup()); - if (rb.getGroupingSpec().getFields().length > 0) { - grouping.addFieldCommand(rb.getGroupingSpec().getFields()[0], req); - } else if (rb.getGroupingSpec().getFunctions().length > 0) { - grouping.addFunctionCommand(rb.getGroupingSpec().getFunctions()[0], req); - } else { - docs = base; - return new ParsedParams(localParams, params, required, facetValue, docs, key, threads); - } - AbstractAllGroupHeadsCollector allGroupHeadsCollector = grouping.getCommands().get(0).createAllGroupCollector(); - searcher.search(base.getTopFilter(), allGroupHeadsCollector); - docs = new BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc())); - } else { - docs = base; - } } - return new ParsedParams(localParams, params, required, facetValue, docs, key, threads); + // get the new base docset for this facet + DocSet base = searcher.getDocSet(qlist); + if (rb.grouping() && rb.getGroupingSpec().isTruncateGroups()) { + Grouping grouping = new Grouping(searcher, null, rb.getQueryCommand(), false, 0, false); + grouping.setGroupSort(rb.getGroupingSpec().getSortWithinGroup()); + if (rb.getGroupingSpec().getFields().length > 0) { + grouping.addFieldCommand(rb.getGroupingSpec().getFields()[0], req); + } else if (rb.getGroupingSpec().getFunctions().length > 0) { + grouping.addFunctionCommand(rb.getGroupingSpec().getFunctions()[0], req); + } else { + return base; + } + AbstractAllGroupHeadsCollector allGroupHeadsCollector = grouping.getCommands().get(0).createAllGroupCollector(); + searcher.search(base.getTopFilter(), allGroupHeadsCollector); + return new BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc())); + } else { + return base; + } } - /** * Looks at various Params to determining if any simple Facet Constraint count * computations are desired. * - * @see #getFacetQueryCounts - * @see #getFacetFieldCounts - * @see #getFacetDateCounts - * @see #getFacetRangeCounts - * @see #getFacetIntervalCounts - * @see FacetParams#FACET * @return a NamedList of Facet Count info or null + * @deprecated use {@link org.apache.solr.handler.component.FacetComponent#getFacetCounts(SimpleFacets)} instead */ + @Deprecated public NamedList getFacetCounts() { - - // if someone called this method, benefit of the doubt: assume true - if (!global.getBool(FacetParams.FACET,true)) - return null; - - facetResponse = new SimpleOrderedMap<>(); - try { - facetResponse.add("facet_queries", getFacetQueryCounts()); - facetResponse.add("facet_fields", getFacetFieldCounts()); - facetResponse.add("facet_dates", getFacetDateCounts()); - facetResponse.add("facet_ranges", getFacetRangeCounts()); - facetResponse.add("facet_intervals", getFacetIntervalCounts()); - facetResponse.add(SpatialHeatmapFacets.RESPONSE_KEY, getHeatmapCounts()); - } catch (IOException e) { - throw new SolrException(ErrorCode.SERVER_ERROR, e); - } catch (SyntaxError e) { - throw new SolrException(ErrorCode.BAD_REQUEST, e); - } - return facetResponse; + return FacetComponent.getFacetCounts(this); } /** - * Returns a list of facet counts for each of the facet queries + * Returns a list of facet counts for each of the facet queries * specified in the params * * @see FacetParams#FACET_QUERY @@ -341,39 +318,44 @@ public class SimpleFacets { if (null != facetQs && 0 != facetQs.length) { for (String q : facetQs) { final ParsedParams parsed = parseParams(FacetParams.FACET_QUERY, q); - - // TODO: slight optimization would prevent double-parsing of any localParams - Query qobj = QParser.getParser(q, null, req).getQuery(); - - if (qobj == null) { - res.add(parsed.key, 0); - } else if (parsed.params.getBool(GroupParams.GROUP_FACET, false)) { - res.add(parsed.key, getGroupedFacetQueryCount(qobj, parsed)); - } else { - res.add(parsed.key, searcher.numDocs(qobj, parsed.docs)); - } + getFacetQueryCount(parsed, res); } } return res; } - + + public void getFacetQueryCount(ParsedParams parsed, NamedList res) throws SyntaxError, IOException { + // TODO: slight optimization would prevent double-parsing of any localParams + // TODO: SOLR-7753 + Query qobj = QParser.getParser(parsed.facetValue, null, req).getQuery(); + + if (qobj == null) { + res.add(parsed.key, 0); + } else if (parsed.params.getBool(GroupParams.GROUP_FACET, false)) { + res.add(parsed.key, getGroupedFacetQueryCount(qobj, parsed.docs)); + } else { + res.add(parsed.key, searcher.numDocs(qobj, parsed.docs)); + } + } + /** * Returns a grouped facet count for the facet query * * @see FacetParams#FACET_QUERY */ - public int getGroupedFacetQueryCount(Query facetQuery, ParsedParams parsed) throws IOException { - String groupField = parsed.params.get(GroupParams.GROUP_FIELD); + public int getGroupedFacetQueryCount(Query facetQuery, DocSet docSet) throws IOException { + // It is okay to retrieve group.field from global because it is never a local param + String groupField = global.get(GroupParams.GROUP_FIELD); if (groupField == null) { throw new SolrException ( SolrException.ErrorCode.BAD_REQUEST, "Specify the group.field as parameter or local parameter" ); } - + TermAllGroupsCollector collector = new TermAllGroupsCollector(groupField); - Filter mainQueryFilter = parsed.docs.getTopFilter(); // This returns a filter that only matches documents matching with q param and fq params + Filter mainQueryFilter = docSet.getTopFilter(); // This returns a filter that only matches documents matching with q param and fq params Query filteredFacetQuery = new BooleanQuery.Builder() .add(facetQuery, Occur.MUST) .add(mainQueryFilter, Occur.FILTER) @@ -488,7 +470,7 @@ public class SimpleFacets { switch (method) { case ENUM: assert TrieField.getMainValuePrefix(ft) == null; - counts = getFacetTermEnumCounts(searcher, docs, field, offset, limit, mincount, missing, sort, prefix, contains, ignoreCase); + counts = getFacetTermEnumCounts(searcher, docs, field, offset, limit, mincount,missing,sort,prefix, contains, ignoreCase, params); break; case FCS: assert !multiToken; @@ -727,7 +709,7 @@ public class SimpleFacets { * @see FacetParams#FACET_ZEROS * @see FacetParams#FACET_MISSING */ - public NamedList getFacetTermEnumCounts(SolrIndexSearcher searcher, DocSet docs, String field, int offset, int limit, int mincount, boolean missing, String sort, String prefix, String contains, boolean ignoreCase) + public NamedList getFacetTermEnumCounts(SolrIndexSearcher searcher, DocSet docs, String field, int offset, int limit, int mincount, boolean missing, String sort, String prefix, String contains, boolean ignoreCase, SolrParams params) throws IOException { /* :TODO: potential optimization... @@ -885,613 +867,6 @@ public class SimpleFacets { return res; } - /** - * Returns a list of value constraints and the associated facet counts - * for each facet date field, range, and interval specified in the - * SolrParams - * - * @see FacetParams#FACET_DATE - * @deprecated Use getFacetRangeCounts which is more generalized - */ - @Deprecated - public NamedList getFacetDateCounts() - throws IOException, SyntaxError { - - final NamedList resOuter = new SimpleOrderedMap<>(); - final String[] fields = global.getParams(FacetParams.FACET_DATE); - - if (null == fields || 0 == fields.length) return resOuter; - - for (String f : fields) { - getFacetDateCounts(f, resOuter); - } - - return resOuter; - } - - /** - * @deprecated Use getFacetRangeCounts which is more generalized - */ - @Deprecated - public void getFacetDateCounts(String dateFacet, NamedList resOuter) - throws IOException, SyntaxError { - - final IndexSchema schema = searcher.getSchema(); - - final ParsedParams parsed = parseParams(FacetParams.FACET_DATE, dateFacet); - final SolrParams params = parsed.params; - final SolrParams required = parsed.required; - final String key = parsed.key; - final String f = parsed.facetValue; - - final NamedList resInner = new SimpleOrderedMap<>(); - resOuter.add(key, resInner); - final SchemaField sf = schema.getField(f); - if (! (sf.getType() instanceof TrieDateField)) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "Can not date facet on a field which is not a TrieDateField: " + f); - } - final TrieDateField ft = (TrieDateField) sf.getType(); - final String startS - = required.getFieldParam(f,FacetParams.FACET_DATE_START); - final Date start; - try { - start = ft.parseMath(null, startS); - } catch (SolrException e) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "date facet 'start' is not a valid Date string: " + startS, e); - } - final String endS - = required.getFieldParam(f,FacetParams.FACET_DATE_END); - Date end; // not final, hardend may change this - try { - end = ft.parseMath(null, endS); - } catch (SolrException e) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "date facet 'end' is not a valid Date string: " + endS, e); - } - - if (end.before(start)) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "date facet 'end' comes before 'start': "+endS+" < "+startS); - } - - final String gap = required.getFieldParam(f,FacetParams.FACET_DATE_GAP); - final DateMathParser dmp = new DateMathParser(); - - final int minCount = params.getFieldInt(f,FacetParams.FACET_MINCOUNT, 0); - - String[] iStrs = params.getFieldParams(f,FacetParams.FACET_DATE_INCLUDE); - // Legacy support for default of [lower,upper,edge] for date faceting - // this is not handled by FacetRangeInclude.parseParam because - // range faceting has differnet defaults - final EnumSet include = - (null == iStrs || 0 == iStrs.length ) ? - EnumSet.of(FacetRangeInclude.LOWER, - FacetRangeInclude.UPPER, - FacetRangeInclude.EDGE) - : FacetRangeInclude.parseParam(iStrs); - - try { - Date low = start; - while (low.before(end)) { - dmp.setNow(low); - String label = ft.toExternal(low); - - Date high = dmp.parseMath(gap); - if (end.before(high)) { - if (params.getFieldBool(f,FacetParams.FACET_DATE_HARD_END,false)) { - high = end; - } else { - end = high; - } - } - if (high.before(low)) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "date facet infinite loop (is gap negative?)"); - } - if (high.equals(low)) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "date facet infinite loop: gap is effectively zero"); - } - final boolean includeLower = - (include.contains(FacetRangeInclude.LOWER) || - (include.contains(FacetRangeInclude.EDGE) && low.equals(start))); - final boolean includeUpper = - (include.contains(FacetRangeInclude.UPPER) || - (include.contains(FacetRangeInclude.EDGE) && high.equals(end))); - - final int count = rangeCount(parsed,sf,low,high,includeLower,includeUpper); - if (count >= minCount) { - resInner.add(label, count); - } - low = high; - } - } catch (java.text.ParseException e) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "date facet 'gap' is not a valid Date Math string: " + gap, e); - } - - // explicitly return the gap and end so all the counts - // (including before/after/between) are meaningful - even if mincount - // has removed the neighboring ranges - resInner.add("gap", gap); - resInner.add("start", start); - resInner.add("end", end); - - final String[] othersP = - params.getFieldParams(f,FacetParams.FACET_DATE_OTHER); - if (null != othersP && 0 < othersP.length ) { - final Set others = EnumSet.noneOf(FacetRangeOther.class); - - for (final String o : othersP) { - others.add(FacetRangeOther.get(o)); - } - - // no matter what other values are listed, we don't do - // anything if "none" is specified. - if (! others.contains(FacetRangeOther.NONE) ) { - boolean all = others.contains(FacetRangeOther.ALL); - - if (all || others.contains(FacetRangeOther.BEFORE)) { - // include upper bound if "outer" or if first gap doesn't already include it - resInner.add(FacetRangeOther.BEFORE.toString(), - rangeCount(parsed,sf,null,start, - false, - (include.contains(FacetRangeInclude.OUTER) || - (! (include.contains(FacetRangeInclude.LOWER) || - include.contains(FacetRangeInclude.EDGE)))))); - } - if (all || others.contains(FacetRangeOther.AFTER)) { - // include lower bound if "outer" or if last gap doesn't already include it - resInner.add(FacetRangeOther.AFTER.toString(), - rangeCount(parsed,sf,end,null, - (include.contains(FacetRangeInclude.OUTER) || - (! (include.contains(FacetRangeInclude.UPPER) || - include.contains(FacetRangeInclude.EDGE)))), - false)); - } - if (all || others.contains(FacetRangeOther.BETWEEN)) { - resInner.add(FacetRangeOther.BETWEEN.toString(), - rangeCount(parsed,sf,start,end, - (include.contains(FacetRangeInclude.LOWER) || - include.contains(FacetRangeInclude.EDGE)), - (include.contains(FacetRangeInclude.UPPER) || - include.contains(FacetRangeInclude.EDGE)))); - } - } - } - } - - - /** - * Returns a list of value constraints and the associated facet - * counts for each facet numerical field, range, and interval - * specified in the SolrParams - * - * @see FacetParams#FACET_RANGE - */ - - public NamedList getFacetRangeCounts() throws IOException, SyntaxError { - final NamedList resOuter = new SimpleOrderedMap<>(); - final String[] fields = global.getParams(FacetParams.FACET_RANGE); - - if (null == fields || 0 == fields.length) return resOuter; - - for (String f : fields) { - getFacetRangeCounts(f, resOuter); - } - - return resOuter; - } - - void getFacetRangeCounts(String facetRange, NamedList resOuter) - throws IOException, SyntaxError { - - final IndexSchema schema = searcher.getSchema(); - - final ParsedParams parsed = parseParams(FacetParams.FACET_RANGE, facetRange); - final String key = parsed.key; - final String f = parsed.facetValue; - String methodStr = parsed.params.get(FacetParams.FACET_RANGE_METHOD); - FacetRangeMethod method = (methodStr==null?FacetRangeMethod.getDefault():FacetRangeMethod.get(methodStr)); - boolean groupFacet = parsed.params.getBool(GroupParams.GROUP_FACET, false); - if (groupFacet && method.equals(FacetRangeMethod.DV)) { - // the user has explicitly selected the FacetRangeMethod.DV method - log.warn("Range facet method '" + FacetRangeMethod.DV + "' is not supported together with '" + - GroupParams.GROUP_FACET + "'. Will use method '" + FacetRangeMethod.FILTER + "' instead"); - method = FacetRangeMethod.FILTER; - } - - final SchemaField sf = schema.getField(f); - final FieldType ft = sf.getType(); - - RangeEndpointCalculator calc = null; - - if (ft instanceof TrieField) { - final TrieField trie = (TrieField)ft; - - switch (trie.getType()) { - case FLOAT: - calc = new FloatRangeEndpointCalculator(sf); - break; - case DOUBLE: - calc = new DoubleRangeEndpointCalculator(sf); - break; - case INTEGER: - calc = new IntegerRangeEndpointCalculator(sf); - break; - case LONG: - calc = new LongRangeEndpointCalculator(sf); - break; - case DATE: - calc = new DateRangeEndpointCalculator(sf, null); - break; - default: - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "Unable to range facet on tried field of unexpected type:" + f); - } - } else if (ft instanceof DateRangeField) { - calc = new DateRangeFieldEndpointCalculator(sf, null); - if (method.equals(FacetRangeMethod.DV)) { - // the user has explicitly selected the FacetRangeMethod.DV method - log.warn("Range facet method '" + FacetRangeMethod.DV + "' is not supported together with field type '" + - DateRangeField.class + "'. Will use method '" + FacetRangeMethod.FILTER + "' instead"); - method = FacetRangeMethod.FILTER; - } - } else { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "Unable to range facet on field:" + sf); - } - if (method.equals(FacetRangeMethod.DV)) { - assert ft instanceof TrieField; - resOuter.add(key, getFacetRangeCountsDocValues(sf, calc, parsed)); - } else { - resOuter.add(key, getFacetRangeCounts(sf, calc, parsed)); - } - } - - private > NamedList getFacetRangeCounts - (final SchemaField sf, - final RangeEndpointCalculator calc, - final ParsedParams parsed) throws IOException { - - final String f = sf.getName(); - final SolrParams params = parsed.params; - final SolrParams required = parsed.required; - final NamedList res = new SimpleOrderedMap<>(); - final NamedList counts = new NamedList<>(); - res.add("counts", counts); - - final T start = calc.getValue(required.getFieldParam(f,FacetParams.FACET_RANGE_START)); - // not final, hardend may change this - T end = calc.getValue(required.getFieldParam(f,FacetParams.FACET_RANGE_END)); - if (end.compareTo(start) < 0) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "range facet 'end' comes before 'start': "+end+" < "+start); - } - - final String gap = required.getFieldParam(f, FacetParams.FACET_RANGE_GAP); - // explicitly return the gap. compute this early so we are more - // likely to catch parse errors before attempting math - res.add("gap", calc.getGap(gap)); - - final int minCount = params.getFieldInt(f,FacetParams.FACET_MINCOUNT, 0); - - final EnumSet include = FacetRangeInclude.parseParam - (params.getFieldParams(f,FacetParams.FACET_RANGE_INCLUDE)); - - T low = start; - - while (low.compareTo(end) < 0) { - T high = calc.addGap(low, gap); - if (end.compareTo(high) < 0) { - if (params.getFieldBool(f,FacetParams.FACET_RANGE_HARD_END,false)) { - high = end; - } else { - end = high; - } - } - if (high.compareTo(low) < 0) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "range facet infinite loop (is gap negative? did the math overflow?)"); - } - if (high.compareTo(low) == 0) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "range facet infinite loop: gap is either zero, or too small relative start/end and caused underflow: " + low + " + " + gap + " = " + high ); - } - - final boolean includeLower = - (include.contains(FacetRangeInclude.LOWER) || - (include.contains(FacetRangeInclude.EDGE) && - 0 == low.compareTo(start))); - final boolean includeUpper = - (include.contains(FacetRangeInclude.UPPER) || - (include.contains(FacetRangeInclude.EDGE) && - 0 == high.compareTo(end))); - - final String lowS = calc.formatValue(low); - final String highS = calc.formatValue(high); - - final int count = rangeCount(parsed, sf, lowS, highS, - includeLower,includeUpper); - if (count >= minCount) { - counts.add(lowS, count); - } - - low = high; - } - - // explicitly return the start and end so all the counts - // (including before/after/between) are meaningful - even if mincount - // has removed the neighboring ranges - res.add("start", start); - res.add("end", end); - - final String[] othersP = - params.getFieldParams(f,FacetParams.FACET_RANGE_OTHER); - if (null != othersP && 0 < othersP.length ) { - Set others = EnumSet.noneOf(FacetRangeOther.class); - - for (final String o : othersP) { - others.add(FacetRangeOther.get(o)); - } - - // no matter what other values are listed, we don't do - // anything if "none" is specified. - if (! others.contains(FacetRangeOther.NONE) ) { - - boolean all = others.contains(FacetRangeOther.ALL); - final String startS = calc.formatValue(start); - final String endS = calc.formatValue(end); - - if (all || others.contains(FacetRangeOther.BEFORE)) { - // include upper bound if "outer" or if first gap doesn't already include it - res.add(FacetRangeOther.BEFORE.toString(), - rangeCount(parsed,sf,null,startS, - false, - (include.contains(FacetRangeInclude.OUTER) || - (! (include.contains(FacetRangeInclude.LOWER) || - include.contains(FacetRangeInclude.EDGE)))))); - - } - if (all || others.contains(FacetRangeOther.AFTER)) { - // include lower bound if "outer" or if last gap doesn't already include it - res.add(FacetRangeOther.AFTER.toString(), - rangeCount(parsed,sf,endS,null, - (include.contains(FacetRangeInclude.OUTER) || - (! (include.contains(FacetRangeInclude.UPPER) || - include.contains(FacetRangeInclude.EDGE)))), - false)); - } - if (all || others.contains(FacetRangeOther.BETWEEN)) { - res.add(FacetRangeOther.BETWEEN.toString(), - rangeCount(parsed,sf,startS,endS, - (include.contains(FacetRangeInclude.LOWER) || - include.contains(FacetRangeInclude.EDGE)), - (include.contains(FacetRangeInclude.UPPER) || - include.contains(FacetRangeInclude.EDGE)))); - - } - } - } - return res; - } - - private > NamedList getFacetRangeCountsDocValues(final SchemaField sf, - final RangeEndpointCalculator calc, ParsedParams parsed) throws IOException { - - final String f = sf.getName(); - final NamedList res = new SimpleOrderedMap<>(); - final NamedList counts = new NamedList<>(); - res.add("counts", counts); - - String globalStartS = parsed.required.getFieldParam(f,FacetParams.FACET_RANGE_START); - String globalEndS = parsed.required.getFieldParam(f,FacetParams.FACET_RANGE_END); - - final T start = calc.getValue(globalStartS); - // not final, hardend may change this - T end = calc.getValue(globalEndS); - if (end.compareTo(start) < 0) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "range facet 'end' comes before 'start': "+end+" < "+start); - } - - final String gap = parsed.required.getFieldParam(f, FacetParams.FACET_RANGE_GAP); - // explicitly return the gap. compute this early so we are more - // likely to catch parse errors before attempting math - res.add("gap", calc.getGap(gap)); - - final int minCount = parsed.params.getFieldInt(f,FacetParams.FACET_MINCOUNT, 0); - - final EnumSet include = FacetRangeInclude.parseParam - (parsed.params.getFieldParams(f,FacetParams.FACET_RANGE_INCLUDE)); - ArrayList intervals = new ArrayList<>(); - - final String[] othersP = - parsed.params.getFieldParams(f,FacetParams.FACET_RANGE_OTHER); - - boolean includeBefore = false; - boolean includeBetween = false; - boolean includeAfter = false; - - if (othersP != null && othersP.length > 0) { - Set others = EnumSet.noneOf(FacetRangeOther.class); - // Intervals must be in order (see IntervalFacets.getSortedIntervals), if "BEFORE" or - // "BETWEEN" are set, they must be added first - for (final String o : othersP) { - others.add(FacetRangeOther.get(o)); - } - // no matter what other values are listed, we don't do - // anything if "none" is specified. - if (!others.contains(FacetRangeOther.NONE)) { - - if (others.contains(FacetRangeOther.ALL) || others.contains(FacetRangeOther.BEFORE)) { - // We'll add an interval later in this position - intervals.add(null); - includeBefore = true; - } - - if (others.contains(FacetRangeOther.ALL) || others.contains(FacetRangeOther.BETWEEN)) { - // We'll add an interval later in this position - intervals.add(null); - includeBetween = true; - } - - if (others.contains(FacetRangeOther.ALL) || others.contains(FacetRangeOther.AFTER)) { - includeAfter = true; - } - } - - } - - - T low = start; - - while (low.compareTo(end) < 0) { - T high = calc.addGap(low, gap); - if (end.compareTo(high) < 0) { - if (parsed.params.getFieldBool(f,FacetParams.FACET_RANGE_HARD_END,false)) { - high = end; - } else { - end = high; - } - } - if (high.compareTo(low) < 0) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "range facet infinite loop (is gap negative? did the math overflow?)"); - } - if (high.compareTo(low) == 0) { - throw new SolrException - (SolrException.ErrorCode.BAD_REQUEST, - "range facet infinite loop: gap is either zero, or too small relative start/end and caused underflow: " + low + " + " + gap + " = " + high ); - } - - final boolean includeLower = - (include.contains(FacetRangeInclude.LOWER) || - (include.contains(FacetRangeInclude.EDGE) && - 0 == low.compareTo(start))); - final boolean includeUpper = - (include.contains(FacetRangeInclude.UPPER) || - (include.contains(FacetRangeInclude.EDGE) && - 0 == high.compareTo(end))); - - final String lowS = calc.formatValue(low); - final String highS = calc.formatValue(high); - - intervals.add(new IntervalFacets.FacetInterval(sf, lowS, highS, includeLower, includeUpper, lowS)); - - low = high; - } - - if (includeBefore) { - // include upper bound if "outer" or if first gap doesn't already include it - intervals.set(0, new IntervalFacets.FacetInterval(sf, "*", globalStartS, true, - include.contains(FacetRangeInclude.OUTER) || - (! (include.contains(FacetRangeInclude.LOWER) || - include.contains(FacetRangeInclude.EDGE))), FacetRangeOther.BEFORE.toString())); - } - - if (includeBetween) { - int intervalIndex = (includeBefore?1:0); - intervals.set(intervalIndex, new IntervalFacets.FacetInterval(sf, globalStartS, calc.formatValue(end), - include.contains(FacetRangeInclude.LOWER) || - include.contains(FacetRangeInclude.EDGE), - include.contains(FacetRangeInclude.UPPER) || - include.contains(FacetRangeInclude.EDGE), - FacetRangeOther.BETWEEN.toString())); - } - - if (includeAfter) { - // include lower bound if "outer" or if last gap doesn't already include it - intervals.add(new IntervalFacets.FacetInterval(sf, calc.formatValue(end), "*", - (include.contains(FacetRangeInclude.OUTER) || - (! (include.contains(FacetRangeInclude.UPPER) || - include.contains(FacetRangeInclude.EDGE)))), - false, FacetRangeOther.AFTER.toString())); - } - - IntervalFacets.FacetInterval[] intervalsArray = intervals.toArray(new IntervalFacets.FacetInterval[intervals.size()]); - // don't use the ArrayList anymore - intervals = null; - - new IntervalFacets(sf, searcher, parsed.docs, intervalsArray); - - int intervalIndex = 0; - int lastIntervalIndex = intervalsArray.length - 1; - // if the user requested "BEFORE", it will be the first of the intervals. Needs to be added to the - // response named list instead of with the counts - if (includeBefore) { - res.add(intervalsArray[intervalIndex].getKey(), intervalsArray[intervalIndex].getCount()); - intervalIndex++; - } - - // if the user requested "BETWEEN", it will be the first or second of the intervals (depending on if - // "BEFORE" was also requested). Needs to be added to the response named list instead of with the counts - if (includeBetween) { - res.add(intervalsArray[intervalIndex].getKey(), intervalsArray[intervalIndex].getCount()); - intervalIndex++; - } - - // if the user requested "AFTER", it will be the last of the intervals. - // Needs to be added to the response named list instead of with the counts - if (includeAfter) { - res.add(intervalsArray[lastIntervalIndex].getKey(), intervalsArray[lastIntervalIndex].getCount()); - lastIntervalIndex--; - } - // now add all other intervals to the counts NL - while (intervalIndex <= lastIntervalIndex) { - FacetInterval interval = intervalsArray[intervalIndex]; - if (interval.getCount() >= minCount) { - counts.add(interval.getKey(), interval.getCount()); - } - intervalIndex++; - } - - res.add("start", start); - res.add("end", end); - return res; -} - - /** - * Macro for getting the numDocs of range over docs - * @see SolrIndexSearcher#numDocs - * @see TermRangeQuery - */ - protected int rangeCount(ParsedParams parsed, SchemaField sf, String low, String high, - boolean iLow, boolean iHigh) throws IOException { - Query rangeQ = sf.getType().getRangeQuery(null, sf, low, high, iLow, iHigh); - if (parsed.params.getBool(GroupParams.GROUP_FACET, false)) { - return getGroupedFacetQueryCount(rangeQ, parsed); - } else { - return searcher.numDocs(rangeQ , parsed.docs); - } - } - - /** - * @deprecated Use rangeCount(SchemaField,String,String,boolean,boolean) which is more generalized - */ - @Deprecated - protected int rangeCount(ParsedParams parsed, SchemaField sf, Date low, Date high, - boolean iLow, boolean iHigh) throws IOException { - Query rangeQ = ((TrieDateField)(sf.getType())).getRangeQuery(null, sf, low, high, iLow, iHigh); - return searcher.numDocs(rangeQ, parsed.docs); - } - /** * A simple key=>val pair whose natural order is such that * higher vals come before lower vals. @@ -1523,215 +898,6 @@ public class SimpleFacets { } - /** - * Perhaps someday instead of having a giant "instanceof" case - * statement to pick an impl, we can add a "RangeFacetable" marker - * interface to FieldTypes and they can return instances of these - * directly from some method -- but until then, keep this locked down - * and private. - */ - private static abstract class RangeEndpointCalculator> { - protected final SchemaField field; - public RangeEndpointCalculator(final SchemaField field) { - this.field = field; - } - - /** - * Formats a Range endpoint for use as a range label name in the response. - * Default Impl just uses toString() - */ - public String formatValue(final T val) { - return val.toString(); - } - /** - * Parses a String param into an Range endpoint value throwing - * a useful exception if not possible - */ - public final T getValue(final String rawval) { - try { - return parseVal(rawval); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Can't parse value "+rawval+" for field: " + - field.getName(), e); - } - } - /** - * Parses a String param into an Range endpoint. - * Can throw a low level format exception as needed. - */ - protected abstract T parseVal(final String rawval) - throws java.text.ParseException; - - /** - * Parses a String param into a value that represents the gap and - * can be included in the response, throwing - * a useful exception if not possible. - * - * Note: uses Object as the return type instead of T for things like - * Date where gap is just a DateMathParser string - */ - public final Object getGap(final String gap) { - try { - return parseGap(gap); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Can't parse gap "+gap+" for field: " + - field.getName(), e); - } - } - - /** - * Parses a String param into a value that represents the gap and - * can be included in the response. - * Can throw a low level format exception as needed. - * - * Default Impl calls parseVal - */ - protected Object parseGap(final String rawval) - throws java.text.ParseException { - return parseVal(rawval); - } - - /** - * Adds the String gap param to a low Range endpoint value to determine - * the corrisponding high Range endpoint value, throwing - * a useful exception if not possible. - */ - public final T addGap(T value, String gap) { - try { - return parseAndAddGap(value, gap); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Can't add gap "+gap+" to value " + value + - " for field: " + field.getName(), e); - } - } - /** - * Adds the String gap param to a low Range endpoint value to determine - * the corrisponding high Range endpoint value. - * Can throw a low level format exception as needed. - */ - protected abstract T parseAndAddGap(T value, String gap) - throws java.text.ParseException; - - } - - private static class FloatRangeEndpointCalculator - extends RangeEndpointCalculator { - - public FloatRangeEndpointCalculator(final SchemaField f) { super(f); } - @Override - protected Float parseVal(String rawval) { - return Float.valueOf(rawval); - } - @Override - public Float parseAndAddGap(Float value, String gap) { - return new Float(value.floatValue() + Float.valueOf(gap).floatValue()); - } - } - private static class DoubleRangeEndpointCalculator - extends RangeEndpointCalculator { - - public DoubleRangeEndpointCalculator(final SchemaField f) { super(f); } - @Override - protected Double parseVal(String rawval) { - return Double.valueOf(rawval); - } - @Override - public Double parseAndAddGap(Double value, String gap) { - return new Double(value.doubleValue() + Double.valueOf(gap).doubleValue()); - } - } - private static class IntegerRangeEndpointCalculator - extends RangeEndpointCalculator { - - public IntegerRangeEndpointCalculator(final SchemaField f) { super(f); } - @Override - protected Integer parseVal(String rawval) { - return Integer.valueOf(rawval); - } - @Override - public Integer parseAndAddGap(Integer value, String gap) { - return new Integer(value.intValue() + Integer.valueOf(gap).intValue()); - } - } - private static class LongRangeEndpointCalculator - extends RangeEndpointCalculator { - - public LongRangeEndpointCalculator(final SchemaField f) { super(f); } - @Override - protected Long parseVal(String rawval) { - return Long.valueOf(rawval); - } - @Override - public Long parseAndAddGap(Long value, String gap) { - return new Long(value.longValue() + Long.valueOf(gap).longValue()); - } - } - private static class DateRangeEndpointCalculator - extends RangeEndpointCalculator { - private static final String TYPE_ERR_MSG = "SchemaField must use field type extending TrieDateField or DateRangeField"; - private final Date now; - public DateRangeEndpointCalculator(final SchemaField f, - final Date now) { - super(f); - this.now = now; - if (! (field.getType() instanceof TrieDateField) ) { - throw new IllegalArgumentException - (TYPE_ERR_MSG); - } - } - @Override - public String formatValue(Date val) { - return ((TrieDateField)field.getType()).toExternal(val); - } - @Override - protected Date parseVal(String rawval) { - return ((TrieDateField)field.getType()).parseMath(now, rawval); - } - @Override - protected Object parseGap(final String rawval) { - return rawval; - } - @Override - public Date parseAndAddGap(Date value, String gap) throws java.text.ParseException { - final DateMathParser dmp = new DateMathParser(); - dmp.setNow(value); - return dmp.parseMath(gap); - } - } - private static class DateRangeFieldEndpointCalculator - extends RangeEndpointCalculator { - private final Date now; - public DateRangeFieldEndpointCalculator(final SchemaField f, - final Date now) { - super(f); - this.now = now; - if (! (field.getType() instanceof DateRangeField) ) { - throw new IllegalArgumentException(DateRangeEndpointCalculator.TYPE_ERR_MSG); - } - } - @Override - public String formatValue(Date val) { - return TrieDateField.formatExternal(val); - } - @Override - protected Date parseVal(String rawval) { - return ((DateRangeField)field.getType()).parseMath(now, rawval); - } - @Override - protected Object parseGap(final String rawval) { - return rawval; - } - @Override - public Date parseAndAddGap(Date value, String gap) throws java.text.ParseException { - final DateMathParser dmp = new DateMathParser(); - dmp.setNow(value); - return dmp.parseMath(gap); - } - } - /** * Returns a NamedList with each entry having the "key" of the interval as name and the count of docs * in that interval as value. All intervals added in the request are included in the returned @@ -1764,7 +930,7 @@ public class SimpleFacets { return res; } - private NamedList getHeatmapCounts() throws IOException, SyntaxError { + public NamedList getHeatmapCounts() throws IOException, SyntaxError { final NamedList resOuter = new SimpleOrderedMap<>(); String[] unparsedFields = rb.req.getParams().getParams(FacetParams.FACET_HEATMAP); if (unparsedFields == null || unparsedFields.length == 0) { @@ -1781,5 +947,21 @@ public class SimpleFacets { } return resOuter; } + + public SolrParams getGlobalParams() { + return global; + } + + public DocSet getDocsOrig() { + return docsOrig; + } + + public SolrQueryRequest getRequest() { + return req; + } + + public ResponseBuilder getResponseBuilder() { + return rb; + } } diff --git a/solr/core/src/java/org/apache/solr/util/PivotListEntry.java b/solr/core/src/java/org/apache/solr/util/PivotListEntry.java index 770da599a88..6e9a16bef86 100644 --- a/solr/core/src/java/org/apache/solr/util/PivotListEntry.java +++ b/solr/core/src/java/org/apache/solr/util/PivotListEntry.java @@ -34,7 +34,9 @@ public enum PivotListEntry { COUNT(2), // optional entries PIVOT, - STATS; + STATS, + QUERIES, + RANGES; private static final int MIN_INDEX_OF_OPTIONAL = 3; diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java index 6c9da194ad7..b0e5ad24574 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java @@ -17,21 +17,21 @@ package org.apache.solr.handler.component; * limitations under the License. */ +import java.io.IOException; import java.util.Date; import java.util.List; -import java.io.IOException; +import junit.framework.AssertionFailedError; import org.apache.solr.BaseDistributedSearchTestCase; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.response.FieldStatsInfo; import org.apache.solr.client.solrj.response.PivotField; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.RangeFacet; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.FacetParams; import org.apache.solr.common.params.SolrParams; - -import junit.framework.AssertionFailedError; import org.junit.Test; public class DistributedFacetPivotLargeTest extends BaseDistributedSearchTestCase { @@ -665,6 +665,7 @@ public class DistributedFacetPivotLargeTest extends BaseDistributedSearchTestCas FacetParams.FACET_OVERREQUEST_COUNT, "0"); doTestDeepPivotStats(); + doTestPivotRanges(); } private void doTestDeepPivotStats() throws Exception { @@ -739,6 +740,101 @@ public class DistributedFacetPivotLargeTest extends BaseDistributedSearchTestCas assertEquals(1741.742422595, microsoftPlaceholder0PivotFieldStatsInfo.getStddev(), 0.1E-7); } + /** + * spot checks some pivot values and the ranges hanging on them + */ + private void doTestPivotRanges() throws Exception { + + // note: 'p0' is only a top level range, not included in per-pivot ranges + for (SolrParams p : new SolrParams[]{ + // results should be identical for all of these + params("facet.range", "{!key=p0 facet.range.gap=500}pay_i", + "facet.range", "{!key=p1 tag=t1 facet.range.gap=100}pay_i", + "facet.range", "{!key=p2 tag=t1 facet.range.gap=200}pay_i", + "facet.range.start", "0", + "facet.range.end", "1000"), + params("facet.range", "{!key=p0 facet.range.gap=500}pay_i", + "facet.range", "{!key=p1 tag=t1 facet.range.gap=100}pay_i", + "facet.range", "{!key=p2 tag=t1 facet.range.gap=200}pay_i", + "f.pay_i.facet.range.start", "0", + "facet.range.end", "1000"), + params("facet.range", "{!key=p0 facet.range.gap=500 facet.range.start=0}pay_i", + "facet.range", "{!key=p1 tag=t1 facet.range.gap=100 facet.range.start=0}pay_i", + "facet.range", "{!key=p2 tag=t1 facet.range.gap=200 facet.range.start=0}pay_i", + "facet.range.end", "1000")}) { + + QueryResponse rsp + = query(SolrParams.wrapDefaults(p, params("q", "*:*", + "rows", "0", + "facet", "true", + "facet.pivot", "{!range=t1}place_s,company_t"))); + + List pivots = rsp.getFacetPivot().get("place_s,company_t"); + PivotField pf = null; // changes as we spot check + List rfc = null; // changes as we spot check + + // 1st sanity check top level ranges + assertEquals(3, rsp.getFacetRanges().size()); + assertRange("p0", 0, 500, 1000, 2, rsp.getFacetRanges().get(0)); + assertRange("p1", 0, 100, 1000, 10, rsp.getFacetRanges().get(1)); + assertRange("p2", 0, 200, 1000, 5, rsp.getFacetRanges().get(2)); + + // check pivots... + + // first top level pivot value + pf = pivots.get(0); + assertPivot("place_s", "cardiff", 257, pf); + assertRange("p1", 0, 100, 1000, 10, pf.getFacetRanges().get(0)); + assertRange("p2", 0, 200, 1000, 5, pf.getFacetRanges().get(1)); + + rfc = pf.getFacetRanges().get(0).getCounts(); + assertEquals("200", rfc.get(2).getValue()); + assertEquals(14, rfc.get(2).getCount()); + assertEquals("300", rfc.get(3).getValue()); + assertEquals(15, rfc.get(3).getCount()); + + rfc = pf.getFacetRanges().get(1).getCounts(); + assertEquals("200", rfc.get(1).getValue()); + assertEquals(29, rfc.get(1).getCount()); + + // drill down one level of the pivot + pf = pf.getPivot().get(0); + assertPivot("company_t", "bbc", 101, pf); + assertRange("p1", 0, 100, 1000, 10, pf.getFacetRanges().get(0)); + assertRange("p2", 0, 200, 1000, 5, pf.getFacetRanges().get(1)); + + rfc = pf.getFacetRanges().get(0).getCounts(); + for (RangeFacet.Count c : rfc) { + assertEquals(0, c.getCount()); // no docs in our ranges for this pivot drill down + } + + // pop back up and spot check a different top level pivot value + pf = pivots.get(53); + assertPivot("place_s", "placeholder0", 1, pf); + assertRange("p1", 0, 100, 1000, 10, pf.getFacetRanges().get(0)); + assertRange("p2", 0, 200, 1000, 5, pf.getFacetRanges().get(1)); + + rfc = pf.getFacetRanges().get(0).getCounts(); + assertEquals("0", rfc.get(0).getValue()); + assertEquals(1, rfc.get(0).getCount()); + assertEquals("100", rfc.get(1).getValue()); + assertEquals(0, rfc.get(1).getCount()); + + // drill down one level of the pivot + pf = pf.getPivot().get(0); + assertPivot("company_t", "compholder0", 1, pf); + assertRange("p1", 0, 100, 1000, 10, pf.getFacetRanges().get(0)); + assertRange("p2", 0, 200, 1000, 5, pf.getFacetRanges().get(1)); + + rfc = pf.getFacetRanges().get(0).getCounts(); + assertEquals("0", rfc.get(0).getValue()); + assertEquals(1, rfc.get(0).getCount()); + assertEquals("100", rfc.get(1).getValue()); + assertEquals(0, rfc.get(1).getCount()); + + } + } + /** * asserts that the actual PivotField matches the expected criteria */ @@ -751,8 +847,18 @@ public class DistributedFacetPivotLargeTest extends BaseDistributedSearchTestCas //assertEquals("#KIDS: " + actual.toString(), numKids, actual.getPivot().size()); } + /** + * asserts that the actual RangeFacet matches the expected criteria + */ + private void assertRange(String name, Object start, Object gap, Object end, int numCount, + RangeFacet actual) { + assertEquals("NAME: " + actual.toString(), name, actual.getName()); + assertEquals("START: " + actual.toString(), start, actual.getStart()); + assertEquals("GAP: " + actual.toString(), gap, actual.getGap()); + assertEquals("END: " + actual.toString(), end, actual.getEnd()); + assertEquals("#COUNT: " + actual.toString(), numCount, actual.getCounts().size()); + } - private void setupDistributedPivotFacetDocuments() throws Exception{ //Clear docs diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java index 6c16cb627ec..2a01c7b5905 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java @@ -21,17 +21,19 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Date; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import junit.framework.AssertionFailedError; import org.apache.solr.BaseDistributedSearchTestCase; import org.apache.solr.client.solrj.response.FieldStatsInfo; import org.apache.solr.client.solrj.response.PivotField; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.RangeFacet; import org.apache.solr.common.params.FacetParams; -import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.ModifiableSolrParams; - -import junit.framework.AssertionFailedError; +import org.apache.solr.common.params.SolrParams; import org.junit.Test; public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCase { @@ -66,7 +68,7 @@ public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCas handle.clear(); handle.put("QTime", SKIPVAL); handle.put("timestamp", SKIPVAL); - handle.put("maxScore", SKIPVAL); + handle.put("maxScore", SKIPVAL); final ModifiableSolrParams params = new ModifiableSolrParams(); @@ -338,6 +340,16 @@ public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCas doTestDeepPivotStats(true); // just the mean price stat doTestPivotStatsFromOneShard(); + + testFacetPivotRange(); + + testFacetPivotQuery(); + + testNegativeFacetQuery(); + + testNegativeRangeQuery(); + + testPivotFacetRangeAndQuery(); } /** @@ -439,6 +451,986 @@ public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCas } } + private void testFacetPivotRange() throws Exception { + final ModifiableSolrParams params = new ModifiableSolrParams(); + setDistributedParams(params); + params.add("q", "*:*"); + params.add("facet", "true"); + params.add("facet.pivot", "{!range=s1}place_t,company_t"); + params.add("facet.range", "{!tag=s1 key=price}price_ti"); + params.add("facet.range.start", "0"); + params.add("facet.range.end", "100"); + params.add("facet.range.gap", "20"); + + QueryResponse rsp = queryServer(params); + + List expectedPlacePivots = new UnorderedEqualityArrayList(); + List expectedCardiffPivots = new UnorderedEqualityArrayList(); + expectedCardiffPivots.add(new ComparablePivotField("company_t", + "microsoft", 2, null, null, createExpectedRange("price", 0, 100, + 20, 1, 0, 0, 0, 0))); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "null", 2, + null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, 0, + 0))); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "bbc", 2, + null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, 0, + 0))); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, null, createExpectedRange("price", 0, 100, 20, 1, 1, 0, + 0, 0))); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "fujitsu", + 1, null, null, createExpectedRange("price", 0, 100, 20, 0, 0, 0, + 0, 0))); + List expectedDublinPivots = new UnorderedEqualityArrayList(); + expectedDublinPivots.add(new ComparablePivotField("company_t", "polecat", + 4, null, null, createExpectedRange("price", 0, 100, 20, 2, 1, 0, + 0, 0))); + expectedDublinPivots.add(new ComparablePivotField("company_t", "microsoft", + 4, null, null, createExpectedRange("price", 0, 100, 20, 2, 1, 0, + 0, 0))); + expectedDublinPivots.add(new ComparablePivotField("company_t", "null", 3, + null, null, createExpectedRange("price", 0, 100, 20, 1, 1, 0, 0, + 0))); + expectedDublinPivots.add(new ComparablePivotField("company_t", "fujitsu", + 2, null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, + 0, 0))); + expectedDublinPivots.add(new ComparablePivotField("company_t", "bbc", 1, + null, null, createExpectedRange("price", 0, 100, 20, 0, 0, 0, 0, + 0))); + List expectedLondonPivots = new UnorderedEqualityArrayList(); + expectedLondonPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, null, createExpectedRange("price", 0, 100, 20, 0, 2, 0, + 0, 0))); + expectedLondonPivots.add(new ComparablePivotField("company_t", "microsoft", + 2, null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, + 0, 0))); + expectedLondonPivots.add(new ComparablePivotField("company_t", "fujitsu", + 2, null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, + 0, 0))); + expectedLondonPivots.add(new ComparablePivotField("company_t", "null", 3, + null, null, createExpectedRange("price", 0, 100, 20, 0, 2, 0, 0, + 0))); + expectedLondonPivots.add(new ComparablePivotField("company_t", "bbc", 2, + null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, 0, + 0))); + List expectedLAPivots = new UnorderedEqualityArrayList(); + expectedLAPivots.add(new ComparablePivotField("company_t", "microsoft", 2, + null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, 0, + 0))); + expectedLAPivots.add(new ComparablePivotField("company_t", "fujitsu", 2, + null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, 0, + 0))); + expectedLAPivots.add(new ComparablePivotField("company_t", "null", 2, null, + null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, 0, 0))); + expectedLAPivots.add(new ComparablePivotField("company_t", "bbc", 1, null, + null, createExpectedRange("price", 0, 100, 20, 0, 0, 0, 0, 0))); + expectedLAPivots.add(new ComparablePivotField("company_t", "polecat", 2, + null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, 0, + 0))); + List expectedKrakowPivots = new UnorderedEqualityArrayList(); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "polecat", + 2, null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, + 0, 0))); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "bbc", 2, + null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, 0, + 0))); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "null", 3, + null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, 0, + 0))); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "fujitsu", + 1, null, null, createExpectedRange("price", 0, 100, 20, 0, 0, 0, + 0, 0))); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "microsoft", + 1, null, null, createExpectedRange("price", 0, 100, 20, 0, 0, 0, + 0, 0))); + List expectedCorkPivots = new UnorderedEqualityArrayList(); + expectedCorkPivots.add(new ComparablePivotField("company_t", "fujitsu", 1, + null, null, createExpectedRange("price", 0, 100, 20, 0, 0, 0, 0, + 0))); + expectedCorkPivots.add(new ComparablePivotField("company_t", "rte", 1, + null, null, createExpectedRange("price", 0, 100, 20, 0, 0, 0, 0, + 0))); + + expectedPlacePivots.add(new ComparablePivotField("place_t", "dublin", 4, + expectedDublinPivots, null, createExpectedRange("price", 0, 100, + 20, 2, 1, 0, 0, 0))); + expectedPlacePivots.add(new ComparablePivotField("place_t", "cardiff", 3, + expectedCardiffPivots, null, createExpectedRange("price", 0, 100, + 20, 1, 1, 0, 0, 0))); + expectedPlacePivots.add(new ComparablePivotField("place_t", "london", 4, + expectedLondonPivots, null, createExpectedRange("price", 0, 100, + 20, 0, 3, 0, 0, 0))); + expectedPlacePivots.add(new ComparablePivotField("place_t", "la", 3, + expectedLAPivots, null, createExpectedRange("price", 0, 100, 20, + 0, 1, 0, 0, 0))); + expectedPlacePivots.add(new ComparablePivotField("place_t", "krakow", 3, + expectedKrakowPivots, null, createExpectedRange("price", 0, 100, + 20, 0, 1, 0, 0, 0))); + expectedPlacePivots.add(new ComparablePivotField("place_t", "cork", 1, + expectedCorkPivots, null, createExpectedRange("price", 0, 100, + 20, 0, 0, 0, 0, 0))); + + List placePivots = rsp.getFacetPivot().get("place_t,company_t"); + + // Useful to check for errors, orders lists and does toString() equality + // check + testOrderedPivotsStringEquality(expectedPlacePivots, placePivots); + + assertEquals(expectedPlacePivots, placePivots); + + // Test sorting by count + + params.set(FacetParams.FACET_SORT, FacetParams.FACET_SORT_COUNT); + + rsp = queryServer(params); + + placePivots = rsp.getFacetPivot().get("place_t,company_t"); + + testCountSorting(placePivots); + + // Test limit + + params.set(FacetParams.FACET_LIMIT, 2); + + rsp = queryServer(params); + + expectedPlacePivots = new UnorderedEqualityArrayList(); + expectedDublinPivots = new UnorderedEqualityArrayList(); + expectedDublinPivots.add(new ComparablePivotField("company_t", "polecat", + 4, null, null, createExpectedRange("price", 0, 100, 20, 2, 1, 0, + 0, 0))); + expectedDublinPivots.add(new ComparablePivotField("company_t", "microsoft", + 4, null, null, createExpectedRange("price", 0, 100, 20, 2, 1, 0, + 0, 0))); + expectedLondonPivots = new UnorderedEqualityArrayList(); + expectedLondonPivots.add(new ComparablePivotField("company_t", "null", 3, + null, null, createExpectedRange("price", 0, 100, 20, 0, 2, 0, 0, + 0))); + expectedLondonPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, null, createExpectedRange("price", 0, 100, 20, 0, 2, 0, + 0, 0))); + expectedPlacePivots.add(new ComparablePivotField("place_t", "dublin", 4, + expectedDublinPivots, null, createExpectedRange("price", 0, 100, + 20, 2, 1, 0, 0, 0))); + expectedPlacePivots.add(new ComparablePivotField("place_t", "london", 4, + expectedLondonPivots, null, createExpectedRange("price", 0, 100, + 20, 0, 3, 0, 0, 0))); + + placePivots = rsp.getFacetPivot().get("place_t,company_t"); + + assertEquals(expectedPlacePivots, placePivots); + + // Test individual facet.limit values + params.remove(FacetParams.FACET_LIMIT); + + params.set("f.place_t." + FacetParams.FACET_LIMIT, 1); + params.set("f.company_t." + FacetParams.FACET_LIMIT, 4); + + rsp = queryServer(params); + + expectedPlacePivots = new UnorderedEqualityArrayList(); + + expectedDublinPivots = new UnorderedEqualityArrayList(); + expectedDublinPivots.add(new ComparablePivotField("company_t", "microsoft", + 4, null, null, createExpectedRange("price", 0, 100, 20, 2, 1, 0, + 0, 0))); + expectedDublinPivots.add(new ComparablePivotField("company_t", "polecat", + 4, null, null, createExpectedRange("price", 0, 100, 20, 2, 1, 0, + 0, 0))); + expectedDublinPivots.add(new ComparablePivotField("company_t", "null", 3, + null, null, createExpectedRange("price", 0, 100, 20, 1, 1, 0, 0, + 0))); + expectedDublinPivots.add(new ComparablePivotField("company_t", "fujitsu", + 2, null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, + 0, 0))); + + expectedLondonPivots = new UnorderedEqualityArrayList(); + expectedLondonPivots.add(new ComparablePivotField("company_t", "null", 3, + null, null, createExpectedRange("price", 0, 100, 20, 0, 2, 0, 0, + 0))); + expectedLondonPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, null, createExpectedRange("price", 0, 100, 20, 0, 2, 0, + 0, 0))); + expectedLondonPivots.add(new ComparablePivotField("company_t", "bbc", 2, + null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, 0, + 0))); + expectedLondonPivots.add(new ComparablePivotField("company_t", "fujitsu", + 2, null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, + 0, 0))); + + expectedCardiffPivots = new UnorderedEqualityArrayList(); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, null, createExpectedRange("price", 0, 100, 20, 1, 1, 0, + 0, 0))); + + expectedKrakowPivots = new UnorderedEqualityArrayList(); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "null", 3, + null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, 0, + 0))); + + expectedLAPivots = new UnorderedEqualityArrayList(); + expectedLAPivots.add(new ComparablePivotField("company_t", "fujitsu", 2, + null, null, createExpectedRange("price", 0, 100, 20, 0, 1, 0, 0, + 0))); + + expectedCorkPivots = new UnorderedEqualityArrayList(); + expectedCorkPivots.add(new ComparablePivotField("company_t", "fujitsu", 1, + null, null, createExpectedRange("price", 0, 100, 20, 0, 0, 0, 0, + 0))); + + expectedPlacePivots.add(new ComparablePivotField("place_t", "dublin", 4, + expectedDublinPivots, null, createExpectedRange("price", 0, 100, + 20, 2, 1, 0, 0, 0))); + + placePivots = rsp.getFacetPivot().get("place_t,company_t"); + assertEquals(expectedPlacePivots, placePivots); + + params.remove("f.company_t." + FacetParams.FACET_LIMIT); + params.remove("f.place_t." + FacetParams.FACET_LIMIT); + params.set(FacetParams.FACET_LIMIT, 2); + + // Test facet.missing=true with diff sorts + + index("id", 777); // NOTE: id=25 has no place as well + commit(); + + SolrParams missingA = params("q", "*:*", "rows", "0", "facet", "true", + "facet.pivot", "place_t,company_t", + // default facet.sort + FacetParams.FACET_MISSING, "true"); + SolrParams missingB = SolrParams.wrapDefaults(missingA, + params(FacetParams.FACET_LIMIT, "4", "facet.sort", "index")); + for (SolrParams p : new SolrParams[]{missingA, missingB}) { + // in either case, the last pivot option should be the same + rsp = query(p); + placePivots = rsp.getFacetPivot().get("place_t,company_t"); + assertTrue("not enough values for pivot: " + p + " => " + placePivots, + 1 < placePivots.size()); + PivotField missing = placePivots.get(placePivots.size() - 1); + assertNull("not the missing place value: " + p, missing.getValue()); + assertEquals("wrong missing place count: " + p, 2, missing.getCount()); + assertTrue("not enough sub-pivots for missing place: " + p + " => " + + missing.getPivot(), 1 < missing.getPivot().size()); + missing = missing.getPivot().get(missing.getPivot().size() - 1); + assertNull("not the missing company value: " + p, missing.getValue()); + assertEquals("wrong missing company count: " + p, 1, missing.getCount()); + assertNull("company shouldn't have sub-pivots: " + p, missing.getPivot()); + } + + // sort=index + mincount + limit + for (SolrParams variableParams : new SolrParams[]{ + // we should get the same results regardless of overrequest + params("facet.overrequest.count", "0", "facet.overrequest.ratio", "0"), + params()}) { + + SolrParams p = SolrParams.wrapDefaults( + params("q", "*:*", "rows", "0", "facet", "true", "facet.pivot", + "company_t", "facet.sort", "index", "facet.pivot.mincount", "4", + "facet.limit", "4"), variableParams); + + try { + List pivots = query(p).getFacetPivot().get("company_t"); + assertEquals(4, pivots.size()); + assertEquals("fujitsu", pivots.get(0).getValue()); + assertEquals(4, pivots.get(0).getCount()); + assertEquals("microsoft", pivots.get(1).getValue()); + assertEquals(5, pivots.get(1).getCount()); + assertEquals("null", pivots.get(2).getValue()); + assertEquals(6, pivots.get(2).getCount()); + assertEquals("polecat", pivots.get(3).getValue()); + assertEquals(6, pivots.get(3).getCount()); + + } catch (AssertionFailedError ae) { + throw new AssertionError(ae.getMessage() + " <== " + p.toString(), ae); + } + } + + // sort=index + mincount + limit + offset + for (SolrParams variableParams : new SolrParams[]{ + // we should get the same results regardless of overrequest + params("facet.overrequest.count", "0", "facet.overrequest.ratio", "0"), + params()}) { + + SolrParams p = SolrParams.wrapDefaults( + params("q", "*:*", "rows", "0", "facet", "true", "facet.pivot", + "company_t", "facet.sort", "index", "facet.pivot.mincount", "4", + "facet.offset", "1", "facet.limit", "4"), variableParams); + try { + List pivots = query(p).getFacetPivot().get("company_t"); + assertEquals(3, pivots.size()); // asked for 4, but not enough meet the + // mincount + assertEquals("microsoft", pivots.get(0).getValue()); + assertEquals(5, pivots.get(0).getCount()); + assertEquals("null", pivots.get(1).getValue()); + assertEquals(6, pivots.get(1).getCount()); + assertEquals("polecat", pivots.get(2).getValue()); + assertEquals(6, pivots.get(2).getCount()); + + } catch (AssertionFailedError ae) { + throw new AssertionError(ae.getMessage() + " <== " + p.toString(), ae); + } + + } + + // sort=index + mincount + limit + offset (more permutations) + for (SolrParams variableParams : new SolrParams[]{ + // all of these combinations should result in the same first value + params("facet.pivot.mincount", "4", "facet.offset", "2"), + params("facet.pivot.mincount", "5", "facet.offset", "1"), + params("facet.pivot.mincount", "6", "facet.offset", "0")}) { + + SolrParams p = SolrParams.wrapDefaults( + params("q", "*:*", "rows", "0", "facet", "true", "facet.limit", "1", + "facet.sort", "index", "facet.overrequest.ratio", "0", + "facet.pivot", "company_t"), variableParams); + + try { + List pivots = query(p).getFacetPivot().get("company_t"); + assertEquals(1, pivots.size()); + assertEquals(pivots.toString(), "null", pivots.get(0).getValue()); + assertEquals(pivots.toString(), 6, pivots.get(0).getCount()); + + } catch (AssertionFailedError ae) { + throw new AssertionError(ae.getMessage() + " <== " + p.toString(), ae); + } + } + } + + private void testFacetPivotQuery() throws Exception { + final ModifiableSolrParams params = new ModifiableSolrParams(); + setDistributedParams(params); + params.add("q", "*:*"); + params.add("facet", "true"); + params.add("facet.pivot", "{!query=s1}place_t,company_t"); + params.add("facet.query", "{!tag=s1 key=highPrice}price_ti:[25 TO 100]"); + + QueryResponse rsp = queryServer(params); + + List expectedPlacePivots = new UnorderedEqualityArrayList(); + List expectedCardiffPivots = new UnorderedEqualityArrayList(); + expectedCardiffPivots.add(new ComparablePivotField("company_t", + "microsoft", 2, null, createExpectedQCount( + new String[]{"highPrice"}, new int[]{0}), null)); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "null", 2, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "bbc", 2, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "fujitsu", + 1, null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{0}), null)); + List expectedDublinPivots = new UnorderedEqualityArrayList(); + expectedDublinPivots.add(new ComparablePivotField("company_t", "polecat", + 4, null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedDublinPivots.add(new ComparablePivotField("company_t", "microsoft", + 4, null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedDublinPivots.add(new ComparablePivotField("company_t", "null", 3, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedDublinPivots.add(new ComparablePivotField("company_t", "fujitsu", + 2, null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedDublinPivots.add(new ComparablePivotField("company_t", "bbc", 1, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{0}), null)); + List expectedLondonPivots = new UnorderedEqualityArrayList(); + expectedLondonPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{2}), null)); + expectedLondonPivots.add(new ComparablePivotField("company_t", "microsoft", + 2, null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedLondonPivots.add(new ComparablePivotField("company_t", "fujitsu", + 2, null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedLondonPivots.add(new ComparablePivotField("company_t", "null", 3, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{2}), null)); + expectedLondonPivots.add(new ComparablePivotField("company_t", "bbc", 2, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + List expectedLAPivots = new UnorderedEqualityArrayList(); + expectedLAPivots.add(new ComparablePivotField("company_t", "microsoft", 2, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedLAPivots.add(new ComparablePivotField("company_t", "fujitsu", 2, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedLAPivots.add(new ComparablePivotField("company_t", "null", 2, null, + createExpectedQCount(new String[]{"highPrice"}, new int[]{1}), + null)); + expectedLAPivots.add(new ComparablePivotField("company_t", "bbc", 1, null, + createExpectedQCount(new String[]{"highPrice"}, new int[]{0}), + null)); + expectedLAPivots.add(new ComparablePivotField("company_t", "polecat", 2, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + List expectedKrakowPivots = new UnorderedEqualityArrayList(); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "polecat", + 2, null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "bbc", 2, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "null", 3, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{1}), null)); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "fujitsu", + 1, null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{0}), null)); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "microsoft", + 1, null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{0}), null)); + List expectedCorkPivots = new UnorderedEqualityArrayList(); + expectedCorkPivots.add(new ComparablePivotField("company_t", "fujitsu", 1, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{0}), null)); + expectedCorkPivots.add(new ComparablePivotField("company_t", "rte", 1, + null, createExpectedQCount(new String[]{"highPrice"}, + new int[]{0}), null)); + + expectedPlacePivots.add(new ComparablePivotField("place_t", "dublin", 4, + expectedDublinPivots, createExpectedQCount( + new String[]{"highPrice"}, new int[]{1}), null)); + expectedPlacePivots.add(new ComparablePivotField("place_t", "cardiff", 3, + expectedCardiffPivots, createExpectedQCount( + new String[]{"highPrice"}, new int[]{1}), null)); + expectedPlacePivots.add(new ComparablePivotField("place_t", "london", 4, + expectedLondonPivots, createExpectedQCount( + new String[]{"highPrice"}, new int[]{3}), null)); + expectedPlacePivots.add(new ComparablePivotField("place_t", "la", 3, + expectedLAPivots, createExpectedQCount( + new String[]{"highPrice"}, new int[]{1}), null)); + expectedPlacePivots.add(new ComparablePivotField("place_t", "krakow", 3, + expectedKrakowPivots, createExpectedQCount( + new String[]{"highPrice"}, new int[]{1}), null)); + expectedPlacePivots.add(new ComparablePivotField("place_t", "cork", 1, + expectedCorkPivots, createExpectedQCount( + new String[]{"highPrice"}, new int[]{0}), null)); + + List placePivots = rsp.getFacetPivot().get("place_t,company_t"); + + // Useful to check for errors, orders lists and does toString() equality + // check + testOrderedPivotsStringEquality(expectedPlacePivots, placePivots); + + assertEquals(expectedPlacePivots, placePivots); + + // Add second query for low price + params.add("facet.query", "{!tag=s1 key=lowPrice}price_ti:[0 TO 20]"); + expectedPlacePivots = new UnorderedEqualityArrayList(); + expectedCardiffPivots = new UnorderedEqualityArrayList(); + expectedCardiffPivots.add(new ComparablePivotField("company_t", + "microsoft", 2, null, createExpectedQCount(new String[]{ + "highPrice", "lowPrice"}, new int[]{0, 1}), null)); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "null", 2, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 0}), null)); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "bbc", 2, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 0}), null)); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 1}), null)); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "fujitsu", + 1, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{0, 0}), null)); + expectedDublinPivots = new UnorderedEqualityArrayList(); + expectedDublinPivots.add(new ComparablePivotField("company_t", "polecat", + 4, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 2}), null)); + expectedDublinPivots.add(new ComparablePivotField("company_t", "microsoft", + 4, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 2}), null)); + expectedDublinPivots.add(new ComparablePivotField("company_t", "null", 3, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 1}), null)); + expectedDublinPivots.add(new ComparablePivotField("company_t", "fujitsu", + 2, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 0}), null)); + expectedDublinPivots.add(new ComparablePivotField("company_t", "bbc", 1, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{0, 0}), null)); + expectedLondonPivots = new UnorderedEqualityArrayList(); + expectedLondonPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{2, 0}), null)); + expectedLondonPivots.add(new ComparablePivotField("company_t", "microsoft", + 2, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 0}), null)); + expectedLondonPivots.add(new ComparablePivotField("company_t", "fujitsu", + 2, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 0}), null)); + expectedLondonPivots.add(new ComparablePivotField("company_t", "null", 3, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{2, 0}), null)); + expectedLondonPivots.add(new ComparablePivotField("company_t", "bbc", 2, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 0}), null)); + expectedLAPivots = new UnorderedEqualityArrayList(); + expectedLAPivots.add(new ComparablePivotField("company_t", "microsoft", 2, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 0}), null)); + expectedLAPivots.add(new ComparablePivotField("company_t", "fujitsu", 2, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 0}), null)); + expectedLAPivots.add(new ComparablePivotField("company_t", "null", 2, null, + createExpectedQCount(new String[]{"highPrice", "lowPrice"}, + new int[]{1, 0}), null)); + expectedLAPivots.add(new ComparablePivotField("company_t", "bbc", 1, null, + createExpectedQCount(new String[]{"highPrice", "lowPrice"}, + new int[]{0, 0}), null)); + expectedLAPivots.add(new ComparablePivotField("company_t", "polecat", 2, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 0}), null)); + expectedKrakowPivots = new UnorderedEqualityArrayList(); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "polecat", + 2, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 0}), null)); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "bbc", 2, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 0}), null)); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "null", 3, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 0}), null)); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "fujitsu", + 1, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{0, 0}), null)); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "microsoft", + 1, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{0, 0}), null)); + expectedCorkPivots = new UnorderedEqualityArrayList(); + expectedCorkPivots.add(new ComparablePivotField("company_t", "fujitsu", 1, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{0, 0}), null)); + expectedCorkPivots.add(new ComparablePivotField("company_t", "rte", 1, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{0, 0}), null)); + + expectedPlacePivots.add(new ComparablePivotField("place_t", "dublin", 4, + expectedDublinPivots, createExpectedQCount(new String[]{ + "highPrice", "lowPrice"}, new int[]{1, 2}), null)); + expectedPlacePivots.add(new ComparablePivotField("place_t", "cardiff", 3, + expectedCardiffPivots, createExpectedQCount(new String[]{ + "highPrice", "lowPrice"}, new int[]{1, 1}), null)); + expectedPlacePivots.add(new ComparablePivotField("place_t", "london", 4, + expectedLondonPivots, createExpectedQCount(new String[]{ + "highPrice", "lowPrice"}, new int[]{3, 0}), null)); + expectedPlacePivots.add(new ComparablePivotField("place_t", "la", 3, + expectedLAPivots, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 0}), null)); + expectedPlacePivots.add(new ComparablePivotField("place_t", "krakow", 3, + expectedKrakowPivots, createExpectedQCount(new String[]{ + "highPrice", "lowPrice"}, new int[]{1, 0}), null)); + expectedPlacePivots.add(new ComparablePivotField("place_t", "cork", 1, + expectedCorkPivots, createExpectedQCount(new String[]{ + "highPrice", "lowPrice"}, new int[]{0, 0}), null)); + + rsp = queryServer(params); + + placePivots = rsp.getFacetPivot().get("place_t,company_t"); + + // Useful to check for errors, orders lists and does toString() equality + // check + testOrderedPivotsStringEquality(expectedPlacePivots, placePivots); + assertEquals(expectedPlacePivots, placePivots); + + // Test sorting by count + + params.set(FacetParams.FACET_SORT, FacetParams.FACET_SORT_COUNT); + + rsp = queryServer(params); + + placePivots = rsp.getFacetPivot().get("place_t,company_t"); + + testCountSorting(placePivots); + + // Test limit + + params.set(FacetParams.FACET_LIMIT, 2); + + rsp = queryServer(params); + + expectedPlacePivots = new UnorderedEqualityArrayList(); + expectedDublinPivots = new UnorderedEqualityArrayList(); + expectedDublinPivots.add(new ComparablePivotField("company_t", "polecat", + 4, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 2}), null)); + expectedDublinPivots.add(new ComparablePivotField("company_t", "microsoft", + 4, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 2}), null)); + expectedLondonPivots = new UnorderedEqualityArrayList(); + expectedLondonPivots.add(new ComparablePivotField("company_t", "null", 3, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{2, 0}), null)); + expectedLondonPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{2, 0}), null)); + expectedPlacePivots.add(new ComparablePivotField("place_t", "dublin", 4, + expectedDublinPivots, createExpectedQCount(new String[]{ + "highPrice", "lowPrice"}, new int[]{1, 2}), null)); + expectedPlacePivots.add(new ComparablePivotField("place_t", "london", 4, + expectedLondonPivots, createExpectedQCount(new String[]{ + "highPrice", "lowPrice"}, new int[]{3, 0}), null)); + + placePivots = rsp.getFacetPivot().get("place_t,company_t"); + + assertEquals(expectedPlacePivots, placePivots); + + // Test individual facet.limit values + params.remove(FacetParams.FACET_LIMIT); + + params.set("f.place_t." + FacetParams.FACET_LIMIT, 1); + params.set("f.company_t." + FacetParams.FACET_LIMIT, 4); + + rsp = queryServer(params); + + expectedPlacePivots = new UnorderedEqualityArrayList(); + + expectedDublinPivots = new UnorderedEqualityArrayList(); + expectedDublinPivots.add(new ComparablePivotField("company_t", "microsoft", + 4, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 2}), null)); + expectedDublinPivots.add(new ComparablePivotField("company_t", "polecat", + 4, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 2}), null)); + expectedDublinPivots.add(new ComparablePivotField("company_t", "null", 3, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 1}), null)); + expectedDublinPivots.add(new ComparablePivotField("company_t", "fujitsu", + 2, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 0}), null)); + + expectedLondonPivots = new UnorderedEqualityArrayList(); + expectedLondonPivots.add(new ComparablePivotField("company_t", "null", 3, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{2, 0}), null)); + expectedLondonPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{2, 0}), null)); + expectedLondonPivots.add(new ComparablePivotField("company_t", "bbc", 2, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 0}), null)); + expectedLondonPivots.add(new ComparablePivotField("company_t", "fujitsu", + 2, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 0}), null)); + + expectedCardiffPivots = new UnorderedEqualityArrayList(); + expectedCardiffPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 1}), null)); + + expectedKrakowPivots = new UnorderedEqualityArrayList(); + expectedKrakowPivots.add(new ComparablePivotField("company_t", "null", 3, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 0}), null)); + + expectedLAPivots = new UnorderedEqualityArrayList(); + expectedLAPivots.add(new ComparablePivotField("company_t", "fujitsu", 2, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{1, 0}), null)); + + expectedCorkPivots = new UnorderedEqualityArrayList(); + expectedCorkPivots.add(new ComparablePivotField("company_t", "fujitsu", 1, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{0, 0}), null)); + + expectedPlacePivots.add(new ComparablePivotField("place_t", "dublin", 4, + expectedDublinPivots, createExpectedQCount(new String[]{ + "highPrice", "lowPrice"}, new int[]{1, 2}), null)); + + placePivots = rsp.getFacetPivot().get("place_t,company_t"); + assertEquals(expectedPlacePivots, placePivots); + + params.remove("f.company_t." + FacetParams.FACET_LIMIT); + params.remove("f.place_t." + FacetParams.FACET_LIMIT); + params.set(FacetParams.FACET_LIMIT, 2); + + // Test facet.missing=true with diff sorts + + index("id", 777); // NOTE: id=25 has no place as well + commit(); + + SolrParams missingA = params("q", "*:*", "rows", "0", "facet", "true", + "facet.pivot", "place_t,company_t", + // default facet.sort + FacetParams.FACET_MISSING, "true"); + SolrParams missingB = SolrParams.wrapDefaults(missingA, + params(FacetParams.FACET_LIMIT, "4", "facet.sort", "index")); + for (SolrParams p : new SolrParams[]{missingA, missingB}) { + // in either case, the last pivot option should be the same + rsp = query(p); + placePivots = rsp.getFacetPivot().get("place_t,company_t"); + assertTrue("not enough values for pivot: " + p + " => " + placePivots, + 1 < placePivots.size()); + PivotField missing = placePivots.get(placePivots.size() - 1); + assertNull("not the missing place value: " + p, missing.getValue()); + assertEquals("wrong missing place count: " + p, 2, missing.getCount()); + assertTrue("not enough sub-pivots for missing place: " + p + " => " + + missing.getPivot(), 1 < missing.getPivot().size()); + missing = missing.getPivot().get(missing.getPivot().size() - 1); + assertNull("not the missing company value: " + p, missing.getValue()); + assertEquals("wrong missing company count: " + p, 1, missing.getCount()); + assertNull("company shouldn't have sub-pivots: " + p, missing.getPivot()); + } + + // sort=index + mincount + limit + for (SolrParams variableParams : new SolrParams[]{ + // we should get the same results regardless of overrequest + params("facet.overrequest.count", "0", "facet.overrequest.ratio", "0"), + params()}) { + + SolrParams p = SolrParams.wrapDefaults( + params("q", "*:*", "rows", "0", "facet", "true", "facet.pivot", + "company_t", "facet.sort", "index", "facet.pivot.mincount", "4", + "facet.limit", "4"), variableParams); + + try { + List pivots = query(p).getFacetPivot().get("company_t"); + assertEquals(4, pivots.size()); + assertEquals("fujitsu", pivots.get(0).getValue()); + assertEquals(4, pivots.get(0).getCount()); + assertEquals("microsoft", pivots.get(1).getValue()); + assertEquals(5, pivots.get(1).getCount()); + assertEquals("null", pivots.get(2).getValue()); + assertEquals(6, pivots.get(2).getCount()); + assertEquals("polecat", pivots.get(3).getValue()); + assertEquals(6, pivots.get(3).getCount()); + + } catch (AssertionFailedError ae) { + throw new AssertionError(ae.getMessage() + " <== " + p.toString(), ae); + } + } + + // sort=index + mincount + limit + offset + for (SolrParams variableParams : new SolrParams[]{ + // we should get the same results regardless of overrequest + params("facet.overrequest.count", "0", "facet.overrequest.ratio", "0"), + params()}) { + + SolrParams p = SolrParams.wrapDefaults( + params("q", "*:*", "rows", "0", "facet", "true", "facet.pivot", + "company_t", "facet.sort", "index", "facet.pivot.mincount", "4", + "facet.offset", "1", "facet.limit", "4"), variableParams); + try { + List pivots = query(p).getFacetPivot().get("company_t"); + assertEquals(3, pivots.size()); // asked for 4, but not enough meet the + // mincount + assertEquals("microsoft", pivots.get(0).getValue()); + assertEquals(5, pivots.get(0).getCount()); + assertEquals("null", pivots.get(1).getValue()); + assertEquals(6, pivots.get(1).getCount()); + assertEquals("polecat", pivots.get(2).getValue()); + assertEquals(6, pivots.get(2).getCount()); + + } catch (AssertionFailedError ae) { + throw new AssertionError(ae.getMessage() + " <== " + p.toString(), ae); + } + + } + + // sort=index + mincount + limit + offset (more permutations) + for (SolrParams variableParams : new SolrParams[]{ + // all of these combinations should result in the same first value + params("facet.pivot.mincount", "4", "facet.offset", "2"), + params("facet.pivot.mincount", "5", "facet.offset", "1"), + params("facet.pivot.mincount", "6", "facet.offset", "0")}) { + + SolrParams p = SolrParams.wrapDefaults( + params("q", "*:*", "rows", "0", "facet", "true", "facet.limit", "1", + "facet.sort", "index", "facet.overrequest.ratio", "0", + "facet.pivot", "company_t"), variableParams); + + try { + List pivots = query(p).getFacetPivot().get("company_t"); + assertEquals(1, pivots.size()); + assertEquals(pivots.toString(), "null", pivots.get(0).getValue()); + assertEquals(pivots.toString(), 6, pivots.get(0).getCount()); + + } catch (AssertionFailedError ae) { + throw new AssertionError(ae.getMessage() + " <== " + p.toString(), ae); + } + } + } + + private void testPivotFacetRangeAndQuery() throws Exception { + SolrParams params = params("q", "*:*", + "rows", "0", + "facet", "true", + "stats", "true", + "facet.pivot", "{!range=s1 query=s2 stats=s3}place_t,company_t", + "facet.range", "{!tag=s1 key=price}price_ti", + "facet.query", "{!tag=s2 key=highPrice}price_ti:[25 TO 100]", + "facet.query", "{!tag=s2 key=lowPrice}price_ti:[0 TO 20]", + "stats.field", ("{!tag=s3 key=avg_price}price_ti"), + "facet.range.start", "0", + "facet.range.end", "100", + "facet.range.gap", "20", + FacetParams.FACET_SORT, FacetParams.FACET_SORT_COUNT, + FacetParams.FACET_LIMIT, "2"); + + UnorderedEqualityArrayList expectedPlacePivots = new UnorderedEqualityArrayList<>(); + UnorderedEqualityArrayList expectedDublinPivots = new UnorderedEqualityArrayList<>(); + expectedDublinPivots.add(new ComparablePivotField("company_t", "polecat", + 4, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 2}), createExpectedRange("price", 0, 100, 20, 2, 1, 0, + 0, 0))); + expectedDublinPivots.add(new ComparablePivotField("company_t", "microsoft", + 4, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{1, 2}), createExpectedRange("price", 0, 100, 20, 2, 1, 0, + 0, 0))); + UnorderedEqualityArrayList expectedLondonPivots = new UnorderedEqualityArrayList<>(); + expectedLondonPivots.add(new ComparablePivotField("company_t", "null", 3, + null, createExpectedQCount( + new String[]{"highPrice", "lowPrice"}, new int[]{2, 0}), createExpectedRange("price", 0, 100, 20, 0, 2, 0, 0, + 0))); + expectedLondonPivots.add(new ComparablePivotField("company_t", "polecat", + 3, null, createExpectedQCount(new String[]{"highPrice", + "lowPrice"}, new int[]{2, 0}), createExpectedRange("price", 0, 100, 20, 0, 2, 0, + 0, 0))); + expectedPlacePivots.add(new ComparablePivotField("place_t", "dublin", 4, + expectedDublinPivots, createExpectedQCount(new String[]{ + "highPrice", "lowPrice"}, new int[]{1, 2}), createExpectedRange("price", 0, 100, + 20, 2, 1, 0, 0, 0))); + expectedPlacePivots.add(new ComparablePivotField("place_t", "london", 4, + expectedLondonPivots, createExpectedQCount(new String[]{ + "highPrice", "lowPrice"}, new int[]{3, 0}), createExpectedRange("price", 0, 100, + 20, 0, 3, 0, 0, 0))); + + QueryResponse rsp = query(params); + List placePivots = rsp.getFacetPivot().get("place_t,company_t"); + assertEquals(expectedPlacePivots, placePivots); + + PivotField dublinPivotField = placePivots.get(0); + assertEquals("dublin", dublinPivotField.getValue()); + assertEquals(4, dublinPivotField.getCount()); + + PivotField microsoftPivotField = dublinPivotField.getPivot().get(0); + assertEquals("microsoft", microsoftPivotField.getValue()); + assertEquals(4, microsoftPivotField.getCount()); + + FieldStatsInfo dublinMicrosoftStatsInfo = microsoftPivotField.getFieldStatsInfo().get("avg_price"); + assertEquals(21.0, (double) dublinMicrosoftStatsInfo.getMean(), 0.1E-7); + assertEquals(15.0, dublinMicrosoftStatsInfo.getMin()); + assertEquals(29.0, dublinMicrosoftStatsInfo.getMax()); + assertEquals(3, (long) dublinMicrosoftStatsInfo.getCount()); + assertEquals(1, (long) dublinMicrosoftStatsInfo.getMissing()); + assertEquals(63.0, dublinMicrosoftStatsInfo.getSum()); + assertEquals(1427.0, dublinMicrosoftStatsInfo.getSumOfSquares(), 0.1E-7); + assertEquals(7.211102550927978, dublinMicrosoftStatsInfo.getStddev(), 0.1E-7); + } + + private void testNegativeFacetQuery() throws Exception { + // this should not hang facet.query under the pivot + SolrParams params = params("q", "*:*", + "rows", "0", + "stats", "true", + "facet.query", "{!tag=ttt}price_ti:[25 TO 100]", + "facet", "true", + "facet.pivot", "{!query=t}place_t,company_t"); + QueryResponse rsp = query(params); + + assertNullFacetTypeInsidePivot(FacetParams.FACET_QUERY, rsp.getFacetPivot().get("place_t,company_t")); + + params = params("q", "*:*", + "rows", "0", + "stats", "true", + "facet", "true", + "facet.pivot", "{!query=t}place_t,company_t"); + rsp = query(params); + assertNullFacetTypeInsidePivot(FacetParams.FACET_QUERY, rsp.getFacetPivot().get("place_t,company_t")); + + params = params("q", "*:*", + "rows", "0", + "facet.query", "{!tag=t}price_ti:[25 TO 100]", + "hang", "", // empty + "facet", "true", + "facet.pivot", "{!query=$hang}place_t,company_t"); + rsp = query(params); + assertNullFacetTypeInsidePivot(FacetParams.FACET_QUERY, rsp.getFacetPivot().get("place_t,company_t")); + + params = params("q", "*:*", + "rows", "0", + "facet.query", "{!tag=t}price_ti:[25 TO 100]", + "hang", "price_ti:[0 TO 20]", // with a query + "facet", "true", + "facet.pivot", "{!query=$hang}place_t,company_t"); + rsp = query(params); + // we aren't going to start calculating facet query unless the query is specified with a 'facet.query' param + // hence hanging an arbitrary query shouldn't work + assertNullFacetTypeInsidePivot(FacetParams.FACET_QUERY, rsp.getFacetPivot().get("place_t,company_t")); + } + + private void testNegativeRangeQuery() throws Exception { + SolrParams params = params("q", "*:*", + "rows", "0", + "stats", "true", + "facet.range", "{!tag=s1 key=price}price_ti", + "facet", "true", + "facet.pivot", "{!range=s}place_t,company_t", + "facet.range.start", "0", + "facet.range.end", "100", + "facet.range.gap", "20"); + QueryResponse rsp = query(params); + assertNullFacetTypeInsidePivot(FacetParams.FACET_RANGE, rsp.getFacetPivot().get("place_t,company_t")); + + params = params("q", "*:*", + "rows", "0", + "stats", "true", + "facet.range", "{!tag=s1 key=price}price_ti", + "facet", "true", + "hang", "", // empty! + "facet.pivot", "{!range=$hang}place_t,company_t", + "facet.range.start", "0", + "facet.range.end", "100", + "facet.range.gap", "20"); + rsp = query(params); + assertNullFacetTypeInsidePivot(FacetParams.FACET_RANGE, rsp.getFacetPivot().get("place_t,company_t")); + + params = params("q", "*:*", + "rows", "0", + "stats", "true", + "facet.range", "{!tag=s1 key=price}price_ti", + "facet", "true", + "hang", "price_ti", + "facet.pivot", "{!range=$hang}place_t,company_t", + "facet.range.start", "0", + "facet.range.end", "100", + "facet.range.gap", "20"); + rsp = query(params); + assertNullFacetTypeInsidePivot(FacetParams.FACET_RANGE, rsp.getFacetPivot().get("place_t,company_t")); + } + + private Map createExpectedQCount(String[] keys, int[] counts) { + Map expectedQCounts = new LinkedHashMap<>(); + for (int idx = 0; idx < keys.length; idx++) { + expectedQCounts.put(keys[idx], counts[idx]); + } + return expectedQCounts; + } + + private void assertNullFacetTypeInsidePivot(String facetType, List pivots) { + for (PivotField pivot : pivots) { + if (facetType == FacetParams.FACET_QUERY) { + assertNull("pivot=" + pivot + " facetType=" + facetType + + " should've been null. Found: " + pivot.getFacetQuery(), pivot.getFacetQuery()); + } else if (facetType == FacetParams.FACET_RANGE) { + assertNull("pivot=" + pivot + " facetType=" + facetType + + " should've been null. Found: " + pivot.getFacetRanges(), pivot.getFacetRanges()); + } + + if (pivot.getPivot() != null) { + assertNullFacetTypeInsidePivot(facetType, pivot.getPivot()); + } + } + } + // Useful to check for errors, orders lists and does toString() equality check private void testOrderedPivotsStringEquality( List expectedPlacePivots, List placePivots) { @@ -526,11 +1518,31 @@ public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCas assertEquals(msg + " stats max", val, stats.getMax()); } - public static class ComparablePivotField extends PivotField { - + private List createExpectedRange(String key, int start, int end, + int gap, int... values) { + List expectedRanges = new ArrayList<>(); + RangeFacet expectedPrices = new RangeFacet.Numeric(key, start, end, gap, null, null, null); + expectedRanges.add(expectedPrices); + int idx = 0; + for (int range = start; range < end; range += gap) { + expectedPrices.addCount(String.valueOf(range), values[idx]); + if (idx < values.length) { + idx++; + } + } + return expectedRanges; + } - public ComparablePivotField(String f, Object v, int count, List pivot) { - super(f,v,count,pivot, null); + public static class ComparablePivotField extends PivotField { + + public ComparablePivotField(String f, Object v, int count, + List pivot, Map queryCounts, List ranges) { + super(f, v, count, pivot, null, queryCounts, ranges); + } + + public ComparablePivotField(String f, Object v, int count, + List pivot) { + super(f, v, count, pivot, null, null, null); } @Override @@ -549,6 +1561,44 @@ public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCas if (getValue() == null) { if (other.getValue() != null) return false; } else if (!getValue().equals(other.getValue())) return false; + if (getFacetRanges() == null) { + if (other.getFacetRanges() != null) return false; + } else { + if (getFacetRanges().size() != other.getFacetRanges().size()) return false; + for (RangeFacet entry : getFacetRanges()) { + boolean found = false; + for (RangeFacet otherRange : other.getFacetRanges()) { + if (otherRange.getName().equals(entry.getName())) { + found = true; + + if (!entry.getGap().equals(otherRange.getGap())) return false; + if (!entry.getStart().equals(otherRange.getStart())) return false; + if (!entry.getEnd().equals(otherRange.getEnd())) return false; + + List myCounts = entry.getCounts(); + List otherRangeCounts = otherRange.getCounts(); + if ( (myCounts == null && otherRangeCounts != null) + || (myCounts != null && otherRangeCounts == null) + || (myCounts.size() != otherRangeCounts.size())) return false; + + for (int i=0; i entry : getFacetQuery().entrySet()) { + Integer otherQCount = other.getFacetQuery().get(entry.getKey()); + if (otherQCount == null || otherQCount != entry.getValue()) return false; + } + } return true; } } @@ -564,7 +1614,7 @@ public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCas equal = true; for (Object objectInOtherList : otherList) { if (!contains(objectInOtherList)) { - equal = false; + return false; } } } @@ -582,7 +1632,7 @@ public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCas } } - public class PivotFieldComparator implements Comparator { + public static class PivotFieldComparator implements Comparator { @Override public int compare(PivotField o1, PivotField o2) { @@ -591,6 +1641,33 @@ public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCas if (compare == 0) { compare = ((String) o2.getValue()).compareTo((String) o1.getValue()); } + if (compare == 0) { + for (Map.Entry entry : o1.getFacetQuery().entrySet()) { + compare = entry.getValue().compareTo( + o2.getFacetQuery().get(entry.getKey())); + if (compare != 0) { + break; + } + } + if (compare == 0) { + compare = Integer.valueOf(o1.getFacetQuery().size()).compareTo( + o2.getFacetQuery().size()); + } + } + if (compare == 0) { + for (RangeFacet entry : o1.getFacetRanges()) { + boolean found = false; + for (RangeFacet otherRangeFacet : o2.getFacetRanges()) { + if (otherRangeFacet.getName().equals(entry.getName())) { + found = true; + } + } + if (!found) { + compare = 1; + break; + } + } + } return compare; } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/PivotField.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/PivotField.java index 3b084f6bddf..6c82b6f3350 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/PivotField.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/PivotField.java @@ -22,6 +22,8 @@ import java.io.Serializable; import java.util.List; import java.util.Map; +import org.apache.solr.common.util.NamedList; + public class PivotField implements Serializable { final String _field; @@ -29,22 +31,26 @@ public class PivotField implements Serializable final int _count; final List _pivot; final Map _statsInfo; + final Map _querycounts; + final List _ranges; /** - * @deprecated Use {@link #PivotField(String,Object,int,List,Map)} with a null statsInfo + * @deprecated Use {@link #PivotField(String,Object,int,List,Map,Map,List)} with null statsInfo, queryCounts and ranges */ @Deprecated public PivotField( String f, Object v, int count, List pivot) { - this(f, v, count, pivot, null); + this(f, v, count, pivot, null, null, null); } - public PivotField( String f, Object v, int count, List pivot, Map statsInfo) + public PivotField( String f, Object v, int count, List pivot, Map statsInfo, Map queryCounts, List ranges) { _field = f; _value = v; _count = count; _pivot = pivot; _statsInfo = statsInfo; + _querycounts= queryCounts; + _ranges= ranges; } public String getField() { @@ -67,6 +73,14 @@ public class PivotField implements Serializable return _statsInfo; } + public Map getFacetQuery() { + return _querycounts; + } + + public List getFacetRanges() { + return _ranges; + } + @Override public String toString() { @@ -88,6 +102,12 @@ public class PivotField implements Serializable out.print("]"); } out.println(); + if(_querycounts != null) { + out.println(_querycounts.toString()); + } + if(_ranges != null) { + out.println(_ranges.toString()); + } if( _pivot != null ) { for( PivotField p : _pivot ) { p.write( out, indent+1 ); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java index fb7c7ee93e8..de596203c6f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java @@ -348,41 +348,7 @@ public class QueryResponse extends SolrResponseBase //Parse range facets NamedList> rf = (NamedList>) info.get("facet_ranges"); if (rf != null) { - _facetRanges = new ArrayList<>( rf.size() ); - for (Map.Entry> facet : rf) { - NamedList values = facet.getValue(); - Object rawGap = values.get("gap"); - - RangeFacet rangeFacet; - if (rawGap instanceof Number) { - Number gap = (Number) rawGap; - Number start = (Number) values.get("start"); - Number end = (Number) values.get("end"); - - Number before = (Number) values.get("before"); - Number after = (Number) values.get("after"); - Number between = (Number) values.get("between"); - - rangeFacet = new RangeFacet.Numeric(facet.getKey(), start, end, gap, before, after, between); - } else { - String gap = (String) rawGap; - Date start = (Date) values.get("start"); - Date end = (Date) values.get("end"); - - Number before = (Number) values.get("before"); - Number after = (Number) values.get("after"); - Number between = (Number) values.get("between"); - - rangeFacet = new RangeFacet.Date(facet.getKey(), start, end, gap, before, after, between); - } - - NamedList counts = (NamedList) values.get("counts"); - for (Map.Entry entry : counts) { - rangeFacet.addCount(entry.getKey(), entry.getValue()); - } - - _facetRanges.add(rangeFacet); - } + _facetRanges = extractRangeFacets(rf); } //Parse pivot facets @@ -408,7 +374,47 @@ public class QueryResponse extends SolrResponseBase } } } - + + private List extractRangeFacets(NamedList> rf) { + List facetRanges = new ArrayList<>( rf.size() ); + + for (Map.Entry> facet : rf) { + NamedList values = facet.getValue(); + Object rawGap = values.get("gap"); + + RangeFacet rangeFacet; + if (rawGap instanceof Number) { + Number gap = (Number) rawGap; + Number start = (Number) values.get("start"); + Number end = (Number) values.get("end"); + + Number before = (Number) values.get("before"); + Number after = (Number) values.get("after"); + Number between = (Number) values.get("between"); + + rangeFacet = new RangeFacet.Numeric(facet.getKey(), start, end, gap, before, after, between); + } else { + String gap = (String) rawGap; + Date start = (Date) values.get("start"); + Date end = (Date) values.get("end"); + + Number before = (Number) values.get("before"); + Number after = (Number) values.get("after"); + Number between = (Number) values.get("between"); + + rangeFacet = new RangeFacet.Date(facet.getKey(), start, end, gap, before, after, between); + } + + NamedList counts = (NamedList) values.get("counts"); + for (Map.Entry entry : counts) { + rangeFacet.addCount(entry.getKey(), entry.getValue()); + } + + facetRanges.add(rangeFacet); + } + return facetRanges; + } + protected List readPivots( List list ) { ArrayList values = new ArrayList<>( list.size() ); @@ -423,6 +429,8 @@ public class QueryResponse extends SolrResponseBase List subPivots = null; Map fieldStatsInfos = null; + Map queryCounts = null; + List ranges = null; if (4 <= nl.size()) { for(int index = 3; index < nl.size(); index++) { @@ -444,6 +452,21 @@ public class QueryResponse extends SolrResponseBase fieldStatsInfos = extractFieldStatsInfo((NamedList) val); break; } + case "queries": { + // Parse the queries + queryCounts = new LinkedHashMap<>(); + NamedList fq = (NamedList) val; + if (fq != null) { + for( Map.Entry entry : fq ) { + queryCounts.put( entry.getKey(), entry.getValue() ); + } + } + break; + } + case "ranges": { + ranges = extractRangeFacets((NamedList>) val); + break; + } default: throw new RuntimeException( "unknown key in pivot: "+ key+ " ["+val+"]"); @@ -451,7 +474,7 @@ public class QueryResponse extends SolrResponseBase } } - values.add( new PivotField( f, v, cnt, subPivots, fieldStatsInfos ) ); + values.add( new PivotField( f, v, cnt, subPivots, fieldStatsInfos, queryCounts, ranges ) ); } return values; } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java index 7641a2fa2cf..4840cb31a6a 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java @@ -42,7 +42,9 @@ import org.apache.solr.client.solrj.response.FieldStatsInfo; import org.apache.solr.client.solrj.response.LukeResponse; import org.apache.solr.client.solrj.response.PivotField; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.RangeFacet; import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.client.solrj.response.RangeFacet.Count; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; @@ -65,6 +67,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Random; @@ -1094,6 +1097,218 @@ abstract public class SolrExampleTests extends SolrExampleTestsBase } + @Test + public void testPivotFacetsQueries() throws Exception { + SolrClient client = getSolrClient(); + + // Empty the database... + client.deleteByQuery("*:*");// delete everything! + client.commit(); + assertNumFound("*:*", 0); // make sure it got in + + int id = 1; + ArrayList docs = new ArrayList<>(); + docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "a", "inStock", true, "popularity", 12, "price", .017)); + docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "a", "inStock", false, "popularity", 13, "price", 16.04)); + docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "a", "inStock", true, "popularity", 14, "price", 12.34)); + docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "b", "inStock", false, "popularity", 24, "price", 51.39)); + docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "b", "inStock", true, "popularity", 28, "price", 131.39)); + docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "a", "inStock", false, "popularity", 32)); + docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "a", "inStock", true, "popularity", 31, "price", 131.39)); + docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", false, "popularity", 36)); + docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", true, "popularity", 37, "price", 1.39)); + docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", false, "popularity", 38, "price", 47.98)); + docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", true, "popularity", -38)); + docs.add(makeTestDoc("id", id++, "cat", "b")); // something not matching all fields + client.add(docs); + client.commit(); + + SolrQuery query = new SolrQuery("*:*"); + query.addFacetPivotField("{!query=s1}features,manu"); + query.addFacetQuery("{!key=highPrice tag=s1}price:[100 TO *]"); + query.addFacetQuery("{!tag=s1 key=lowPrice}price:[0 TO 50]"); + query.setFacetMinCount(0); + query.setRows(0); + QueryResponse rsp = client.query(query); + + Map map = rsp.getFacetQuery(); + assertEquals(2, map.get("highPrice").intValue()); + assertEquals(5, map.get("lowPrice").intValue()); + + NamedList> pivots = rsp.getFacetPivot(); + List pivotValues = pivots.get("features,manu"); + + PivotField featuresBBBPivot = pivotValues.get(0); + assertEquals("features", featuresBBBPivot.getField()); + assertEquals("bbb", featuresBBBPivot.getValue()); + assertNotNull(featuresBBBPivot.getFacetQuery()); + assertEquals(2, featuresBBBPivot.getFacetQuery().size()); + assertEquals(1, featuresBBBPivot.getFacetQuery().get("highPrice").intValue()); + assertEquals(2, featuresBBBPivot.getFacetQuery().get("lowPrice").intValue()); + + PivotField featuresAAAPivot = pivotValues.get(1); + assertEquals("features", featuresAAAPivot.getField()); + assertEquals("aaa", featuresAAAPivot.getValue()); + assertNotNull(featuresAAAPivot.getFacetQuery()); + assertEquals(2, featuresAAAPivot.getFacetQuery().size()); + assertEquals(1, featuresAAAPivot.getFacetQuery().get("highPrice").intValue()); + assertEquals(3, featuresAAAPivot.getFacetQuery().get("lowPrice").intValue()); + } + + @Test + public void testPivotFacetsRanges() throws Exception { + SolrClient client = getSolrClient(); + + // Empty the database... + client.deleteByQuery("*:*");// delete everything! + client.commit(); + assertNumFound("*:*", 0); // make sure it got in + + int id = 1; + ArrayList docs = new ArrayList<>(); + docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "a", "inStock", true, "popularity", 12, "price", .017)); + docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "a", "inStock", false, "popularity", 13, "price", 16.04)); + docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "a", "inStock", true, "popularity", 14, "price", 12.34)); + docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "b", "inStock", false, "popularity", 24, "price", 51.39)); + docs.add(makeTestDoc("id", id++, "features", "aaa", "cat", "b", "inStock", true, "popularity", 28, "price", 131.39)); + docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "a", "inStock", false, "popularity", 32)); + docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "a", "inStock", true, "popularity", 31, "price", 131.39)); + docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", false, "popularity", 36)); + docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", true, "popularity", 37, "price", 1.39)); + docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", false, "popularity", 38, "price", 47.98)); + docs.add(makeTestDoc("id", id++, "features", "bbb", "cat", "b", "inStock", true, "popularity", -38)); + docs.add(makeTestDoc("id", id++, "cat", "b")); // something not matching all fields + client.add(docs); + client.commit(); + + SolrQuery query = new SolrQuery("*:*"); + query.addFacetPivotField("{!range=s1}features,manu"); + query.add(FacetParams.FACET_RANGE, "{!key=price1 tag=s1}price"); + query.add(String.format(Locale.ROOT, "f.%s.%s", "price", FacetParams.FACET_RANGE_START), "0"); + query.add(String.format(Locale.ROOT, "f.%s.%s", "price", FacetParams.FACET_RANGE_END), "200"); + query.add(String.format(Locale.ROOT, "f.%s.%s", "price", FacetParams.FACET_RANGE_GAP), "50"); + query.set(FacetParams.FACET, true); + query.add(FacetParams.FACET_RANGE, "{!key=price2 tag=s1}price"); + query.setFacetMinCount(0); + query.setRows(0); + QueryResponse rsp = client.query(query); + + List list = rsp.getFacetRanges(); + assertEquals(2, list.size()); + @SuppressWarnings("unchecked") + RangeFacet range1 = list.get(0); + assertEquals("price1", range1.getName()); + assertEquals(0, range1.getStart().intValue()); + assertEquals(200, range1.getEnd().intValue()); + assertEquals(50, range1.getGap().intValue()); + List counts1 = range1.getCounts(); + assertEquals(4, counts1.size()); + assertEquals(5, counts1.get(0).getCount()); + assertEquals("0.0", counts1.get(0).getValue()); + assertEquals(1, counts1.get(1).getCount()); + assertEquals("50.0", counts1.get(1).getValue()); + assertEquals(2, counts1.get(2).getCount()); + assertEquals("100.0", counts1.get(2).getValue()); + assertEquals(0, counts1.get(3).getCount()); + assertEquals("150.0", counts1.get(3).getValue()); + @SuppressWarnings("unchecked") + RangeFacet range2 = list.get(1); + assertEquals("price2", range2.getName()); + assertEquals(0, range2.getStart().intValue()); + assertEquals(200, range2.getEnd().intValue()); + assertEquals(50, range2.getGap().intValue()); + List counts2 = range2.getCounts(); + assertEquals(4, counts2.size()); + assertEquals(5, counts2.get(0).getCount()); + assertEquals("0.0", counts2.get(0).getValue()); + assertEquals(1, counts2.get(1).getCount()); + assertEquals("50.0", counts2.get(1).getValue()); + assertEquals(2, counts2.get(2).getCount()); + assertEquals("100.0", counts2.get(2).getValue()); + assertEquals(0, counts2.get(3).getCount()); + assertEquals("150.0", counts2.get(3).getValue()); + + NamedList> pivots = rsp.getFacetPivot(); + List pivotValues = pivots.get("features,manu"); + + PivotField featuresBBBPivot = pivotValues.get(0); + assertEquals("features", featuresBBBPivot.getField()); + assertEquals("bbb", featuresBBBPivot.getValue()); + List featuresBBBRanges = featuresBBBPivot.getFacetRanges(); + + for (RangeFacet range : featuresBBBRanges) { + if (range.getName().equals("price1")) { + assertNotNull(range); + assertEquals(0, ((Float)range.getStart()).intValue()); + assertEquals(200, ((Float)range.getEnd()).intValue()); + assertEquals(50, ((Float)range.getGap()).intValue()); + List counts = range.getCounts(); + assertEquals(4, counts.size()); + for (Count count : counts) { + switch (count.getValue()) { + case "0.0": assertEquals(2, count.getCount()); break; + case "50.0": assertEquals(0, count.getCount()); break; + case "100.0": assertEquals(1, count.getCount()); break; + case "150.0": assertEquals(0, count.getCount()); break; + } + } + } else if (range.getName().equals("price2")) { + assertNotNull(range); + assertEquals(0, ((Float) range.getStart()).intValue()); + assertEquals(200, ((Float) range.getEnd()).intValue()); + assertEquals(50, ((Float) range.getGap()).intValue()); + List counts = range.getCounts(); + assertEquals(4, counts.size()); + for (Count count : counts) { + switch (count.getValue()) { + case "0.0": assertEquals(2, count.getCount()); break; + case "50.0": assertEquals(0, count.getCount()); break; + case "100.0": assertEquals(1, count.getCount()); break; + case "150.0": assertEquals(0, count.getCount()); break; + } + } + } + } + + PivotField featuresAAAPivot = pivotValues.get(1); + assertEquals("features", featuresAAAPivot.getField()); + assertEquals("aaa", featuresAAAPivot.getValue()); + List facetRanges = featuresAAAPivot.getFacetRanges(); + for (RangeFacet range : facetRanges) { + if (range.getName().equals("price1")) { + assertNotNull(range); + assertEquals(0, ((Float)range.getStart()).intValue()); + assertEquals(200, ((Float)range.getEnd()).intValue()); + assertEquals(50, ((Float)range.getGap()).intValue()); + List counts = range.getCounts(); + assertEquals(4, counts.size()); + for (Count count : counts) { + switch (count.getValue()) { + case "0.0": assertEquals(3, count.getCount()); break; + case "50.0": assertEquals(1, count.getCount()); break; + case "100.0": assertEquals(1, count.getCount()); break; + case "150.0": assertEquals(0, count.getCount()); break; + } + } + } else if (range.getName().equals("price2")) { + assertNotNull(range); + assertEquals(0, ((Float)range.getStart()).intValue()); + assertEquals(200, ((Float)range.getEnd()).intValue()); + assertEquals(50, ((Float)range.getGap()).intValue()); + List counts = range.getCounts(); + assertEquals(4, counts.size()); + for (Count count : counts) { + switch (count.getValue()) { + case "0.0": assertEquals(3, count.getCount()); break; + case "50.0": assertEquals(1, count.getCount()); break; + case "100.0": assertEquals(1, count.getCount()); break; + case "150.0": assertEquals(0, count.getCount()); break; + } + } + } + } + } + public void testPivotFacetsMissing() throws Exception { doPivotFacetTest(true); } @@ -1870,4 +2085,4 @@ abstract public class SolrExampleTests extends SolrExampleTestsBase } return sdoc; } -} \ No newline at end of file +}