mirror of https://github.com/apache/lucene.git
SOLR-7005: New facet.heatmap on spatial RPT fields
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1658614 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
David Wayne Smiley
parent
f3279129a6
commit
ac50da1613
|
|
@ -101,10 +101,13 @@ New Features
|
|||
for setting 'highlight' and 'allTermsRequired' in the suggester configuration.
|
||||
(Boon Low, Varun Thacker via Tomás Fernández Löbbe)
|
||||
|
||||
* SOLR-7083:Support managing all named components in solrconfig such as
|
||||
* SOLR-7083: Support managing all named components in solrconfig such as
|
||||
requestHandler, queryParser, queryResponseWriter, valueSourceParser,
|
||||
transformer, queryConverter (Noble Paul)
|
||||
|
||||
* SOLR-7005: Spatial 2D heatmap faceting on RPT fields via new facet.heatmap with PNG and
|
||||
2D int array formats. (David Smiley)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -51,8 +51,7 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* TODO!
|
||||
*
|
||||
* Computes facets -- aggregations with counts of terms or ranges over the whole search results.
|
||||
*
|
||||
* @since solr 1.3
|
||||
*/
|
||||
|
|
@ -66,9 +65,9 @@ public class FacetComponent extends SearchComponent {
|
|||
private static final String PIVOT_REFINE_PREFIX = "{!"+PivotFacet.REFINE_PARAM+"=";
|
||||
|
||||
/**
|
||||
* incrememented counter used to track the values being refined in a given request.
|
||||
* Incremented counter used to track the values being refined in a given request.
|
||||
* This counter is used in conjunction with {@link PivotFacet#REFINE_PARAM} to identify
|
||||
* which refinement values are associated with which pivots
|
||||
* which refinement values are associated with which pivots.
|
||||
*/
|
||||
int pivotRefinementCounter = 0;
|
||||
|
||||
|
|
@ -214,6 +213,7 @@ public class FacetComponent extends SearchComponent {
|
|||
shardsRefineRequest.params.set(FacetParams.FACET, "true");
|
||||
shardsRefineRequest.params.remove(FacetParams.FACET_FIELD);
|
||||
shardsRefineRequest.params.remove(FacetParams.FACET_QUERY);
|
||||
//TODO remove interval faceting, and ranges and heatmap too?
|
||||
|
||||
for (int i = 0; i < distribFieldFacetRefinements.size();) {
|
||||
String facetCommand = distribFieldFacetRefinements.get(i++);
|
||||
|
|
@ -318,6 +318,8 @@ public class FacetComponent extends SearchComponent {
|
|||
modifyRequestForRangeFacets(sreq, fi);
|
||||
|
||||
modifyRequestForPivotFacets(rb, sreq, fi.pivotFacets);
|
||||
|
||||
SpatialHeatmapFacets.distribModifyRequest(sreq, fi.heatmapFacets);
|
||||
|
||||
sreq.params.remove(FacetParams.FACET_MINCOUNT);
|
||||
sreq.params.remove(FacetParams.FACET_OFFSET);
|
||||
|
|
@ -332,17 +334,11 @@ public class FacetComponent extends SearchComponent {
|
|||
// we must get all the range buckets back in order to have coherent lists at the end, see SOLR-6154
|
||||
private void modifyRequestForRangeFacets(ShardRequest sreq, FacetInfo fi) {
|
||||
// Collect all the range fields.
|
||||
if (sreq.params.getParams(FacetParams.FACET_RANGE) == null) {
|
||||
return;
|
||||
}
|
||||
List<String> rangeFields = new ArrayList<>();
|
||||
for (String field : sreq.params.getParams(FacetParams.FACET_RANGE)) {
|
||||
rangeFields.add(field);
|
||||
}
|
||||
|
||||
for (String field : rangeFields) {
|
||||
sreq.params.remove("f." + field + ".facet.mincount");
|
||||
sreq.params.add("f." + field + ".facet.mincount", "0");
|
||||
final String[] fields = sreq.params.getParams(FacetParams.FACET_RANGE);
|
||||
if (fields != null) {
|
||||
for (String field : fields) {
|
||||
sreq.params.set("f." + field + ".facet.mincount", "0");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -550,6 +546,9 @@ public class FacetComponent extends SearchComponent {
|
|||
// refinement reqs still needed (below) once we've considered every shard
|
||||
doDistribPivots(rb, shardNum, facet_counts);
|
||||
|
||||
// Distributed facet_heatmaps
|
||||
SpatialHeatmapFacets.distribHandleResponse(fi.heatmapFacets, facet_counts);
|
||||
|
||||
} // end for-each-response-in-shard-request...
|
||||
|
||||
// refine each pivot based on the new shard data
|
||||
|
|
@ -1046,6 +1045,8 @@ public class FacetComponent extends SearchComponent {
|
|||
facet_counts.add("facet_dates", fi.dateFacets);
|
||||
facet_counts.add("facet_ranges", fi.rangeFacets);
|
||||
facet_counts.add("facet_intervals", fi.intervalFacets);
|
||||
facet_counts.add(SpatialHeatmapFacets.RESPONSE_KEY,
|
||||
SpatialHeatmapFacets.distribFinish(fi.heatmapFacets, rb));
|
||||
|
||||
if (fi.pivotFacets != null && fi.pivotFacets.size() > 0) {
|
||||
facet_counts.add(PIVOT_KEY, createPivotFacetOutput(rb));
|
||||
|
|
@ -1112,6 +1113,7 @@ public class FacetComponent extends SearchComponent {
|
|||
= new SimpleOrderedMap<>();
|
||||
public SimpleOrderedMap<PivotFacet> pivotFacets
|
||||
= new SimpleOrderedMap<>();
|
||||
public LinkedHashMap<String,SpatialHeatmapFacets.HeatmapFacet> heatmapFacets;
|
||||
|
||||
void parse(SolrParams params, ResponseBuilder rb) {
|
||||
queryFacets = new LinkedHashMap<>();
|
||||
|
|
@ -1142,6 +1144,8 @@ public class FacetComponent extends SearchComponent {
|
|||
pivotFacets.add(pf.getKey(), pf);
|
||||
}
|
||||
}
|
||||
|
||||
heatmapFacets = SpatialHeatmapFacets.distribParse(params, rb);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,489 @@
|
|||
package org.apache.solr.handler.component;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.imageio.ImageReader;
|
||||
import javax.imageio.spi.ImageReaderSpi;
|
||||
import javax.imageio.stream.ImageInputStream;
|
||||
import javax.imageio.stream.ImageInputStreamImpl;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.util.AbstractList;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.spatial4j.core.context.SpatialContext;
|
||||
import com.spatial4j.core.shape.Shape;
|
||||
import org.apache.lucene.spatial.prefix.HeatmapFacetCounter;
|
||||
import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
|
||||
import org.apache.lucene.spatial.query.SpatialArgs;
|
||||
import org.apache.lucene.spatial.query.SpatialOperation;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.FacetParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.schema.AbstractSpatialPrefixTreeFieldType;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.schema.SpatialRecursivePrefixTreeFieldType;
|
||||
import org.apache.solr.search.DocSet;
|
||||
import org.apache.solr.search.QueryParsing;
|
||||
import org.apache.solr.util.SpatialUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/** A 2D spatial faceting summary of a rectangular region. Used by {@link org.apache.solr.handler.component.FacetComponent}
|
||||
* and {@link org.apache.solr.request.SimpleFacets}. */
|
||||
public class SpatialHeatmapFacets {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
//underneath facet_counts we put this here:
|
||||
public static final String RESPONSE_KEY = "facet_heatmaps";
|
||||
|
||||
public static final String FORMAT_PNG = "png";
|
||||
public static final String FORMAT_INTS2D = "ints2D";
|
||||
//note: if we change or add more formats, remember to update the javadoc on the format param
|
||||
//TODO for more format ideas, see formatCountsAndAddToNL
|
||||
|
||||
public static final double DEFAULT_DIST_ERR_PCT = 0.15;
|
||||
|
||||
/** Called by {@link org.apache.solr.request.SimpleFacets} to compute heatmap facets. */
|
||||
public static NamedList<Object> getHeatmapForField(String fieldKey, String fieldName, ResponseBuilder rb, SolrParams params, DocSet docSet) throws IOException {
|
||||
//get the strategy from the field type
|
||||
final SchemaField schemaField = rb.req.getSchema().getField(fieldName);
|
||||
final FieldType type = schemaField.getType();
|
||||
if (!(type instanceof AbstractSpatialPrefixTreeFieldType)) {
|
||||
//FYI we support the term query one too but few people use that one
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "heatmap field needs to be of type "
|
||||
+ SpatialRecursivePrefixTreeFieldType.class);
|
||||
}
|
||||
AbstractSpatialPrefixTreeFieldType rptType = (AbstractSpatialPrefixTreeFieldType) type;
|
||||
final PrefixTreeStrategy strategy = (PrefixTreeStrategy) rptType.getStrategy(fieldName);
|
||||
final SpatialContext ctx = strategy.getSpatialContext();
|
||||
|
||||
//get the bbox (query Rectangle)
|
||||
String geomStr = params.getFieldParam(fieldKey, FacetParams.FACET_HEATMAP_GEOM);
|
||||
final Shape boundsShape = geomStr == null ? ctx.getWorldBounds() : SpatialUtils.parseGeomSolrException(geomStr, ctx);
|
||||
|
||||
//get the grid level (possibly indirectly via distErr or distErrPct)
|
||||
final int gridLevel;
|
||||
Integer gridLevelObj = params.getFieldInt(fieldKey, FacetParams.FACET_HEATMAP_LEVEL);
|
||||
final int maxGridLevel = strategy.getGrid().getMaxLevels();
|
||||
if (gridLevelObj != null) {
|
||||
gridLevel = gridLevelObj;
|
||||
if (gridLevel <= 0 || gridLevel > maxGridLevel) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
FacetParams.FACET_HEATMAP_LEVEL +" should be > 0 and <= " + maxGridLevel);
|
||||
}
|
||||
} else {
|
||||
//SpatialArgs has utility methods to resolve a 'distErr' from optionally set distErr & distErrPct. Arguably that
|
||||
// should be refactored to feel less weird than using it like this.
|
||||
SpatialArgs spatialArgs = new SpatialArgs(SpatialOperation.Intersects/*ignored*/,
|
||||
boundsShape == null ? ctx.getWorldBounds() : boundsShape);
|
||||
final Double distErrObj = params.getFieldDouble(fieldKey, FacetParams.FACET_HEATMAP_DIST_ERR);
|
||||
if (distErrObj != null) {
|
||||
// convert distErr units based on configured units
|
||||
spatialArgs.setDistErr(distErrObj * rptType.getDistanceUnits().multiplierFromThisUnitToDegrees());
|
||||
}
|
||||
spatialArgs.setDistErrPct(params.getFieldDouble(fieldKey, FacetParams.FACET_HEATMAP_DIST_ERR_PCT));
|
||||
double distErr = spatialArgs.resolveDistErr(ctx, DEFAULT_DIST_ERR_PCT);
|
||||
if (distErr <= 0) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
FacetParams.FACET_HEATMAP_DIST_ERR_PCT + " or " + FacetParams.FACET_HEATMAP_DIST_ERR
|
||||
+ " should be > 0 or instead provide " + FacetParams.FACET_HEATMAP_LEVEL + "=" + maxGridLevel
|
||||
+ " if you insist on maximum detail");
|
||||
}
|
||||
//The SPT (grid) can lookup a grid level satisfying an error distance constraint
|
||||
gridLevel = strategy.getGrid().getLevelForDistance(distErr);
|
||||
}
|
||||
|
||||
//Compute!
|
||||
final HeatmapFacetCounter.Heatmap heatmap;
|
||||
try {
|
||||
heatmap = HeatmapFacetCounter.calcFacets(
|
||||
strategy,
|
||||
rb.req.getSearcher().getTopReaderContext(),
|
||||
docSet.getTopFilter(),
|
||||
boundsShape,
|
||||
gridLevel,
|
||||
params.getFieldInt(fieldKey, FacetParams.FACET_HEATMAP_MAX_CELLS, 100_000) // will throw if exceeded
|
||||
);
|
||||
} catch (IllegalArgumentException e) {//e.g. too many cells
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e.toString(), e);
|
||||
}
|
||||
|
||||
//Populate response
|
||||
NamedList<Object> result = new NamedList<>();
|
||||
result.add("gridLevel", gridLevel);
|
||||
result.add("columns", heatmap.columns);
|
||||
result.add("rows", heatmap.rows);
|
||||
result.add("minX", heatmap.region.getMinX());
|
||||
result.add("maxX", heatmap.region.getMaxX());
|
||||
result.add("minY", heatmap.region.getMinY());
|
||||
result.add("maxY", heatmap.region.getMaxY());
|
||||
|
||||
boolean hasNonZero = false;
|
||||
for (int count : heatmap.counts) {
|
||||
if (count > 0) {
|
||||
hasNonZero = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
formatCountsAndAddToNL(fieldKey, rb, params, heatmap.columns, heatmap.rows, hasNonZero ? heatmap.counts : null, result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void formatCountsAndAddToNL(String fieldKey, ResponseBuilder rb, SolrParams params,
|
||||
int columns, int rows, int[] counts, NamedList<Object> result) {
|
||||
final String format = params.getFieldParam(fieldKey, FacetParams.FACET_HEATMAP_FORMAT, FORMAT_INTS2D);
|
||||
final Object countsVal;
|
||||
switch (format) {
|
||||
case FORMAT_INTS2D: //A List of List of Integers. Good for small heatmaps and ease of consumption
|
||||
countsVal = counts != null ? asInts2D(columns, rows, counts) : null;
|
||||
break;
|
||||
case FORMAT_PNG: //A PNG graphic; compressed. Good for large & dense heatmaps; hard to consume.
|
||||
countsVal = counts != null ? asPngBytes(columns, rows, counts, rb) : null;
|
||||
break;
|
||||
//TODO case skipList: //A sequence of values; negative values are actually how many 0's to insert.
|
||||
// Good for small or large but sparse heatmaps.
|
||||
//TODO auto choose png or skipList; use skipList when < ~25% full or <= ~512 cells
|
||||
// remember to augment error list below when we add more formats.
|
||||
default:
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"format should be " + FORMAT_INTS2D + " or " + FORMAT_PNG);
|
||||
}
|
||||
result.add("counts_" + format, countsVal);
|
||||
}
|
||||
|
||||
static List<List<Integer>> asInts2D(final int columns, final int rows, final int[] counts) {
|
||||
//Returns a view versus returning a copy. This saves memory.
|
||||
//The data is oriented naturally for human/developer viewing: one row at a time top-down
|
||||
return new AbstractList<List<Integer>>() {
|
||||
@Override
|
||||
public List<Integer> get(final int rowIdx) {//top-down remember; the heatmap.counts is bottom up
|
||||
//check if all zeroes and return null if so
|
||||
boolean hasNonZero = false;
|
||||
int y = rows - rowIdx - 1;//flip direction for 'y'
|
||||
for (int c = 0; c < columns; c++) {
|
||||
if (counts[c * rows + y] > 0) {
|
||||
hasNonZero = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!hasNonZero) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new AbstractList<Integer>() {
|
||||
@Override
|
||||
public Integer get(int columnIdx) {
|
||||
return counts[columnIdx * rows + y];
|
||||
}
|
||||
|
||||
@Override
|
||||
public int size() {
|
||||
return columns;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public int size() {
|
||||
return rows;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
//package access for tests
|
||||
static byte[] asPngBytes(final int columns, final int rows, final int[] counts, ResponseBuilder rb) {
|
||||
long startTimeNano = System.nanoTime();
|
||||
BufferedImage image = PngHelper.newImage(columns, rows);
|
||||
for (int c = 0; c < columns; c++) {
|
||||
for (int r = 0; r < rows; r++) {
|
||||
PngHelper.writeCountAtColumnRow(image, rows, c, r, counts[c * rows + r]);
|
||||
}
|
||||
}
|
||||
byte[] bytes = PngHelper.writeImage(image);
|
||||
long durationMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNano);
|
||||
log.debug("heatmap nativeSize={} pngSize={} pngTime={}", (counts.length * 4), bytes.length, durationMs);
|
||||
if (rb != null && rb.isDebugTimings()) {
|
||||
rb.addDebug(durationMs, "timing", "heatmap png generation");
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
//
|
||||
// Distributed Support
|
||||
//
|
||||
|
||||
/** Parses request to "HeatmapFacet" instances. */
|
||||
public static LinkedHashMap<String,HeatmapFacet> distribParse(SolrParams params, ResponseBuilder rb) {
|
||||
final LinkedHashMap<String, HeatmapFacet> heatmapFacets = new LinkedHashMap<>();
|
||||
final String[] heatmapFields = params.getParams(FacetParams.FACET_HEATMAP);
|
||||
if (heatmapFields != null) {
|
||||
for (String heatmapField : heatmapFields) {
|
||||
HeatmapFacet facet = new HeatmapFacet(rb, heatmapField);
|
||||
heatmapFacets.put(facet.getKey(), facet);
|
||||
}
|
||||
}
|
||||
return heatmapFacets;
|
||||
}
|
||||
|
||||
/** Called by FacetComponent's impl of
|
||||
* {@link org.apache.solr.handler.component.SearchComponent#modifyRequest(ResponseBuilder, SearchComponent, ShardRequest)}. */
|
||||
public static void distribModifyRequest(ShardRequest sreq, LinkedHashMap<String, HeatmapFacet> heatmapFacets) {
|
||||
// Set the format to PNG because it's compressed and it's the only format we have code to read at the moment.
|
||||
// Changing a param is sadly tricky because field-specific params can show up as local-params (highest precedence)
|
||||
// or as f.key.facet.heatmap.whatever. Ugh. So we re-write the facet.heatmap list with the local-params
|
||||
// moved out to the "f.key." prefix, but we need to keep the key local-param because that's the only way to
|
||||
// set an output key. This approach means we only need to know about the parameter we're changing, not of
|
||||
// all possible heatmap params.
|
||||
|
||||
//Remove existing heatmap field param vals; we will rewrite
|
||||
sreq.params.remove(FacetParams.FACET_HEATMAP);
|
||||
for (Map.Entry<String, HeatmapFacet> entry : heatmapFacets.entrySet()) {
|
||||
final String key = entry.getKey();
|
||||
final HeatmapFacet facet = entry.getValue();
|
||||
//add heatmap field param
|
||||
if (!key.equals(facet.facetOn)) {
|
||||
sreq.params.add(FacetParams.FACET_HEATMAP,
|
||||
"{!" + CommonParams.OUTPUT_KEY + "=" + QueryParsing.encodeLocalParamVal(key) + "}" + facet.facetOn);
|
||||
} else {
|
||||
sreq.params.add(FacetParams.FACET_HEATMAP, facet.facetOn);
|
||||
}
|
||||
// Turn local-params into top-level f.key.param=value style params
|
||||
if (facet.localParams != null) {
|
||||
final Iterator<String> localNameIter = facet.localParams.getParameterNamesIterator();
|
||||
while (localNameIter.hasNext()) {
|
||||
String pname = localNameIter.next();
|
||||
if (!pname.startsWith(FacetParams.FACET_HEATMAP)) {
|
||||
continue; // could be 'key', or 'v' even
|
||||
}
|
||||
String pval = facet.localParams.get(pname);
|
||||
sreq.params.set("f." + key + "." + pname, pval);
|
||||
}
|
||||
}
|
||||
// Remove existing format specifier
|
||||
sreq.params.remove("f." + key + "." + FacetParams.FACET_HEATMAP_FORMAT);
|
||||
}
|
||||
// Set format to PNG (applies to all heatmaps)
|
||||
sreq.params.set(FacetParams.FACET_HEATMAP_FORMAT, FORMAT_PNG);
|
||||
}
|
||||
|
||||
/** Called by FacetComponent.countFacets which is in turn called by FC's impl of
|
||||
* {@link org.apache.solr.handler.component.SearchComponent#handleResponses(ResponseBuilder, ShardRequest)}. */
|
||||
@SuppressWarnings("unchecked")
|
||||
public static void distribHandleResponse(LinkedHashMap<String, HeatmapFacet> heatmapFacets, NamedList srsp_facet_counts) {
|
||||
NamedList<NamedList<Object>> facet_heatmaps = (NamedList<NamedList<Object>>) srsp_facet_counts.get(RESPONSE_KEY);
|
||||
if (facet_heatmaps == null) {
|
||||
return;
|
||||
}
|
||||
// (should the caller handle the above logic? Arguably yes.)
|
||||
for (Map.Entry<String, NamedList<Object>> entry : facet_heatmaps) {
|
||||
String fieldKey = entry.getKey();
|
||||
NamedList<Object> shardNamedList = entry.getValue();
|
||||
final HeatmapFacet facet = heatmapFacets.get(fieldKey);
|
||||
if (facet == null) {
|
||||
log.error("received heatmap for field/key {} that we weren't expecting", fieldKey);
|
||||
continue;
|
||||
}
|
||||
facet.counts = addPngToIntArray((byte[]) shardNamedList.remove("counts_" + FORMAT_PNG), facet.counts);
|
||||
if (facet.namedList == null) {
|
||||
// First shard
|
||||
facet.namedList = shardNamedList;
|
||||
} else {
|
||||
assert facet.namedList.equals(shardNamedList);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//package access for tests
|
||||
static int[] addPngToIntArray(byte[] pngBytes, int[] counts) {
|
||||
if (pngBytes == null) {
|
||||
return counts;
|
||||
}
|
||||
//read PNG
|
||||
final BufferedImage image = PngHelper.readImage(pngBytes);
|
||||
int columns = image.getWidth();
|
||||
int rows = image.getHeight();
|
||||
if (counts == null) {
|
||||
counts = new int[columns * rows];
|
||||
} else {
|
||||
assert counts.length == columns * rows;
|
||||
}
|
||||
for (int c = 0; c < columns; c++) {
|
||||
for (int r = 0; r < rows; r++) {
|
||||
counts[c * rows + r] += PngHelper.getCountAtColumnRow(image, rows, c, r);
|
||||
}
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
/** Called by FacetComponent's impl of
|
||||
* {@link org.apache.solr.handler.component.SearchComponent#finishStage(ResponseBuilder)}. */
|
||||
public static NamedList distribFinish(LinkedHashMap<String, HeatmapFacet> heatmapInfos, ResponseBuilder rb) {
|
||||
NamedList<NamedList<Object>> result = new SimpleOrderedMap<>();
|
||||
for (Map.Entry<String, HeatmapFacet> entry : heatmapInfos.entrySet()) {
|
||||
final HeatmapFacet facet = entry.getValue();
|
||||
final NamedList<Object> namedList = facet.namedList;
|
||||
if (namedList == null) {
|
||||
continue;//should never happen but play it safe
|
||||
}
|
||||
formatCountsAndAddToNL(entry.getKey(), rb, SolrParams.wrapDefaults(facet.localParams, rb.req.getParams()),
|
||||
(int) namedList.get("columns"), (int) namedList.get("rows"), facet.counts, namedList);
|
||||
result.add(entry.getKey(), namedList);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Goes in {@link org.apache.solr.handler.component.FacetComponent.FacetInfo#heatmapFacets}, created by
|
||||
* {@link #distribParse(org.apache.solr.common.params.SolrParams, ResponseBuilder)}. */
|
||||
public static class HeatmapFacet extends FacetComponent.FacetBase {
|
||||
//note: 'public' following-suit with FacetBase & existing subclasses... though should this really be?
|
||||
|
||||
//Holds response NamedList for this field, with counts pulled out. Taken from 1st shard response.
|
||||
public NamedList<Object> namedList;
|
||||
//Like Heatmap.counts in Lucene spatial, although null if it would be all-0.
|
||||
public int[] counts;
|
||||
|
||||
public HeatmapFacet(ResponseBuilder rb, String facetStr) {
|
||||
super(rb, FacetParams.FACET_HEATMAP, facetStr);
|
||||
//note: logic in super (FacetBase) is partially redundant with SimpleFacet.parseParams :-(
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// PngHelper
|
||||
//
|
||||
|
||||
//package access for tests
|
||||
static class PngHelper {
|
||||
|
||||
static final ImageReaderSpi imageReaderSpi;//thread-safe
|
||||
static {
|
||||
final Iterator<ImageReader> imageReaders = ImageIO.getImageReadersByFormatName("png");
|
||||
if (!imageReaders.hasNext()) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Can't find png image reader, neaded for heatmaps!");
|
||||
}
|
||||
ImageReader imageReader = imageReaders.next();
|
||||
imageReaderSpi = imageReader.getOriginatingProvider();
|
||||
}
|
||||
|
||||
static BufferedImage readImage(final byte[] bytes) {
|
||||
// Wrap ImageInputStream around the bytes. We could use MemoryCacheImageInputStream but it will
|
||||
// cache the data which is quite unnecessary given we have it all in-memory already.
|
||||
ImageInputStream imageInputStream = new ImageInputStreamImpl() {
|
||||
//TODO re-use this instance; superclass has 8KB buffer.
|
||||
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
checkClosed();
|
||||
bitOffset = 0;
|
||||
if (streamPos >= bytes.length) {
|
||||
return -1;
|
||||
} else {
|
||||
return bytes[(int) streamPos++];
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(byte[] b, int off, int len) throws IOException {
|
||||
checkClosed();
|
||||
bitOffset = 0;
|
||||
if (streamPos >= bytes.length) {
|
||||
return -1;
|
||||
} else {
|
||||
int copyLen = Math.min(len, bytes.length - (int)streamPos);
|
||||
System.arraycopy(bytes, (int)streamPos, b, off, copyLen);
|
||||
streamPos += copyLen;
|
||||
return copyLen;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long length() {
|
||||
return bytes.length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isCached() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isCachedMemory() {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
try {
|
||||
//TODO can/should we re-use an imageReader instance on FacetInfo?
|
||||
ImageReader imageReader = imageReaderSpi.createReaderInstance();
|
||||
|
||||
imageReader.setInput(imageInputStream,
|
||||
false,//forwardOnly
|
||||
true);//ignoreMetadata
|
||||
return imageReader.read(0);//read first & only image
|
||||
} catch (IOException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Problem reading png heatmap: " + e);
|
||||
}
|
||||
}
|
||||
|
||||
static byte[] writeImage(BufferedImage image) {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream(
|
||||
// initialize to roughly 1/4th the size a native int would take per-pixel
|
||||
image.getWidth() * image.getHeight() + 1024
|
||||
);
|
||||
try {
|
||||
ImageIO.write(image, FORMAT_PNG, baos);
|
||||
} catch (IOException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "While generating PNG: " + e);
|
||||
}
|
||||
//too bad we can't access the raw byte[]; this copies to a new one
|
||||
return baos.toByteArray();
|
||||
}
|
||||
|
||||
// We abuse the image for storing integers (4 bytes), and so we need a 4-byte ABGR.
|
||||
// first (low) byte is blue, next byte is green, next byte red, and last (high) byte is alpha.
|
||||
static BufferedImage newImage(int columns, int rows) {
|
||||
return new BufferedImage(columns, rows, BufferedImage.TYPE_4BYTE_ABGR);
|
||||
}
|
||||
|
||||
// 'y' dimension goes top-down, so invert.
|
||||
// Alpha chanel is high byte; 0 means transparent. So XOR those bits with '1' so that we need
|
||||
// to have counts > 16M before the picture starts to fade
|
||||
|
||||
static void writeCountAtColumnRow(BufferedImage image, int rows, int c, int r, int val) {
|
||||
image.setRGB(c, rows - 1 - r, val ^ 0xFF_00_00_00);
|
||||
}
|
||||
|
||||
static int getCountAtColumnRow(BufferedImage image, int rows, int c, int r) {
|
||||
return image.getRGB(c, rows - 1 - r) ^ 0xFF_00_00_00;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -72,6 +72,7 @@ import org.apache.solr.common.util.NamedList;
|
|||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.handler.component.ResponseBuilder;
|
||||
import org.apache.solr.handler.component.SpatialHeatmapFacets;
|
||||
import org.apache.solr.request.IntervalFacets.FacetInterval;
|
||||
import org.apache.solr.schema.BoolField;
|
||||
import org.apache.solr.schema.DateRangeField;
|
||||
|
|
@ -259,7 +260,7 @@ public class SimpleFacets {
|
|||
facetResponse.add("facet_dates", getFacetDateCounts());
|
||||
facetResponse.add("facet_ranges", getFacetRangeCounts());
|
||||
facetResponse.add("facet_intervals", getFacetIntervalCounts());
|
||||
|
||||
facetResponse.add(SpatialHeatmapFacets.RESPONSE_KEY, getHeatmapCounts());
|
||||
} catch (IOException e) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, e);
|
||||
} catch (SyntaxError e) {
|
||||
|
|
@ -1519,5 +1520,22 @@ public class SimpleFacets {
|
|||
return res;
|
||||
}
|
||||
|
||||
private NamedList getHeatmapCounts() throws IOException, SyntaxError {
|
||||
final NamedList<Object> resOuter = new SimpleOrderedMap<>();
|
||||
String[] unparsedFields = rb.req.getParams().getParams(FacetParams.FACET_HEATMAP);
|
||||
if (unparsedFields == null || unparsedFields.length == 0) {
|
||||
return resOuter;
|
||||
}
|
||||
if (params.getBool(GroupParams.GROUP_FACET, false)) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"Heatmaps can't be used with " + GroupParams.GROUP_FACET);
|
||||
}
|
||||
for (String unparsedField : unparsedFields) {
|
||||
parseParams(FacetParams.FACET_HEATMAP, unparsedField); // populates facetValue, rb, params, docs
|
||||
|
||||
resOuter.add(key, SpatialHeatmapFacets.getHeatmapForField(key, facetValue, rb, params, docs));
|
||||
}
|
||||
return resOuter;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,9 +17,13 @@ package org.apache.solr.util;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.text.ParseException;
|
||||
|
||||
import com.spatial4j.core.context.SpatialContext;
|
||||
import com.spatial4j.core.exception.InvalidShapeException;
|
||||
import com.spatial4j.core.shape.Point;
|
||||
import com.spatial4j.core.shape.Rectangle;
|
||||
import com.spatial4j.core.shape.Shape;
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
||||
/** Utility methods pertaining to spatial. */
|
||||
|
|
@ -27,6 +31,28 @@ public class SpatialUtils {
|
|||
|
||||
private SpatialUtils() {}
|
||||
|
||||
/**
|
||||
* Parses a 'geom' parameter (might also be used to parse shapes for indexing). {@code geomStr} can either be WKT or
|
||||
* a rectangle-range syntax (see {@link #parseRectangle(String, com.spatial4j.core.context.SpatialContext)}.
|
||||
*/
|
||||
public static Shape parseGeomSolrException(String geomStr, SpatialContext ctx) {
|
||||
if (geomStr.length() == 0) {
|
||||
throw new IllegalArgumentException("0-length geometry string");
|
||||
}
|
||||
char c = geomStr.charAt(0);
|
||||
if (c == '[' || c == '{') {
|
||||
return parseRectangeSolrException(geomStr, ctx);
|
||||
}
|
||||
//TODO parse a raw point?
|
||||
try {
|
||||
return ctx.readShapeFromWkt(geomStr);
|
||||
} catch (ParseException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"Expecting WKT or '[minPoint TO maxPoint]': " + e, e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/** Parses either "lat, lon" (spaces optional on either comma side) or "x y" style formats. Spaces can be basically
|
||||
* anywhere. And not any whitespace, just the space char.
|
||||
*
|
||||
|
|
@ -37,8 +63,6 @@ public class SpatialUtils {
|
|||
*/
|
||||
public static Point parsePoint(String str, SpatialContext ctx) throws InvalidShapeException {
|
||||
//note we don't do generic whitespace, just a literal space char detection
|
||||
//TODO: decide on if we should pick one format decided by ctx.isGeo()
|
||||
// Perhaps 5x use isGeo; 4x use either?
|
||||
try {
|
||||
double x, y;
|
||||
str = str.trim();//TODO use findIndexNotSpace instead?
|
||||
|
|
@ -89,4 +113,53 @@ public class SpatialUtils {
|
|||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses {@code str} in the format of '[minPoint TO maxPoint]' where {@code minPoint} is the lower left corner
|
||||
* and maxPoint is the upper-right corner of the bounding box. Both corners may optionally be wrapped with a quote
|
||||
* and then it's parsed via {@link #parsePoint(String, com.spatial4j.core.context.SpatialContext)}.
|
||||
* @param str Non-null; may *not* have leading or trailing spaces
|
||||
* @param ctx Non-null
|
||||
* @return the Rectangle
|
||||
* @throws InvalidShapeException If for any reason there was a problem parsing the string or creating the rectangle.
|
||||
*/
|
||||
public static Rectangle parseRectangle(String str, SpatialContext ctx) throws InvalidShapeException {
|
||||
//note we don't do generic whitespace, just a literal space char detection
|
||||
try {
|
||||
int toIdx = str.indexOf(" TO ");
|
||||
if (toIdx == -1 || str.charAt(0) != '[' || str.charAt(str.length() - 1) != ']') {
|
||||
throw new InvalidShapeException("expecting '[bottomLeft TO topRight]'");
|
||||
}
|
||||
String leftPart = unwrapQuotes(str.substring(1, toIdx).trim());
|
||||
String rightPart = unwrapQuotes(str.substring(toIdx + " TO ".length(), str.length() - 1).trim());
|
||||
return ctx.makeRectangle(parsePoint(leftPart, ctx), parsePoint(rightPart, ctx));
|
||||
} catch (InvalidShapeException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new InvalidShapeException(e.toString(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls {@link #parseRectangle(String, com.spatial4j.core.context.SpatialContext)} and wraps the exception with
|
||||
* {@link org.apache.solr.common.SolrException} with a helpful message.
|
||||
*/
|
||||
public static Rectangle parseRectangeSolrException(String externalVal, SpatialContext ctx) throws SolrException {
|
||||
try {
|
||||
return parseRectangle(externalVal, ctx);
|
||||
} catch (InvalidShapeException e) {
|
||||
String message = e.getMessage();
|
||||
if (!message.contains(externalVal))
|
||||
message = "Can't parse rectangle '" + externalVal + "' because: " + message;
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message, e);
|
||||
}
|
||||
}
|
||||
|
||||
private static String unwrapQuotes(String str) {
|
||||
if (str.length() >= 2 && str.charAt(0) == '\"' && str.charAt(str.length()-1) == '\"') {
|
||||
return str.substring(1, str.length()-1);
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,22 +17,6 @@
|
|||
|
||||
package org.apache.solr;
|
||||
|
||||
import org.apache.lucene.index.LogDocMergePolicy;
|
||||
import org.noggit.JSONUtil;
|
||||
import org.noggit.ObjectBuilder;
|
||||
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.GroupParams;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrRequestInfo;
|
||||
import org.apache.solr.response.BinaryResponseWriter;
|
||||
import org.apache.solr.response.ResultContext;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.schema.IndexSchema;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.util.ArrayList;
|
||||
|
|
@ -46,11 +30,28 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.lucene.index.LogDocMergePolicy;
|
||||
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.GroupParams;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrRequestInfo;
|
||||
import org.apache.solr.response.BinaryResponseWriter;
|
||||
import org.apache.solr.response.ResultContext;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.schema.IndexSchema;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.noggit.JSONUtil;
|
||||
import org.noggit.ObjectBuilder;
|
||||
|
||||
public class TestGroupingSearch extends SolrTestCaseJ4 {
|
||||
|
||||
public static final String FOO_STRING_FIELD = "foo_s1";
|
||||
public static final String SMALL_STRING_FIELD = "small_s1";
|
||||
public static final String SMALL_INT_FIELD = "small_i";
|
||||
static final String EMPTY_FACETS = "'facet_dates':{},'facet_ranges':{},'facet_intervals':{},'facet_heatmaps':{}";
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeTests() throws Exception {
|
||||
|
|
@ -317,7 +318,7 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
|
|||
assertJQ(
|
||||
req,
|
||||
"/grouped=={'value1_s1':{'matches':5,'groups':[{'groupValue':'1','doclist':{'numFound':3,'start':0,'docs':[{'id':'1'}]}}]}}",
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',3,'b',2]},'facet_dates':{},'facet_ranges':{},'facet_intervals':{}}"
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',3,'b',2]}," + EMPTY_FACETS + "}"
|
||||
);
|
||||
|
||||
// Facet counts based on groups
|
||||
|
|
@ -326,7 +327,7 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
|
|||
assertJQ(
|
||||
req,
|
||||
"/grouped=={'value1_s1':{'matches':5,'groups':[{'groupValue':'1','doclist':{'numFound':3,'start':0,'docs':[{'id':'1'}]}}]}}",
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',1,'b',1]},'facet_dates':{},'facet_ranges':{},'facet_intervals':{}}"
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',1,'b',1]}," + EMPTY_FACETS + "}"
|
||||
);
|
||||
|
||||
// Facet counts based on groups and with group.func. This should trigger FunctionAllGroupHeadsCollector
|
||||
|
|
@ -335,7 +336,7 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
|
|||
assertJQ(
|
||||
req,
|
||||
"/grouped=={'strdist(1,value1_s1,edit)':{'matches':5,'groups':[{'groupValue':1.0,'doclist':{'numFound':3,'start':0,'docs':[{'id':'1'}]}}]}}",
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',1,'b',1]},'facet_dates':{},'facet_ranges':{},'facet_intervals':{}}"
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',1,'b',1]}," + EMPTY_FACETS + "}"
|
||||
);
|
||||
|
||||
// Facet counts based on groups without sort on an int field.
|
||||
|
|
@ -344,7 +345,7 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
|
|||
assertJQ(
|
||||
req,
|
||||
"/grouped=={'value4_i':{'matches':5,'groups':[{'groupValue':1,'doclist':{'numFound':3,'start':0,'docs':[{'id':'1'}]}}]}}",
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',1,'b',1]},'facet_dates':{},'facet_ranges':{},'facet_intervals':{}}"
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',1,'b',1]}," + EMPTY_FACETS + "}"
|
||||
);
|
||||
|
||||
// Multi select facets AND group.truncate=true
|
||||
|
|
@ -353,7 +354,7 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
|
|||
assertJQ(
|
||||
req,
|
||||
"/grouped=={'value4_i':{'matches':2,'groups':[{'groupValue':2,'doclist':{'numFound':2,'start':0,'docs':[{'id':'3'}]}}]}}",
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',1,'b',1]},'facet_dates':{},'facet_ranges':{},'facet_intervals':{}}"
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',1,'b',1]}," + EMPTY_FACETS + "}"
|
||||
);
|
||||
|
||||
// Multi select facets AND group.truncate=false
|
||||
|
|
@ -362,7 +363,7 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
|
|||
assertJQ(
|
||||
req,
|
||||
"/grouped=={'value4_i':{'matches':2,'groups':[{'groupValue':2,'doclist':{'numFound':2,'start':0,'docs':[{'id':'3'}]}}]}}",
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',3,'b',2]},'facet_dates':{},'facet_ranges':{},'facet_intervals':{}}"
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',3,'b',2]}," + EMPTY_FACETS + "}"
|
||||
);
|
||||
|
||||
// Multi select facets AND group.truncate=true
|
||||
|
|
@ -371,7 +372,7 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
|
|||
assertJQ(
|
||||
req,
|
||||
"/grouped=={'sub(value4_i,1)':{'matches':2,'groups':[{'groupValue':1.0,'doclist':{'numFound':2,'start':0,'docs':[{'id':'3'}]}}]}}",
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',1,'b',1]},'facet_dates':{},'facet_ranges':{},'facet_intervals':{}}"
|
||||
"/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',1,'b',1]}," + EMPTY_FACETS + "}"
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -394,7 +395,7 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
|
|||
assertJQ(
|
||||
req,
|
||||
"/grouped=={'cat_sI':{'matches':2,'groups':[{'groupValue':'a','doclist':{'numFound':1,'start':0,'docs':[{'id':'5'}]}}]}}",
|
||||
"/facet_counts=={'facet_queries':{'LW1':2,'LM1':2,'LM3':2},'facet_fields':{},'facet_dates':{},'facet_ranges':{},'facet_intervals':{}}"
|
||||
"/facet_counts=={'facet_queries':{'LW1':2,'LM1':2,'LM3':2},'facet_fields':{}," + EMPTY_FACETS + "}"
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,194 @@
|
|||
package org.apache.solr.handler.component;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import com.carrotsearch.randomizedtesting.annotations.Repeat;
|
||||
import org.apache.solr.BaseDistributedSearchTestCase;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.FacetParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
public class SpatialHeatmapFacetsTest extends BaseDistributedSearchTestCase {
|
||||
private static final String FIELD = "srpt_quad";
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeSuperClass() throws Exception {
|
||||
schemaString = "schema-spatial.xml";
|
||||
configString = "solrconfig-basic.xml";
|
||||
|
||||
//Strictly not necessary (set already in Ant & Maven) but your IDE might not have this set
|
||||
System.setProperty("java.awt.headless", "true");
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
public void test() throws Exception {
|
||||
handle.clear();
|
||||
handle.put("QTime", SKIPVAL);
|
||||
handle.put("timestamp", SKIPVAL);
|
||||
handle.put("maxScore", SKIPVAL);
|
||||
|
||||
SolrParams baseParams = params("q", "*:*", "rows", "0", "facet", "true", FacetParams.FACET_HEATMAP, FIELD);
|
||||
|
||||
final String testBox = "[\"50 50\" TO \"180 90\"]";//top-right somewhere on edge (whatever)
|
||||
|
||||
//----- First we test gridLevel derivation
|
||||
try {
|
||||
getHmObj(query(params(baseParams, FacetParams.FACET_HEATMAP_GEOM, testBox, FacetParams.FACET_HEATMAP_DIST_ERR, "0"))).get("gridLevel");
|
||||
fail();
|
||||
} catch (SolrException e) {
|
||||
assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code());
|
||||
}
|
||||
try {
|
||||
getHmObj(query(params(baseParams, FacetParams.FACET_HEATMAP_GEOM, testBox, FacetParams.FACET_HEATMAP_DIST_ERR_PCT, "0"))).get("gridLevel");
|
||||
fail();
|
||||
} catch (SolrException e) {
|
||||
assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code());
|
||||
}
|
||||
// Monkeying with these params changes the gridLevel in different directions. We don't test the exact
|
||||
// computation here; that's not _that_ relevant, and is Lucene spatial's job (not Solr) any way.
|
||||
assertEquals(7, getHmObj(query(params(baseParams, FacetParams.FACET_HEATMAP_GEOM, testBox))).get("gridLevel"));//default
|
||||
assertEquals(3, getHmObj(query(params(baseParams, FacetParams.FACET_HEATMAP_GEOM, testBox, FacetParams.FACET_HEATMAP_LEVEL, "3"))).get("gridLevel"));
|
||||
assertEquals(2, getHmObj(query(params(baseParams, FacetParams.FACET_HEATMAP_GEOM, testBox, FacetParams.FACET_HEATMAP_DIST_ERR, "100"))).get("gridLevel"));
|
||||
//TODO test impact of distance units
|
||||
assertEquals(9, getHmObj(query(params(baseParams, FacetParams.FACET_HEATMAP_GEOM, testBox, FacetParams.FACET_HEATMAP_DIST_ERR_PCT, "0.05"))).get("gridLevel"));
|
||||
assertEquals(6, getHmObj(query(params(baseParams, FacetParams.FACET_HEATMAP_DIST_ERR_PCT, "0.10"))).get("gridLevel"));
|
||||
|
||||
//test key output label doing 2 heatmaps with different settings on the same field
|
||||
{
|
||||
final ModifiableSolrParams params = params(baseParams, FacetParams.FACET_HEATMAP_DIST_ERR_PCT, "0.10");
|
||||
String courseFormat = random().nextBoolean() ? "png" : "ints2D";
|
||||
params.add(FacetParams.FACET_HEATMAP, "{!key=course "
|
||||
+ FacetParams.FACET_HEATMAP_LEVEL + "=2 "
|
||||
+ FacetParams.FACET_HEATMAP_FORMAT + "=" + courseFormat
|
||||
+ "}" + FIELD);
|
||||
final QueryResponse response = query(params);
|
||||
assertEquals(6, getHmObj(response).get("gridLevel"));//same test as above
|
||||
assertEquals(2, response.getResponse().findRecursive("facet_counts", "facet_heatmaps", "course", "gridLevel"));
|
||||
assertTrue(((NamedList<Object>) response.getResponse().findRecursive("facet_counts", "facet_heatmaps", "course"))
|
||||
.asMap(0).containsKey("counts_" + courseFormat));
|
||||
}
|
||||
|
||||
// ------ Index data
|
||||
|
||||
index("id", "0", FIELD, "ENVELOPE(100, 120, 80, 40)");// on right side
|
||||
index("id", "1", FIELD, "ENVELOPE(-120, -110, 80, 20)");// on left side (outside heatmap)
|
||||
index("id", "3", FIELD, "POINT(70 60)");//just left of BOX 0
|
||||
index("id", "4", FIELD, "POINT(91 89)");//just outside box 0 (above it) near pole,
|
||||
|
||||
commit();
|
||||
|
||||
// ----- Search
|
||||
// this test simply has some 0's, nulls, 1's and a 2 in there.
|
||||
NamedList hmObj = getHmObj(query(params(baseParams,
|
||||
FacetParams.FACET_HEATMAP_GEOM, "[\"50 20\" TO \"180 90\"]",
|
||||
FacetParams.FACET_HEATMAP_LEVEL, "4")));
|
||||
List<List<Integer>> counts = (List<List<Integer>>) hmObj.get("counts_ints2D");
|
||||
assertEquals(
|
||||
Arrays.asList(
|
||||
Arrays.asList(0, 0, 2, 1, 0, 0),
|
||||
Arrays.asList(0, 0, 1, 1, 0, 0),
|
||||
Arrays.asList(0, 1, 1, 1, 0, 0),
|
||||
Arrays.asList(0, 0, 1, 1, 0, 0),
|
||||
Arrays.asList(0, 0, 1, 1, 0, 0),
|
||||
null,
|
||||
null
|
||||
),
|
||||
counts
|
||||
);
|
||||
|
||||
// test using a circle input shape
|
||||
hmObj = getHmObj(query(params(baseParams,
|
||||
FacetParams.FACET_HEATMAP_GEOM, "BUFFER(POINT(110 40), 7)",
|
||||
FacetParams.FACET_HEATMAP_LEVEL, "7")));
|
||||
counts = (List<List<Integer>>) hmObj.get("counts_ints2D");
|
||||
assertEquals(
|
||||
Arrays.asList(
|
||||
Arrays.asList(0, 1, 1, 1, 1, 1, 1, 0),//curved; we have a 0
|
||||
Arrays.asList(0, 1, 1, 1, 1, 1, 1, 0),//curved; we have a 0
|
||||
Arrays.asList(0, 1, 1, 1, 1, 1, 1, 0),//curved; we have a 0
|
||||
Arrays.asList(1, 1, 1, 1, 1, 1, 1, 1),
|
||||
Arrays.asList(1, 1, 1, 1, 1, 1, 1, 1),
|
||||
Arrays.asList(1, 1, 1, 1, 1, 1, 1, 1),
|
||||
null, null, null, null, null//no data here (below edge of rect 0)
|
||||
),
|
||||
counts
|
||||
);
|
||||
|
||||
// Search in no-where ville and get null counts
|
||||
assertNull(getHmObj(query(params(baseParams, FacetParams.FACET_HEATMAP_GEOM, "ENVELOPE(0, 10, -80, -90)"))).get("counts_ints2D"));
|
||||
|
||||
Object v = getHmObj(query(params(baseParams, FacetParams.FACET_HEATMAP_FORMAT, "png"))).get("counts_png");
|
||||
assertTrue(v instanceof byte[]);
|
||||
//simply test we can read the image
|
||||
assertNotNull(SpatialHeatmapFacets.PngHelper.readImage((byte[]) v));
|
||||
//good enough for this test method
|
||||
}
|
||||
|
||||
private NamedList getHmObj(QueryResponse response) {
|
||||
return (NamedList) response.getResponse().findRecursive("facet_counts", "facet_heatmaps", FIELD);
|
||||
}
|
||||
|
||||
private ModifiableSolrParams params(SolrParams baseParams, String... moreParams) {
|
||||
final ModifiableSolrParams params = new ModifiableSolrParams(baseParams);
|
||||
params.add(params(moreParams));//actually replaces
|
||||
return params;
|
||||
}
|
||||
|
||||
@Test
|
||||
@Repeat(iterations = 3)
|
||||
public void testPng() {
|
||||
//We test via round-trip randomized data:
|
||||
|
||||
// Make random data
|
||||
int columns = random().nextInt(100) + 1;
|
||||
int rows = random().nextInt(100) + 1;
|
||||
int[] counts = new int[columns * rows];
|
||||
for (int i = 0; i < counts.length; i++) {
|
||||
final int ri = random().nextInt(10);
|
||||
if (ri >= 0 && ri <= 3) {
|
||||
counts[i] = ri; // 0 thru 3 will be made common
|
||||
} else if (ri > 3) {
|
||||
counts[i] = Math.abs(random().nextInt());//lots of other possible values up to max
|
||||
}
|
||||
}
|
||||
// Round-trip
|
||||
final byte[] bytes = SpatialHeatmapFacets.asPngBytes(columns, rows, counts, null);
|
||||
int[] countsOut = random().nextBoolean() ? new int[columns * rows] : null;
|
||||
int base = 0;
|
||||
if (countsOut != null) {
|
||||
base = 9;
|
||||
Arrays.fill(countsOut, base);
|
||||
}
|
||||
countsOut = SpatialHeatmapFacets.addPngToIntArray(bytes, countsOut);
|
||||
// Test equal
|
||||
assertEquals(counts.length, countsOut.length);
|
||||
for (int i = 0; i < countsOut.length; i++) {
|
||||
assertEquals(counts[i], countsOut[i] - base);//back out the base input to prove we added
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -17,11 +17,11 @@
|
|||
|
||||
package org.apache.solr.common.params;
|
||||
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
||||
import java.util.EnumSet;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
||||
/**
|
||||
* Facet parameters
|
||||
*/
|
||||
|
|
@ -282,6 +282,41 @@ public interface FacetParams {
|
|||
*/
|
||||
public static final String FACET_INTERVAL_SET = FACET_INTERVAL + ".set";
|
||||
|
||||
/** A spatial RPT field to generate a 2D "heatmap" (grid of facet counts) on. Just like the other faceting types,
|
||||
* this may include a 'key' or local-params to facet multiple times. All parameters with this suffix can be
|
||||
* overridden on a per-field basis. */
|
||||
public static final String FACET_HEATMAP = "facet.heatmap";
|
||||
|
||||
/** The format of the heatmap: either png or ints2D (default). */
|
||||
public static final String FACET_HEATMAP_FORMAT = FACET_HEATMAP + ".format";
|
||||
|
||||
/** The region the heatmap should minimally enclose. It defaults to the world if not set. The format can either be
|
||||
* a minimum to maximum point range format: <pre>["-150 10" TO "-100 30"]</pre> (the first is bottom-left and second
|
||||
* is bottom-right, both of which are parsed as points are parsed). OR, any WKT can be provided and it's bounding
|
||||
* box will be taken. */
|
||||
public static final String FACET_HEATMAP_GEOM = FACET_HEATMAP + ".geom";
|
||||
|
||||
/** Specify the heatmap grid level explicitly, instead of deriving it via distErr or distErrPct. */
|
||||
public static final String FACET_HEATMAP_LEVEL = FACET_HEATMAP + ".gridLevel";
|
||||
|
||||
/** Used to determine the heatmap grid level to compute, defaulting to 0.15. It has the same interpretation of
|
||||
* distErrPct when searching on RPT, but relative to the shape in 'bbox'. It's a fraction (not a %) of the radius of
|
||||
* the shape that grid squares must fit into without exceeding. > 0 and <= 0.5.
|
||||
* Mutually exclusive with distErr & gridLevel. */
|
||||
public static final String FACET_HEATMAP_DIST_ERR_PCT = FACET_HEATMAP + ".distErrPct";
|
||||
|
||||
/** Used to determine the heatmap grid level to compute (optional). It has the same interpretation of maxDistErr or
|
||||
* distErr with RPT. It's an absolute distance (in units of what's specified on the field type) that a grid square
|
||||
* must maximally fit into (width & height). It can be used to to more explicitly specify the maximum grid square
|
||||
* size without knowledge of what particular grid levels translate to. This can in turn be used with
|
||||
* knowledge of the size of 'bbox' to get a target minimum number of grid cells.
|
||||
* Mutually exclusive with distErrPct & gridLevel. */
|
||||
public static final String FACET_HEATMAP_DIST_ERR = FACET_HEATMAP + ".distErr";
|
||||
|
||||
/** The maximum number of cells (grid squares) the client is willing to handle. If this limit would be exceeded, we
|
||||
* throw an error instead. Defaults to 100k. */
|
||||
public static final String FACET_HEATMAP_MAX_CELLS = FACET_HEATMAP + ".maxCells";
|
||||
|
||||
/**
|
||||
* An enumeration of the legal values for {@link #FACET_RANGE_OTHER} and {@link #FACET_DATE_OTHER} ...
|
||||
* <ul>
|
||||
|
|
|
|||
Loading…
Reference in New Issue