LUCENE-5371: faster range faceting using segment trees

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene5339@1552086 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2013-12-18 20:40:35 +00:00
parent 8da43e722e
commit 8e1879225a
9 changed files with 634 additions and 84 deletions

View File

@ -68,6 +68,11 @@ New Features
* LUCENE-5336: Add SimpleQueryParser: parser for human-entered queries.
(Jack Conradson via Robert Muir)
* LUCENE-5371: Speed up Lucene range faceting from O(N) per hit to
O(log(N)) per hit using segment trees; this only really starts to
matter in practice if the number of ranges is over 10 or so. (Mike
McCandless)
Build
* LUCENE-5217: Maven config: get dependencies from Ant+Ivy config; disable

View File

@ -28,6 +28,7 @@ import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.NumericUtils;
/** Represents a range over double values. */
public final class DoubleRange extends Range {
@ -40,10 +41,10 @@ public final class DoubleRange extends Range {
public final boolean maxInclusive;
/** Create a DoubleRange. */
public DoubleRange(String label, double min, boolean minInclusive, double max, boolean maxInclusive) {
public DoubleRange(String label, double minIn, boolean minInclusive, double maxIn, boolean maxInclusive) {
super(label);
this.min = min;
this.max = max;
this.min = minIn;
this.max = maxIn;
this.minInclusive = minInclusive;
this.maxInclusive = maxInclusive;
@ -56,7 +57,7 @@ public final class DoubleRange extends Range {
throw new IllegalArgumentException("min cannot be NaN");
}
if (!minInclusive) {
min = Math.nextUp(min);
minIn = Math.nextUp(minIn);
}
if (Double.isNaN(max)) {
@ -64,17 +65,32 @@ public final class DoubleRange extends Range {
}
if (!maxInclusive) {
// Why no Math.nextDown?
max = Math.nextAfter(max, Double.NEGATIVE_INFINITY);
maxIn = Math.nextAfter(maxIn, Double.NEGATIVE_INFINITY);
}
this.minIncl = min;
this.maxIncl = max;
if (minIn > maxIn) {
failNoMatch();
}
this.minIncl = minIn;
this.maxIncl = maxIn;
}
public boolean accept(double value) {
return value >= minIncl && value <= maxIncl;
}
LongRange toLongRange() {
return new LongRange(label,
NumericUtils.doubleToSortableLong(minIncl), true,
NumericUtils.doubleToSortableLong(maxIncl), true);
}
@Override
public String toString() {
return "DoubleRange(" + minIncl + " to " + maxIncl + ")";
}
/** Returns a new {@link Filter} accepting only documents
* in this range. Note that this filter is not
* efficient: it's a linear scan of all docs, testing

View File

@ -28,14 +28,16 @@ import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.DoubleFieldSource;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource; // javadocs
import org.apache.lucene.util.NumericUtils;
/** {@link Facets} implementation that computes counts for
* dynamic double ranges from a provided {@link
* ValueSource}, using {@link FunctionValues#doubleVal}. Use
* this for dimensions that change in real-time (e.g. a
* relative time based dimension like "Past day", "Past 2
* days", etc.) or that change for each user (e.g. a
* distance dimension like "< 1 km", "< 2 km", etc.).
* days", etc.) or that change for each request (e.g.
* distance from the user's location, "< 1 km", "< 2 km",
* etc.).
*
* <p> If you had indexed your field using {@link
* FloatDocValuesField} then pass {@link FloatFieldSource}
@ -64,6 +66,16 @@ public class DoubleRangeFacetCounts extends RangeFacetCounts {
DoubleRange[] ranges = (DoubleRange[]) this.ranges;
LongRange[] longRanges = new LongRange[ranges.length];
for(int i=0;i<ranges.length;i++) {
DoubleRange range = ranges[i];
longRanges[i] = new LongRange(range.label,
NumericUtils.doubleToSortableLong(range.minIncl), true,
NumericUtils.doubleToSortableLong(range.maxIncl), true);
}
LongRangeCounter counter = new LongRangeCounter(longRanges);
// Compute min & max over all ranges:
double minIncl = Double.POSITIVE_INFINITY;
double maxIncl = Double.NEGATIVE_INFINITY;
@ -72,9 +84,7 @@ public class DoubleRangeFacetCounts extends RangeFacetCounts {
maxIncl = Math.max(maxIncl, range.maxIncl);
}
// TODO: test if this is faster (in the past it was
// faster to do MatchingDocs on the inside) ... see
// patches on LUCENE-4965):
int missingCount = 0;
for (MatchingDocs hits : matchingDocs) {
FunctionValues fv = valueSource.getValues(Collections.emptyMap(), hits.context);
final int length = hits.bits.length();
@ -83,27 +93,15 @@ public class DoubleRangeFacetCounts extends RangeFacetCounts {
while (doc < length && (doc = hits.bits.nextSetBit(doc)) != -1) {
// Skip missing docs:
if (fv.exists(doc)) {
double v = fv.doubleVal(doc);
if (v < minIncl || v > maxIncl) {
doc++;
continue;
}
// TODO: if all ranges are non-overlapping, we
// should instead do a bin-search up front
// (really, a specialized case of the interval
// tree)
// TODO: use interval tree instead of linear search:
for (int j = 0; j < ranges.length; j++) {
if (ranges[j].accept(v)) {
counts[j]++;
}
}
counter.add(NumericUtils.doubleToSortableLong(fv.doubleVal(doc)));
} else {
missingCount++;
}
doc++;
}
}
missingCount += counter.fillCounts(counts);
totCount -= missingCount;
}
}

View File

@ -43,29 +43,46 @@ public final class LongRange extends Range {
// Double/FloatRange too)
/** Create a LongRange. */
public LongRange(String label, long min, boolean minInclusive, long max, boolean maxInclusive) {
public LongRange(String label, long minIn, boolean minInclusive, long maxIn, boolean maxInclusive) {
super(label);
this.min = min;
this.max = max;
this.min = minIn;
this.max = maxIn;
this.minInclusive = minInclusive;
this.maxInclusive = maxInclusive;
if (!minInclusive && min != Long.MAX_VALUE) {
min++;
if (!minInclusive) {
if (minIn != Long.MAX_VALUE) {
minIn++;
} else {
failNoMatch();
}
}
if (!maxInclusive && max != Long.MIN_VALUE) {
max--;
if (!maxInclusive) {
if (maxIn != Long.MIN_VALUE) {
maxIn--;
} else {
failNoMatch();
}
}
this.minIncl = min;
this.maxIncl = max;
if (minIn > maxIn) {
failNoMatch();
}
this.minIncl = minIn;
this.maxIncl = maxIn;
}
public boolean accept(long value) {
return value >= minIncl && value <= maxIncl;
}
@Override
public String toString() {
return "LongRange(" + minIncl + " to " + maxIncl + ")";
}
/** Returns a new {@link Filter} accepting only documents
* in this range. Note that this filter is not
* efficient: it's a linear scan of all docs, testing

View File

@ -0,0 +1,318 @@
package org.apache.lucene.facet;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/** Counts how many times each range was seen;
* per-hit it's just a binary search ({@link #add})
* against the elementary intervals, and in the end we
* rollup back to the original ranges. */
final class LongRangeCounter {
final LongRangeNode root;
final long[] boundaries;
final int[] leafCounts;
// Used during rollup
private int leafUpto;
private int missingCount;
public LongRangeCounter(LongRange[] ranges) {
// Maps all range inclusive endpoints to int flags; 1
// = start of interval, 2 = end of interval. We need to
// track the start vs end case separately because if a
// given point is both, then it must be its own
// elementary interval:
Map<Long,Integer> endsMap = new HashMap<Long,Integer>();
endsMap.put(Long.MIN_VALUE, 1);
endsMap.put(Long.MAX_VALUE, 2);
for(LongRange range : ranges) {
Integer cur = endsMap.get(range.minIncl);
if (cur == null) {
endsMap.put(range.minIncl, 1);
} else {
endsMap.put(range.minIncl, cur.intValue() | 1);
}
cur = endsMap.get(range.maxIncl);
if (cur == null) {
endsMap.put(range.maxIncl, 2);
} else {
endsMap.put(range.maxIncl, cur.intValue() | 2);
}
}
List<Long> endsList = new ArrayList<Long>(endsMap.keySet());
Collections.sort(endsList);
// Build elementaryIntervals (a 1D Venn diagram):
List<InclusiveRange> elementaryIntervals = new ArrayList<InclusiveRange>();
int upto0 = 1;
long v = endsList.get(0);
long prev;
if (endsMap.get(v) == 3) {
elementaryIntervals.add(new InclusiveRange(v, v));
prev = v+1;
} else {
prev = v;
}
while (upto0 < endsList.size()) {
v = endsList.get(upto0);
int flags = endsMap.get(v);
//System.out.println(" v=" + v + " flags=" + flags);
if (flags == 3) {
// This point is both an end and a start; we need to
// separate it:
if (v > prev) {
elementaryIntervals.add(new InclusiveRange(prev, v-1));
}
elementaryIntervals.add(new InclusiveRange(v, v));
prev = v+1;
} else if (flags == 1) {
// This point is only the start of an interval;
// attach it to next interval:
if (v > prev) {
elementaryIntervals.add(new InclusiveRange(prev, v-1));
}
prev = v;
} else {
assert flags == 2;
// This point is only the end of an interval; attach
// it to last interval:
elementaryIntervals.add(new InclusiveRange(prev, v));
prev = v+1;
}
//System.out.println(" ints=" + elementaryIntervals);
upto0++;
}
// Build binary tree on top of intervals:
root = split(0, elementaryIntervals.size(), elementaryIntervals);
// Set outputs, so we know which range to output for
// each node in the tree:
for(int i=0;i<ranges.length;i++) {
root.addOutputs(i, ranges[i]);
}
// Set boundaries (ends of each elementary interval):
boundaries = new long[elementaryIntervals.size()];
for(int i=0;i<boundaries.length;i++) {
boundaries[i] = elementaryIntervals.get(i).end;
}
leafCounts = new int[boundaries.length];
//System.out.println("ranges: " + Arrays.toString(ranges));
//System.out.println("intervals: " + elementaryIntervals);
//System.out.println("boundaries: " + Arrays.toString(boundaries));
//System.out.println("root:\n" + root);
}
public void add(long v) {
//System.out.println("add v=" + v);
// NOTE: this works too, but it's ~6% slower on a simple
// test with a high-freq TermQuery w/ range faceting on
// wikimediumall:
/*
int index = Arrays.binarySearch(boundaries, v);
if (index < 0) {
index = -index-1;
}
leafCounts[index]++;
*/
// Binary search to find matched elementary range; we
// are guaranteed to find a match because the last
// boundary is Long.MAX_VALUE:
int lo = 0;
int hi = boundaries.length - 1;
int count = 0;
while (true) {
int mid = (lo + hi) >>> 1;
//System.out.println(" cycle lo=" + lo + " hi=" + hi + " mid=" + mid + " boundary=" + boundaries[mid] + " to " + boundaries[mid+1]);
if (v <= boundaries[mid]) {
if (mid == 0) {
leafCounts[0]++;
return;
} else {
hi = mid - 1;
}
} else if (v > boundaries[mid+1]) {
lo = mid + 1;
} else {
leafCounts[mid+1]++;
//System.out.println(" incr @ " + (mid+1) + "; now " + leafCounts[mid+1]);
return;
}
}
}
/** Fills counts corresponding to the original input
* ranges, returning the missing count (how many hits
* didn't match any ranges). */
public int fillCounts(int[] counts) {
//System.out.println(" rollup");
missingCount = 0;
leafUpto = 0;
rollup(root, counts, false);
return missingCount;
}
private int rollup(LongRangeNode node, int[] counts, boolean sawOutputs) {
int count;
sawOutputs |= node.outputs != null;
if (node.left != null) {
count = rollup(node.left, counts, sawOutputs);
count += rollup(node.right, counts, sawOutputs);
} else {
// Leaf:
count = leafCounts[leafUpto];
leafUpto++;
if (!sawOutputs) {
// This is a missing count (no output ranges were
// seen "above" us):
missingCount += count;
}
}
if (node.outputs != null) {
for(int rangeIndex : node.outputs) {
counts[rangeIndex] += count;
}
}
//System.out.println(" rollup node=" + node.start + " to " + node.end + ": count=" + count);
return count;
}
private static LongRangeNode split(int start, int end, List<InclusiveRange> elementaryIntervals) {
if (start == end-1) {
// leaf
InclusiveRange range = elementaryIntervals.get(start);
return new LongRangeNode(range.start, range.end, null, null, start);
} else {
int mid = (start + end) >>> 1;
LongRangeNode left = split(start, mid, elementaryIntervals);
LongRangeNode right = split(mid, end, elementaryIntervals);
return new LongRangeNode(left.start, right.end, left, right, -1);
}
}
private static final class InclusiveRange {
public final long start;
public final long end;
public InclusiveRange(long start, long end) {
assert end >= start;
this.start = start;
this.end = end;
}
@Override
public String toString() {
return start + " to " + end;
}
}
/** Holds one node of the segment tree. */
public static final class LongRangeNode {
final LongRangeNode left;
final LongRangeNode right;
// Our range, inclusive:
final long start;
final long end;
// If we are a leaf, the index into elementary ranges that
// we point to:
final int leafIndex;
// Which range indices to output when a query goes
// through this node:
List<Integer> outputs;
public LongRangeNode(long start, long end, LongRangeNode left, LongRangeNode right, int leafIndex) {
this.start = start;
this.end = end;
this.left = left;
this.right = right;
this.leafIndex = leafIndex;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
toString(sb, 0);
return sb.toString();
}
static void indent(StringBuilder sb, int depth) {
for(int i=0;i<depth;i++) {
sb.append(" ");
}
}
/** Recursively assigns range outputs to each node. */
void addOutputs(int index, LongRange range) {
if (start >= range.minIncl && end <= range.maxIncl) {
// Our range is fully included in the incoming
// range; add to our output list:
if (outputs == null) {
outputs = new ArrayList<Integer>();
}
outputs.add(index);
} else if (left != null) {
assert right != null;
// Recurse:
left.addOutputs(index, range);
right.addOutputs(index, range);
}
}
void toString(StringBuilder sb, int depth) {
indent(sb, depth);
if (left == null) {
assert right == null;
sb.append("leaf: " + start + " to " + end);
} else {
sb.append("node: " + start + " to " + end);
}
if (outputs != null) {
sb.append(" outputs=");
sb.append(outputs);
}
sb.append('\n');
if (left != null) {
assert right != null;
left.toString(sb, depth+1);
right.toString(sb, depth+1);
}
}
}
}

View File

@ -31,8 +31,9 @@ import org.apache.lucene.queries.function.valuesource.LongFieldSource;
* using {@link FunctionValues#longVal}. Use
* this for dimensions that change in real-time (e.g. a
* relative time based dimension like "Past day", "Past 2
* days", etc.) or that change for each user (e.g. a
* distance dimension like "< 1 km", "< 2 km", etc.).
* days", etc.) or that change for each request (e.g.
* distance from the user's location, "< 1 km", "< 2 km",
* etc.).
*
* @lucene.experimental */
public class LongRangeFacetCounts extends RangeFacetCounts {
@ -62,9 +63,9 @@ public class LongRangeFacetCounts extends RangeFacetCounts {
maxIncl = Math.max(maxIncl, range.maxIncl);
}
// TODO: test if this is faster (in the past it was
// faster to do MatchingDocs on the inside) ... see
// patches on LUCENE-4965):
LongRangeCounter counter = new LongRangeCounter(ranges);
int missingCount = 0;
for (MatchingDocs hits : matchingDocs) {
FunctionValues fv = valueSource.getValues(Collections.emptyMap(), hits.context);
final int length = hits.bits.length();
@ -73,27 +74,20 @@ public class LongRangeFacetCounts extends RangeFacetCounts {
while (doc < length && (doc = hits.bits.nextSetBit(doc)) != -1) {
// Skip missing docs:
if (fv.exists(doc)) {
long v = fv.longVal(doc);
if (v < minIncl || v > maxIncl) {
doc++;
continue;
}
// TODO: if all ranges are non-overlapping, we
// should instead do a bin-search up front
// (really, a specialized case of the interval
// tree)
// TODO: use interval tree instead of linear search:
for (int j = 0; j < ranges.length; j++) {
if (ranges[j].accept(v)) {
counts[j]++;
}
}
counter.add(fv.longVal(doc));
} else {
missingCount++;
}
doc++;
}
}
int x = counter.fillCounts(counts);
missingCount += x;
//System.out.println("totCount " + totCount + " missingCount " + counter.missingCount);
totCount -= missingCount;
}
}

View File

@ -24,6 +24,13 @@ public abstract class Range {
public final String label;
protected Range(String label) {
if (label == null) {
throw new NullPointerException("label cannot be null");
}
this.label = label;
}
protected void failNoMatch() {
throw new IllegalArgumentException("range \"" + label + "\" matches nothing");
}
}

View File

@ -39,9 +39,6 @@ abstract class RangeFacetCounts extends Facets {
counts = new int[ranges.length];
}
// nocommit all args are ... unused ... this doesn't "fit"
// very well:
@Override
public FacetResult getTopChildren(int topN, String dim, String... path) {
if (dim.equals(field) == false) {

View File

@ -62,6 +62,8 @@ public class TestRangeFacetCounts extends FacetTestCase {
field.setLongValue(l);
w.addDocument(doc);
}
// Also add Long.MAX_VALUE
field.setLongValue(Long.MAX_VALUE);
w.addDocument(doc);
@ -78,9 +80,107 @@ public class TestRangeFacetCounts extends FacetTestCase {
new LongRange("over 90", 90L, false, 100L, false),
new LongRange("90 or above", 90L, true, 100L, false),
new LongRange("over 1000", 1000L, false, Long.MAX_VALUE, true));
FacetResult result = facets.getTopChildren(10, "field");
assertEquals("dim=field path=[] value=22 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (1)\n",
result.toString());
r.close();
d.close();
}
public void testUselessRange() {
try {
new LongRange("useless", 7, true, 6, true);
fail("did not hit expected exception");
} catch (IllegalArgumentException iae) {
// expected
}
try {
new LongRange("useless", 7, true, 7, false);
fail("did not hit expected exception");
} catch (IllegalArgumentException iae) {
// expected
}
try {
new DoubleRange("useless", 7.0, true, 6.0, true);
fail("did not hit expected exception");
} catch (IllegalArgumentException iae) {
// expected
}
try {
new DoubleRange("useless", 7.0, true, 7.0, false);
fail("did not hit expected exception");
} catch (IllegalArgumentException iae) {
// expected
}
}
public void testLongMinMax() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
Document doc = new Document();
NumericDocValuesField field = new NumericDocValuesField("field", 0L);
doc.add(field);
field.setLongValue(Long.MIN_VALUE);
w.addDocument(doc);
field.setLongValue(0);
w.addDocument(doc);
field.setLongValue(Long.MAX_VALUE);
w.addDocument(doc);
IndexReader r = w.getReader();
w.close();
FacetsCollector fc = new FacetsCollector();
IndexSearcher s = newSearcher(r);
s.search(new MatchAllDocsQuery(), fc);
Facets facets = new LongRangeFacetCounts("field", fc,
new LongRange("min", Long.MIN_VALUE, true, Long.MIN_VALUE, true),
new LongRange("max", Long.MAX_VALUE, true, Long.MAX_VALUE, true),
new LongRange("all0", Long.MIN_VALUE, true, Long.MAX_VALUE, true),
new LongRange("all1", Long.MIN_VALUE, false, Long.MAX_VALUE, true),
new LongRange("all2", Long.MIN_VALUE, true, Long.MAX_VALUE, false),
new LongRange("all3", Long.MIN_VALUE, false, Long.MAX_VALUE, false));
FacetResult result = facets.getTopChildren(10, "field");
assertEquals("dim=field path=[] value=3 childCount=6\n min (1)\n max (1)\n all0 (3)\n all1 (2)\n all2 (2)\n all3 (1)\n",
result.toString());
r.close();
d.close();
}
public void testOverlappedEndStart() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
Document doc = new Document();
NumericDocValuesField field = new NumericDocValuesField("field", 0L);
doc.add(field);
for(long l=0;l<100;l++) {
field.setLongValue(l);
w.addDocument(doc);
}
field.setLongValue(Long.MAX_VALUE);
w.addDocument(doc);
IndexReader r = w.getReader();
w.close();
FacetsCollector fc = new FacetsCollector();
IndexSearcher s = newSearcher(r);
s.search(new MatchAllDocsQuery(), fc);
Facets facets = new LongRangeFacetCounts("field", fc,
new LongRange("0-10", 0L, true, 10L, true),
new LongRange("10-20", 10L, true, 20L, true),
new LongRange("20-30", 20L, true, 30L, true),
new LongRange("30-40", 30L, true, 40L, true));
FacetResult result = facets.getTopChildren(10, "field");
assertEquals("dim=field path=[] value=101 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (1)\n",
assertEquals("dim=field path=[] value=41 childCount=4\n 0-10 (11)\n 10-20 (11)\n 20-30 (11)\n 30-40 (11)\n",
result.toString());
r.close();
@ -162,7 +262,7 @@ public class TestRangeFacetCounts extends FacetTestCase {
assertEquals(100, dsr.hits.totalHits);
assertEquals("dim=dim path=[] value=100 childCount=2\n b (75)\n a (25)\n", dsr.facets.getTopChildren(10, "dim").toString());
assertEquals("dim=field path=[] value=100 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n",
assertEquals("dim=field path=[] value=21 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n",
dsr.facets.getTopChildren(10, "field").toString());
// Second search, drill down on dim=b:
@ -172,7 +272,7 @@ public class TestRangeFacetCounts extends FacetTestCase {
assertEquals(75, dsr.hits.totalHits);
assertEquals("dim=dim path=[] value=100 childCount=2\n b (75)\n a (25)\n", dsr.facets.getTopChildren(10, "dim").toString());
assertEquals("dim=field path=[] value=75 childCount=5\n less than 10 (7)\n less than or equal to 10 (8)\n over 90 (7)\n 90 or above (8)\n over 1000 (0)\n",
assertEquals("dim=field path=[] value=16 childCount=5\n less than 10 (7)\n less than or equal to 10 (8)\n over 90 (7)\n 90 or above (8)\n over 1000 (0)\n",
dsr.facets.getTopChildren(10, "field").toString());
// Third search, drill down on "less than or equal to 10":
@ -182,7 +282,7 @@ public class TestRangeFacetCounts extends FacetTestCase {
assertEquals(11, dsr.hits.totalHits);
assertEquals("dim=dim path=[] value=11 childCount=2\n b (8)\n a (3)\n", dsr.facets.getTopChildren(10, "dim").toString());
assertEquals("dim=field path=[] value=100 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n",
assertEquals("dim=field path=[] value=21 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n",
dsr.facets.getTopChildren(10, "field").toString());
IOUtils.close(tw, tr, td, w, r, d);
}
@ -211,7 +311,7 @@ public class TestRangeFacetCounts extends FacetTestCase {
new DoubleRange("90 or above", 90.0, true, 100.0, false),
new DoubleRange("over 1000", 1000.0, false, Double.POSITIVE_INFINITY, false));
assertEquals("dim=field path=[] value=100 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n",
assertEquals("dim=field path=[] value=21 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n",
facets.getTopChildren(10, "field").toString());
IOUtils.close(w, r, d);
@ -242,7 +342,7 @@ public class TestRangeFacetCounts extends FacetTestCase {
new DoubleRange("90 or above", 90.0f, true, 100.0f, false),
new DoubleRange("over 1000", 1000.0f, false, Double.POSITIVE_INFINITY, false));
assertEquals("dim=field path=[] value=100 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n",
assertEquals("dim=field path=[] value=21 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n",
facets.getTopChildren(10, "field").toString());
IOUtils.close(w, r, d);
@ -253,6 +353,9 @@ public class TestRangeFacetCounts extends FacetTestCase {
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
int numDocs = atLeast(1000);
if (VERBOSE) {
System.out.println("TEST: numDocs=" + numDocs);
}
long[] values = new long[numDocs];
for(int i=0;i<numDocs;i++) {
Document doc = new Document();
@ -272,20 +375,53 @@ public class TestRangeFacetCounts extends FacetTestCase {
if (VERBOSE) {
System.out.println("TEST: iter=" + iter);
}
int numRange = _TestUtil.nextInt(random(), 1, 5);
int numRange = _TestUtil.nextInt(random(), 1, 100);
LongRange[] ranges = new LongRange[numRange];
int[] expectedCounts = new int[numRange];
for(int rangeID=0;rangeID<numRange;rangeID++) {
long min = random().nextLong();
long max = random().nextLong();
long min;
if (rangeID > 0 && random().nextInt(10) == 7) {
// Use an existing boundary:
LongRange prevRange = ranges[random().nextInt(rangeID)];
if (random().nextBoolean()) {
min = prevRange.min;
} else {
min = prevRange.max;
}
} else {
min = random().nextLong();
}
long max;
if (rangeID > 0 && random().nextInt(10) == 7) {
// Use an existing boundary:
LongRange prevRange = ranges[random().nextInt(rangeID)];
if (random().nextBoolean()) {
max = prevRange.min;
} else {
max = prevRange.max;
}
} else {
max = random().nextLong();
}
if (min > max) {
long x = min;
min = max;
max = x;
}
boolean minIncl = random().nextBoolean();
boolean maxIncl = random().nextBoolean();
boolean minIncl;
boolean maxIncl;
if (min == max) {
minIncl = true;
maxIncl = true;
} else {
minIncl = random().nextBoolean();
maxIncl = random().nextBoolean();
}
ranges[rangeID] = new LongRange("r" + rangeID, min, minIncl, max, maxIncl);
if (VERBOSE) {
System.out.println(" range " + rangeID + ": " + ranges[rangeID]);
}
// Do "slow but hopefully correct" computation of
// expected count:
@ -360,15 +496,46 @@ public class TestRangeFacetCounts extends FacetTestCase {
DoubleRange[] ranges = new DoubleRange[numRange];
int[] expectedCounts = new int[numRange];
for(int rangeID=0;rangeID<numRange;rangeID++) {
double min = random().nextDouble();
double max = random().nextDouble();
double min;
if (rangeID > 0 && random().nextInt(10) == 7) {
// Use an existing boundary:
DoubleRange prevRange = ranges[random().nextInt(rangeID)];
if (random().nextBoolean()) {
min = prevRange.min;
} else {
min = prevRange.max;
}
} else {
min = random().nextDouble();
}
double max;
if (rangeID > 0 && random().nextInt(10) == 7) {
// Use an existing boundary:
DoubleRange prevRange = ranges[random().nextInt(rangeID)];
if (random().nextBoolean()) {
max = prevRange.min;
} else {
max = prevRange.max;
}
} else {
max = random().nextDouble();
}
if (min > max) {
double x = min;
min = max;
max = x;
}
boolean minIncl = random().nextBoolean();
boolean maxIncl = random().nextBoolean();
boolean minIncl;
boolean maxIncl;
if (min == max) {
minIncl = true;
maxIncl = true;
} else {
minIncl = random().nextBoolean();
maxIncl = random().nextBoolean();
}
ranges[rangeID] = new DoubleRange("r" + rangeID, min, minIncl, max, maxIncl);
// Do "slow but hopefully correct" computation of
@ -444,15 +611,46 @@ public class TestRangeFacetCounts extends FacetTestCase {
DoubleRange[] ranges = new DoubleRange[numRange];
int[] expectedCounts = new int[numRange];
for(int rangeID=0;rangeID<numRange;rangeID++) {
double min = random().nextDouble();
double max = random().nextDouble();
double min;
if (rangeID > 0 && random().nextInt(10) == 7) {
// Use an existing boundary:
DoubleRange prevRange = ranges[random().nextInt(rangeID)];
if (random().nextBoolean()) {
min = prevRange.min;
} else {
min = prevRange.max;
}
} else {
min = random().nextDouble();
}
double max;
if (rangeID > 0 && random().nextInt(10) == 7) {
// Use an existing boundary:
DoubleRange prevRange = ranges[random().nextInt(rangeID)];
if (random().nextBoolean()) {
max = prevRange.min;
} else {
max = prevRange.max;
}
} else {
max = random().nextDouble();
}
if (min > max) {
double x = min;
min = max;
max = x;
}
boolean minIncl = random().nextBoolean();
boolean maxIncl = random().nextBoolean();
boolean minIncl;
boolean maxIncl;
if (min == max) {
minIncl = true;
maxIncl = true;
} else {
minIncl = random().nextBoolean();
maxIncl = random().nextBoolean();
}
ranges[rangeID] = new DoubleRange("r" + rangeID, min, minIncl, max, maxIncl);
// Do "slow but hopefully correct" computation of
@ -531,7 +729,7 @@ public class TestRangeFacetCounts extends FacetTestCase {
new LongRange("90 or above", 90L, true, 100L, false),
new LongRange("over 1000", 1000L, false, Long.MAX_VALUE, false));
assertEquals("dim=field path=[] value=100 childCount=5\n less than 10 (8)\n less than or equal to 10 (8)\n over 90 (8)\n 90 or above (8)\n over 1000 (0)\n",
assertEquals("dim=field path=[] value=16 childCount=5\n less than 10 (8)\n less than or equal to 10 (8)\n over 90 (8)\n 90 or above (8)\n over 1000 (0)\n",
facets.getTopChildren(10, "field").toString());
IOUtils.close(w, r, d);