mirror of https://github.com/apache/lucene.git
LUCENE-6422: New spatial PackedQuadPrefixTree. Thanks Nick!
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1675538 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
David Wayne Smiley
parent
88831572bf
commit
9d1c2019a8
|
|
@ -73,6 +73,11 @@ New Features
|
|||
* LUCENE-6423: New LimitTokenOffsetFilter that limits tokens to those before
|
||||
a configured maximum start offset. (David Smiley)
|
||||
|
||||
* LUCENE-6422: New spatial PackedQuadPrefixTree, a generally more efficient
|
||||
choice than QuadPrefixTree, especially for high precision shapes.
|
||||
When used, you should typically disable RPT's pruneLeafyBranches option.
|
||||
(Nick Knize, David Smiley)
|
||||
|
||||
Optimizations
|
||||
|
||||
* LUCENE-6379: IndexWriter.deleteDocuments(Query...) now detects if
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import org.apache.lucene.document.Field;
|
|||
import org.apache.lucene.spatial.SpatialStrategy;
|
||||
import org.apache.lucene.spatial.composite.CompositeSpatialStrategy;
|
||||
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
|
||||
import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree;
|
||||
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
|
||||
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTreeFactory;
|
||||
import org.apache.lucene.spatial.serialized.SerializedDVStrategy;
|
||||
|
|
@ -111,7 +112,13 @@ public class SpatialDocMaker extends DocMaker {
|
|||
|
||||
RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategy(grid, spatialField);
|
||||
strategy.setPointsOnly(config.get("spatial.docPointsOnly", false));
|
||||
strategy.setPruneLeafyBranches(config.get("spatial.pruneLeafyBranches", true));
|
||||
final boolean pruneLeafyBranches = config.get("spatial.pruneLeafyBranches", true);
|
||||
if (grid instanceof PackedQuadPrefixTree) {
|
||||
((PackedQuadPrefixTree) grid).setPruneLeafyBranches(pruneLeafyBranches);
|
||||
strategy.setPruneLeafyBranches(false);//always leave it to packed grid, even though it isn't the same
|
||||
} else {
|
||||
strategy.setPruneLeafyBranches(pruneLeafyBranches);
|
||||
}
|
||||
|
||||
int prefixGridScanLevel = config.get("query.spatial.prefixGridScanLevel", -4);
|
||||
if (prefixGridScanLevel < 0)
|
||||
|
|
|
|||
|
|
@ -92,9 +92,14 @@ public class RecursivePrefixTreeStrategy extends PrefixTreeStrategy {
|
|||
return pruneLeafyBranches;
|
||||
}
|
||||
|
||||
/** An optional hint affecting non-point shapes: it will
|
||||
* simplify/aggregate sets of complete leaves in a cell to its parent, resulting in ~20-25%
|
||||
* fewer indexed cells. However, it will likely be removed in the future. (default=true)
|
||||
/**
|
||||
* An optional hint affecting non-point shapes: it will
|
||||
* prune away a complete set sibling leaves to their parent (recursively), resulting in ~20-50%
|
||||
* fewer indexed cells, and consequently that much less disk and that much faster indexing.
|
||||
* So if it's a quad tree and all 4 sub-cells are there marked as a leaf, then they will be
|
||||
* removed (pruned) and the parent is marked as a leaf instead. This occurs recursively on up. Unfortunately, the
|
||||
* current implementation will buffer all cells to do this, so consider disabling for high precision (low distErrPct)
|
||||
* shapes. (default=true)
|
||||
*/
|
||||
public void setPruneLeafyBranches(boolean pruneLeafyBranches) {
|
||||
this.pruneLeafyBranches = pruneLeafyBranches;
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ public abstract class CellIterator implements Iterator<Cell> {
|
|||
}
|
||||
|
||||
@Override
|
||||
public final Cell next() {
|
||||
public Cell next() {
|
||||
if (nextCell == null) {
|
||||
if (!hasNext())
|
||||
throw new NoSuchElementException();
|
||||
|
|
|
|||
|
|
@ -36,9 +36,9 @@ public abstract class LegacyCell implements Cell {
|
|||
private static final byte LEAF_BYTE = '+';//NOTE: must sort before letters & numbers
|
||||
|
||||
//Arguably we could simply use a BytesRef, using an extra Object.
|
||||
private byte[] bytes;//generally bigger to potentially hold a leaf
|
||||
private int b_off;
|
||||
private int b_len;//doesn't reflect leaf; same as getLevel()
|
||||
protected byte[] bytes;//generally bigger to potentially hold a leaf
|
||||
protected int b_off;
|
||||
protected int b_len;//doesn't reflect leaf; same as getLevel()
|
||||
|
||||
protected boolean isLeaf;
|
||||
|
||||
|
|
@ -68,7 +68,7 @@ public abstract class LegacyCell implements Cell {
|
|||
readLeafAdjust();
|
||||
}
|
||||
|
||||
private void readLeafAdjust() {
|
||||
protected void readLeafAdjust() {
|
||||
isLeaf = (b_len > 0 && bytes[b_off + b_len - 1] == LEAF_BYTE);
|
||||
if (isLeaf)
|
||||
b_len--;
|
||||
|
|
@ -76,18 +76,6 @@ public abstract class LegacyCell implements Cell {
|
|||
isLeaf = true;
|
||||
}
|
||||
|
||||
// @Override
|
||||
// public void copyFrom(Cell source) {
|
||||
// LegacyCell src = (LegacyCell) source;
|
||||
// shapeRel = src.shapeRel;
|
||||
// shape = src.shape;
|
||||
// isLeaf = src.isLeaf;
|
||||
// //we don't actually copy the bytes because in LegacyCell the bytes aren't modified. (leaf byte doesn't count)
|
||||
// bytes = src.bytes;
|
||||
// b_off = src.b_off;
|
||||
// b_len = src.b_len;
|
||||
// }
|
||||
|
||||
protected abstract SpatialPrefixTree getGrid();
|
||||
|
||||
protected abstract int getMaxLevels();
|
||||
|
|
@ -214,7 +202,7 @@ public abstract class LegacyCell implements Cell {
|
|||
|
||||
/** Copied from {@link BytesRef#compareTo(BytesRef)}.
|
||||
* This is to avoid creating a BytesRef. */
|
||||
private static int compare(byte[] aBytes, int aUpto, int a_length, byte[] bBytes, int bUpto, int b_length) {
|
||||
protected static int compare(byte[] aBytes, int aUpto, int a_length, byte[] bBytes, int bUpto, int b_length) {
|
||||
final int aStop = aUpto + Math.min(a_length, b_length);
|
||||
while(aUpto < aStop) {
|
||||
int aByte = aBytes[aUpto++] & 0xff;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,460 @@
|
|||
package org.apache.lucene.spatial.prefix.tree;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
|
||||
import com.spatial4j.core.context.SpatialContext;
|
||||
import com.spatial4j.core.shape.Point;
|
||||
import com.spatial4j.core.shape.Rectangle;
|
||||
import com.spatial4j.core.shape.Shape;
|
||||
import com.spatial4j.core.shape.SpatialRelation;
|
||||
import com.spatial4j.core.shape.impl.RectangleImpl;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
|
||||
/**
|
||||
* Uses a compact binary representation of 8 bytes to encode a spatial quad trie.
|
||||
*
|
||||
* The binary representation is as follows:
|
||||
* <pre>
|
||||
* CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDL
|
||||
*
|
||||
* Where C = Cell bits (2 per quad)
|
||||
* D = Depth bits (5 with max of 29 levels)
|
||||
* L = isLeaf bit
|
||||
* </pre>
|
||||
*
|
||||
* It includes a built-in "pruneLeafyBranches" setting (true by default) similar to
|
||||
* {@link org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy#setPruneLeafyBranches(boolean)} although
|
||||
* this one only prunes at the target detail level (where it has the most effect). Usually you should disable RPT's
|
||||
* prune, since it is very memory in-efficient.
|
||||
*
|
||||
* @lucene.experimental
|
||||
*/
|
||||
public class PackedQuadPrefixTree extends QuadPrefixTree {
|
||||
public static final int MAX_LEVELS_POSSIBLE = 29;
|
||||
protected static final byte[] QUAD = new byte[] {0x00, 0x01, 0x02, 0x03};
|
||||
|
||||
protected boolean leafyPrune = true;
|
||||
|
||||
/**
|
||||
* Factory for creating {@link PackedQuadPrefixTree} instances with useful defaults.
|
||||
*/
|
||||
public static class Factory extends QuadPrefixTree.Factory {
|
||||
@Override
|
||||
protected SpatialPrefixTree newSPT() {
|
||||
return new PackedQuadPrefixTree(ctx, maxLevels != null ? maxLevels : MAX_LEVELS_POSSIBLE);
|
||||
}
|
||||
}
|
||||
|
||||
public PackedQuadPrefixTree(SpatialContext ctx, int maxLevels) {
|
||||
super(ctx, maxLevels);
|
||||
if (maxLevels > MAX_LEVELS_POSSIBLE) {
|
||||
throw new IllegalArgumentException("maxLevels of " + maxLevels + " exceeds limit of " + MAX_LEVELS_POSSIBLE);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getClass().getSimpleName() + "(maxLevels:" + maxLevels + ",ctx:" + ctx + ",prune:" + leafyPrune + ")";
|
||||
}
|
||||
|
||||
@Override
|
||||
public Cell getWorldCell() {
|
||||
return new PackedQuadCell(0x0L);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Cell getCell(Point p, int level) {
|
||||
List<Cell> cells = new ArrayList<>(1);
|
||||
build(xmid, ymid, 0, cells, 0x0L, ctx.makePoint(p.getX(), p.getY()), level);
|
||||
return cells.get(0);//note cells could be longer if p on edge
|
||||
}
|
||||
|
||||
protected void build(double x, double y, int level, List<Cell> matches, long term, Shape shape, int maxLevel) {
|
||||
double w = levelW[level] / 2;
|
||||
double h = levelH[level] / 2;
|
||||
|
||||
// Z-Order
|
||||
// http://en.wikipedia.org/wiki/Z-order_%28curve%29
|
||||
checkBattenberg(QUAD[0], x - w, y + h, level, matches, term, shape, maxLevel);
|
||||
checkBattenberg(QUAD[1], x + w, y + h, level, matches, term, shape, maxLevel);
|
||||
checkBattenberg(QUAD[2], x - w, y - h, level, matches, term, shape, maxLevel);
|
||||
checkBattenberg(QUAD[3], x + w, y - h, level, matches, term, shape, maxLevel);
|
||||
}
|
||||
|
||||
protected void checkBattenberg(byte quad, double cx, double cy, int level, List<Cell> matches,
|
||||
long term, Shape shape, int maxLevel) {
|
||||
// short-circuit if we find a match for the point (no need to continue recursion)
|
||||
if (shape instanceof Point && !matches.isEmpty())
|
||||
return;
|
||||
double w = levelW[level] / 2;
|
||||
double h = levelH[level] / 2;
|
||||
|
||||
SpatialRelation v = shape.relate(ctx.makeRectangle(cx - w, cx + w, cy - h, cy + h));
|
||||
|
||||
if (SpatialRelation.DISJOINT == v) {
|
||||
return;
|
||||
}
|
||||
|
||||
// set bits for next level
|
||||
term |= (((long)(quad))<<(64-(++level<<1)));
|
||||
// increment level
|
||||
term = ((term>>>1)+1)<<1;
|
||||
|
||||
if (SpatialRelation.CONTAINS == v || (level >= maxLevel)) {
|
||||
matches.add(new PackedQuadCell(term, v.transpose()));
|
||||
} else {// SpatialRelation.WITHIN, SpatialRelation.INTERSECTS
|
||||
build(cx, cy, level, matches, term, shape, maxLevel);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Cell readCell(BytesRef term, Cell scratch) {
|
||||
PackedQuadCell cell = (PackedQuadCell) scratch;
|
||||
if (cell == null)
|
||||
cell = (PackedQuadCell) getWorldCell();
|
||||
cell.readCell(term);
|
||||
return cell;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CellIterator getTreeCellIterator(Shape shape, int detailLevel) {
|
||||
if (detailLevel > maxLevels) {
|
||||
throw new IllegalArgumentException("detailLevel:" + detailLevel +" exceed max: " + maxLevels);
|
||||
}
|
||||
return new PrefixTreeIterator(shape, (short) detailLevel);
|
||||
}
|
||||
|
||||
public boolean isPruneLeafyBranches() {
|
||||
return leafyPrune;
|
||||
}
|
||||
|
||||
/** Like {@link org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy#setPruneLeafyBranches(boolean)}
|
||||
* but more memory efficient and only applies to the detailLevel, where it has the most effect. */
|
||||
public void setPruneLeafyBranches( boolean pruneLeafyBranches ) {
|
||||
this.leafyPrune = pruneLeafyBranches;
|
||||
}
|
||||
|
||||
/** See binary representation in the javadocs of {@link PackedQuadPrefixTree}. */
|
||||
protected class PackedQuadCell extends QuadCell {
|
||||
private long term;
|
||||
|
||||
PackedQuadCell(long term) {
|
||||
super(null, 0, 0);
|
||||
this.term = term;
|
||||
this.b_off = 0;
|
||||
this.bytes = longToByteArray(this.term);
|
||||
this.b_len = 8;
|
||||
readLeafAdjust();
|
||||
}
|
||||
|
||||
PackedQuadCell(long term, SpatialRelation shapeRel) {
|
||||
this(term);
|
||||
this.shapeRel = shapeRel;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void readCell(BytesRef bytes) {
|
||||
shapeRel = null;
|
||||
shape = null;
|
||||
this.bytes = bytes.bytes;
|
||||
this.b_off = bytes.offset;
|
||||
this.b_len = (short) bytes.length;
|
||||
this.term = longFromByteArray(this.bytes, bytes.offset);
|
||||
readLeafAdjust();
|
||||
}
|
||||
|
||||
private final int getShiftForLevel(final int level) {
|
||||
return 64 - (level<<1);
|
||||
}
|
||||
|
||||
public boolean isEnd(final int level, final int shift) {
|
||||
return (term != 0x0L && ((((0x1L<<(level<<1))-1)-(term>>>shift)) == 0x0L));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the next cell in the tree without using recursion. descend parameter requests traversal to the child nodes,
|
||||
* setting this to false will step to the next sibling.
|
||||
* Note: This complies with lexicographical ordering, once you've moved to the next sibling there is no backtracking.
|
||||
*/
|
||||
public PackedQuadCell nextCell(boolean descend) {
|
||||
final int level = getLevel();
|
||||
final int shift = getShiftForLevel(level);
|
||||
// base case: can't go further
|
||||
if ( (!descend && isEnd(level, shift)) || isEnd(maxLevels, getShiftForLevel(maxLevels))) {
|
||||
return null;
|
||||
}
|
||||
long newTerm;
|
||||
final boolean isLeaf = (term&0x1L)==0x1L;
|
||||
// if descend requested && we're not at the maxLevel
|
||||
if ((descend && !isLeaf && (level != maxLevels)) || level == 0) {
|
||||
// simple case: increment level bits (next level)
|
||||
newTerm = ((term>>>1)+0x1L)<<1;
|
||||
} else { // we're not descending or we can't descend
|
||||
newTerm = term + (0x1L<<shift);
|
||||
// we're at the last sibling...force descend
|
||||
if (((term>>>shift)&0x3L) == 0x3L) {
|
||||
// adjust level for number popping up
|
||||
newTerm = ((newTerm>>>1) - (Long.numberOfTrailingZeros(newTerm>>>shift)>>>1))<<1;
|
||||
}
|
||||
}
|
||||
return new PackedQuadCell(newTerm);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void readLeafAdjust() {
|
||||
isLeaf = ((0x1L)&term) == 0x1L;
|
||||
if (getLevel() == getMaxLevels()) {
|
||||
isLeaf = true;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getTokenBytesWithLeaf(BytesRef result) {
|
||||
if (isLeaf) {
|
||||
term |= 0x1L;
|
||||
}
|
||||
return getTokenBytesNoLeaf(result);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getTokenBytesNoLeaf(BytesRef result) {
|
||||
if (result == null)
|
||||
return new BytesRef(bytes, b_off, b_len);
|
||||
result.bytes = longToByteArray(this.term);
|
||||
result.offset = 0;
|
||||
result.length = result.bytes.length;
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareToNoLeaf(Cell fromCell) {
|
||||
PackedQuadCell b = (PackedQuadCell) fromCell;
|
||||
final long thisTerm = (((0x1L)&term) == 0x1L) ? term-1 : term;
|
||||
final long fromTerm = (((0x1L)&b.term) == 0x1L) ? b.term-1 : b.term;
|
||||
final int result = Long.compareUnsigned(thisTerm, fromTerm);
|
||||
assert Math.signum(result)
|
||||
== Math.signum(compare(longToByteArray(thisTerm), 0, 8, longToByteArray(fromTerm), 0, 8)); // TODO remove
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getLevel() {
|
||||
int l = (int)((term >>> 1)&0x1FL);
|
||||
return l;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<Cell> getSubCells() {
|
||||
List<Cell> cells = new ArrayList<>(4);
|
||||
PackedQuadCell pqc = (new PackedQuadCell(((term&0x1)==0x1) ? this.term-1 : this.term))
|
||||
.nextCell(true);
|
||||
cells.add(pqc);
|
||||
cells.add((pqc = pqc.nextCell(false)));
|
||||
cells.add((pqc = pqc.nextCell(false)));
|
||||
cells.add(pqc.nextCell(false));
|
||||
return cells;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected QuadCell getSubCell(Point p) {
|
||||
return (PackedQuadCell) PackedQuadPrefixTree.this.getCell(p, getLevel() + 1);//not performant!
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isPrefixOf(Cell c) {
|
||||
PackedQuadCell cell = (PackedQuadCell)c;
|
||||
return (this.term == 0x0L) || isInternalPrefix(cell);
|
||||
}
|
||||
|
||||
protected boolean isInternalPrefix(PackedQuadCell c) {
|
||||
final int shift = 64 - (getLevel()<<1);
|
||||
return ((term>>>shift)-(c.term>>>shift)) == 0x0L;
|
||||
}
|
||||
|
||||
protected long concat(byte postfix) {
|
||||
// extra leaf bit
|
||||
return this.term | (((long)(postfix))<<((getMaxLevels()-getLevel()<<1)+6));
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a bounding box shape out of the encoded cell
|
||||
*/
|
||||
@Override
|
||||
protected Rectangle makeShape() {
|
||||
double xmin = PackedQuadPrefixTree.this.xmin;
|
||||
double ymin = PackedQuadPrefixTree.this.ymin;
|
||||
int level = getLevel();
|
||||
|
||||
byte b;
|
||||
for (short l=0, i=1; l<level; ++l, ++i) {
|
||||
b = (byte) ((term>>>(64-(i<<1))) & 0x3L);
|
||||
|
||||
switch (b) {
|
||||
case 0x00:
|
||||
ymin += levelH[l];
|
||||
break;
|
||||
case 0x01:
|
||||
xmin += levelW[l];
|
||||
ymin += levelH[l];
|
||||
break;
|
||||
case 0x02:
|
||||
break;//nothing really
|
||||
case 0x03:
|
||||
xmin += levelW[l];
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException("unexpected quadrant");
|
||||
}
|
||||
}
|
||||
|
||||
double width, height;
|
||||
if (level > 0) {
|
||||
width = levelW[level - 1];
|
||||
height = levelH[level - 1];
|
||||
} else {
|
||||
width = gridW;
|
||||
height = gridH;
|
||||
}
|
||||
return new RectangleImpl(xmin, xmin + width, ymin, ymin + height, ctx);
|
||||
}
|
||||
|
||||
private long fromBytes(byte b1, byte b2, byte b3, byte b4, byte b5, byte b6, byte b7, byte b8) {
|
||||
return ((long)b1 & 255L) << 56 | ((long)b2 & 255L) << 48 | ((long)b3 & 255L) << 40
|
||||
| ((long)b4 & 255L) << 32 | ((long)b5 & 255L) << 24 | ((long)b6 & 255L) << 16
|
||||
| ((long)b7 & 255L) << 8 | (long)b8 & 255L;
|
||||
}
|
||||
|
||||
private byte[] longToByteArray(long value) {
|
||||
byte[] result = new byte[8];
|
||||
for(int i = 7; i >= 0; --i) {
|
||||
result[i] = (byte)((int)(value & 255L));
|
||||
value >>= 8;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private long longFromByteArray(byte[] bytes, int ofs) {
|
||||
assert bytes.length >= 8;
|
||||
return fromBytes(bytes[0+ofs], bytes[1+ofs], bytes[2+ofs], bytes[3+ofs],
|
||||
bytes[4+ofs], bytes[5+ofs], bytes[6+ofs], bytes[7+ofs]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Used for debugging, this will print the bits of the cell
|
||||
*/
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder s = new StringBuilder(64);
|
||||
final int numberOfLeadingZeros = Long.numberOfLeadingZeros(term);
|
||||
for (int i = 0; i < numberOfLeadingZeros; i++) {
|
||||
s.append('0');
|
||||
}
|
||||
if (term != 0)
|
||||
s.append(Long.toBinaryString(term));
|
||||
return s.toString();
|
||||
}
|
||||
} // PackedQuadCell
|
||||
|
||||
/** This is a streamlined version of TreeCellIterator, with built-in support to prune at detailLevel
|
||||
* (but not recursively upwards). */
|
||||
protected class PrefixTreeIterator extends CellIterator {
|
||||
private Shape shape;
|
||||
private PackedQuadCell thisCell;
|
||||
private PackedQuadCell nextCell;
|
||||
|
||||
private short level;
|
||||
private final short detailLevel;
|
||||
private CellIterator pruneIter;
|
||||
|
||||
PrefixTreeIterator(Shape shape, short detailLevel) {
|
||||
this.shape = shape;
|
||||
this.thisCell = ((PackedQuadCell)(getWorldCell())).nextCell(true);
|
||||
this.detailLevel = detailLevel;
|
||||
this.nextCell = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (nextCell != null) {
|
||||
return true;
|
||||
}
|
||||
SpatialRelation rel;
|
||||
// loop until we're at the end of the quad tree or we hit a relation
|
||||
while (thisCell != null) {
|
||||
rel = thisCell.getShape().relate(shape);
|
||||
if (rel == SpatialRelation.DISJOINT) {
|
||||
thisCell = thisCell.nextCell(false);
|
||||
} else { // within || intersects || contains
|
||||
thisCell.setShapeRel(rel);
|
||||
nextCell = thisCell;
|
||||
if (rel == SpatialRelation.WITHIN) {
|
||||
thisCell.setLeaf();
|
||||
thisCell = thisCell.nextCell(false);
|
||||
} else { // intersects || contains
|
||||
level = (short) (thisCell.getLevel());
|
||||
if (level == detailLevel || pruned(rel)) {
|
||||
thisCell.setLeaf();
|
||||
if (shape instanceof Point) {
|
||||
thisCell.setShapeRel(SpatialRelation.WITHIN);
|
||||
thisCell = null;
|
||||
} else {
|
||||
thisCell = thisCell.nextCell(false);
|
||||
}
|
||||
break;
|
||||
}
|
||||
thisCell = thisCell.nextCell(true);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
return nextCell != null;
|
||||
}
|
||||
|
||||
private boolean pruned(SpatialRelation rel) {
|
||||
int leaves;
|
||||
if (rel == SpatialRelation.INTERSECTS && leafyPrune && level == detailLevel - 1) {
|
||||
for (leaves=0, pruneIter=thisCell.getNextLevelCells(shape); pruneIter.hasNext(); pruneIter.next(), ++leaves);
|
||||
return leaves == 4;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Cell next() {
|
||||
if (nextCell == null) {
|
||||
if (!hasNext()) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
}
|
||||
// overriding since this implementation sets thisCell in hasNext
|
||||
Cell temp = nextCell;
|
||||
nextCell = null;
|
||||
return temp;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
//no-op
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -62,14 +62,14 @@ public class QuadPrefixTree extends LegacyPrefixTree {
|
|||
public static final int MAX_LEVELS_POSSIBLE = 50;//not really sure how big this should be
|
||||
|
||||
public static final int DEFAULT_MAX_LEVELS = 12;
|
||||
private final double xmin;
|
||||
private final double xmax;
|
||||
private final double ymin;
|
||||
private final double ymax;
|
||||
private final double xmid;
|
||||
private final double ymid;
|
||||
protected final double xmin;
|
||||
protected final double xmax;
|
||||
protected final double ymin;
|
||||
protected final double ymax;
|
||||
protected final double xmid;
|
||||
protected final double ymid;
|
||||
|
||||
private final double gridW;
|
||||
protected final double gridW;
|
||||
public final double gridH;
|
||||
|
||||
final double[] levelW;
|
||||
|
|
@ -178,7 +178,7 @@ public class QuadPrefixTree extends LegacyPrefixTree {
|
|||
// if we actually use the range property in the query, this could be useful
|
||||
}
|
||||
|
||||
private void checkBattenberg(
|
||||
protected void checkBattenberg(
|
||||
char c,
|
||||
double cx,
|
||||
double cy,
|
||||
|
|
@ -215,7 +215,7 @@ public class QuadPrefixTree extends LegacyPrefixTree {
|
|||
str.length = strlen;
|
||||
}
|
||||
|
||||
private class QuadCell extends LegacyCell {
|
||||
protected class QuadCell extends LegacyCell {
|
||||
|
||||
QuadCell(byte[] bytes, int off, int len) {
|
||||
super(bytes, off, len);
|
||||
|
|
@ -244,7 +244,7 @@ public class QuadPrefixTree extends LegacyPrefixTree {
|
|||
return cells;
|
||||
}
|
||||
|
||||
private BytesRef concat(BytesRef source, byte b) {
|
||||
protected BytesRef concat(BytesRef source, byte b) {
|
||||
//+2 for new char + potential leaf
|
||||
final byte[] buffer = Arrays.copyOfRange(source.bytes, source.offset, source.offset + source.length + 2);
|
||||
BytesRef target = new BytesRef(buffer);
|
||||
|
|
@ -270,7 +270,7 @@ public class QuadPrefixTree extends LegacyPrefixTree {
|
|||
return shape;
|
||||
}
|
||||
|
||||
private Rectangle makeShape() {
|
||||
protected Rectangle makeShape() {
|
||||
BytesRef token = getTokenBytesNoLeaf(null);
|
||||
double xmin = QuadPrefixTree.this.xmin;
|
||||
double ymin = QuadPrefixTree.this.ymin;
|
||||
|
|
|
|||
|
|
@ -17,11 +17,11 @@
|
|||
|
||||
package org.apache.lucene.spatial.prefix.tree;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import com.spatial4j.core.context.SpatialContext;
|
||||
import com.spatial4j.core.distance.DistanceUtils;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Abstract Factory for creating {@link SpatialPrefixTree} instances with useful
|
||||
* defaults and passed on configurations defined in a Map.
|
||||
|
|
@ -52,6 +52,8 @@ public abstract class SpatialPrefixTreeFactory {
|
|||
instance = new GeohashPrefixTree.Factory();
|
||||
else if ("quad".equalsIgnoreCase(cname))
|
||||
instance = new QuadPrefixTree.Factory();
|
||||
else if ("packedQuad".equalsIgnoreCase(cname))
|
||||
instance = new PackedQuadPrefixTree.Factory();
|
||||
else {
|
||||
try {
|
||||
Class<?> c = classLoader.loadClass(cname);
|
||||
|
|
|
|||
|
|
@ -22,22 +22,23 @@ import java.util.ArrayList;
|
|||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import com.carrotsearch.randomizedtesting.annotations.Name;
|
||||
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
|
||||
import com.spatial4j.core.context.SpatialContext;
|
||||
import com.spatial4j.core.shape.Point;
|
||||
import com.spatial4j.core.shape.Shape;
|
||||
import org.apache.lucene.document.FieldType;
|
||||
import org.apache.lucene.index.IndexOptions;
|
||||
import org.apache.lucene.spatial.bbox.BBoxStrategy;
|
||||
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
|
||||
import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy;
|
||||
import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
|
||||
import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree;
|
||||
import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
|
||||
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
|
||||
import org.apache.lucene.spatial.serialized.SerializedDVStrategy;
|
||||
import org.apache.lucene.spatial.vector.PointVectorStrategy;
|
||||
import org.junit.Test;
|
||||
import com.carrotsearch.randomizedtesting.annotations.Name;
|
||||
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
|
||||
import com.spatial4j.core.context.SpatialContext;
|
||||
import com.spatial4j.core.shape.Point;
|
||||
import com.spatial4j.core.shape.Shape;
|
||||
|
||||
public class DistanceStrategyTest extends StrategyTestCase {
|
||||
|
||||
|
|
@ -57,6 +58,10 @@ public class DistanceStrategyTest extends StrategyTestCase {
|
|||
strategy = new TermQueryPrefixTreeStrategy(grid, "termquery_geohash");
|
||||
ctorArgs.add(new Object[]{new Param(strategy)});
|
||||
|
||||
grid = new PackedQuadPrefixTree(ctx,25);
|
||||
strategy = new RecursivePrefixTreeStrategy(grid, "recursive_packedquad");
|
||||
ctorArgs.add(new Object[]{new Param(strategy)});
|
||||
|
||||
strategy = new PointVectorStrategy(ctx, "pointvector");
|
||||
ctorArgs.add(new Object[]{new Param(strategy)});
|
||||
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ import org.apache.lucene.spatial.StrategyTestCase;
|
|||
import org.apache.lucene.spatial.prefix.tree.Cell;
|
||||
import org.apache.lucene.spatial.prefix.tree.CellIterator;
|
||||
import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
|
||||
import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree;
|
||||
import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
|
||||
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
|
||||
import org.apache.lucene.spatial.query.SpatialArgs;
|
||||
|
|
@ -72,14 +73,17 @@ public class RandomSpatialOpFuzzyPrefixTreeTest extends StrategyTestCase {
|
|||
|
||||
public void setupGrid(int maxLevels) throws IOException {
|
||||
if (randomBoolean())
|
||||
setupQuadGrid(maxLevels);
|
||||
setupQuadGrid(maxLevels, randomBoolean());
|
||||
else
|
||||
setupGeohashGrid(maxLevels);
|
||||
setupCtx2D(ctx);
|
||||
|
||||
//((PrefixTreeStrategy) strategy).setDistErrPct(0);//fully precise to grid
|
||||
|
||||
// set prune independently on strategy & grid randomly; should work
|
||||
((RecursivePrefixTreeStrategy)strategy).setPruneLeafyBranches(randomBoolean());
|
||||
if (this.grid instanceof PackedQuadPrefixTree) {
|
||||
((PackedQuadPrefixTree) this.grid).setPruneLeafyBranches(randomBoolean());
|
||||
}
|
||||
|
||||
if (maxLevels == -1 && rarely()) {
|
||||
((PrefixTreeStrategy) strategy).setPointsOnly(true);
|
||||
}
|
||||
|
|
@ -97,7 +101,7 @@ public class RandomSpatialOpFuzzyPrefixTreeTest extends StrategyTestCase {
|
|||
ctx2D = ctxFactory.newSpatialContext();
|
||||
}
|
||||
|
||||
private void setupQuadGrid(int maxLevels) {
|
||||
private void setupQuadGrid(int maxLevels, boolean packedQuadPrefixTree) {
|
||||
//non-geospatial makes this test a little easier (in gridSnap), and using boundary values 2^X raises
|
||||
// the prospect of edge conditions we want to test, plus makes for simpler numbers (no decimals).
|
||||
SpatialContextFactory factory = new SpatialContextFactory();
|
||||
|
|
@ -107,7 +111,11 @@ public class RandomSpatialOpFuzzyPrefixTreeTest extends StrategyTestCase {
|
|||
//A fairly shallow grid, and default 2.5% distErrPct
|
||||
if (maxLevels == -1)
|
||||
maxLevels = randomIntBetween(1, 8);//max 64k cells (4^8), also 256*256
|
||||
this.grid = new QuadPrefixTree(ctx, maxLevels);
|
||||
if (packedQuadPrefixTree) {
|
||||
this.grid = new PackedQuadPrefixTree(ctx, maxLevels);
|
||||
} else {
|
||||
this.grid = new QuadPrefixTree(ctx, maxLevels);
|
||||
}
|
||||
this.strategy = newRPT();
|
||||
}
|
||||
|
||||
|
|
@ -148,7 +156,7 @@ public class RandomSpatialOpFuzzyPrefixTreeTest extends StrategyTestCase {
|
|||
/** See LUCENE-5062, {@link ContainsPrefixTreeFilter#multiOverlappingIndexedShapes}. */
|
||||
@Test
|
||||
public void testContainsPairOverlap() throws IOException {
|
||||
setupQuadGrid(3);
|
||||
setupQuadGrid(3, randomBoolean());
|
||||
adoc("0", new ShapePair(ctx.makeRectangle(0, 33, -128, 128), ctx.makeRectangle(33, 128, -128, 128), true));
|
||||
commit();
|
||||
Query query = strategy.makeQuery(new SpatialArgs(SpatialOperation.Contains,
|
||||
|
|
@ -159,7 +167,7 @@ public class RandomSpatialOpFuzzyPrefixTreeTest extends StrategyTestCase {
|
|||
|
||||
@Test
|
||||
public void testWithinDisjointParts() throws IOException {
|
||||
setupQuadGrid(7);
|
||||
setupQuadGrid(7, randomBoolean());
|
||||
//one shape comprised of two parts, quite separated apart
|
||||
adoc("0", new ShapePair(ctx.makeRectangle(0, 10, -120, -100), ctx.makeRectangle(220, 240, 110, 125), false));
|
||||
commit();
|
||||
|
|
@ -173,7 +181,7 @@ public class RandomSpatialOpFuzzyPrefixTreeTest extends StrategyTestCase {
|
|||
|
||||
@Test /** LUCENE-4916 */
|
||||
public void testWithinLeafApproxRule() throws IOException {
|
||||
setupQuadGrid(2);//4x4 grid
|
||||
setupQuadGrid(2, randomBoolean());//4x4 grid
|
||||
//indexed shape will simplify to entire right half (2 top cells)
|
||||
adoc("0", ctx.makeRectangle(192, 204, -128, 128));
|
||||
commit();
|
||||
|
|
|
|||
|
|
@ -17,10 +17,11 @@ package org.apache.solr.schema;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
|
||||
import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree;
|
||||
|
||||
/**
|
||||
* @see RecursivePrefixTreeStrategy
|
||||
* @lucene.experimental
|
||||
|
|
@ -45,6 +46,11 @@ public class SpatialRecursivePrefixTreeFieldType extends AbstractSpatialPrefixTr
|
|||
RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategy(grid, fieldName);
|
||||
if (prefixGridScanLevel != null)
|
||||
strategy.setPrefixGridScanLevel(prefixGridScanLevel);
|
||||
if (grid instanceof PackedQuadPrefixTree) {
|
||||
// This grid has a (usually) better prune leafy branch implementation
|
||||
((PackedQuadPrefixTree) grid).setPruneLeafyBranches(true);
|
||||
strategy.setPruneLeafyBranches(false);
|
||||
}
|
||||
return strategy;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,6 +35,9 @@
|
|||
<fieldType name="srpt_quad" class="solr.SpatialRecursivePrefixTreeFieldType"
|
||||
prefixTree="quad" distanceUnits="degrees"
|
||||
/>
|
||||
<fieldType name="srpt_packedquad" class="solr.SpatialRecursivePrefixTreeFieldType"
|
||||
prefixTree="packedQuad" distanceUnits="degrees"
|
||||
/>
|
||||
<fieldType name="srpt_100km" class="solr.SpatialRecursivePrefixTreeFieldType"
|
||||
maxDistErr="100" distanceUnits="kilometers"
|
||||
/>
|
||||
|
|
@ -58,6 +61,7 @@
|
|||
|
||||
<field name="srpt_geohash" type="srpt_geohash" multiValued="true" />
|
||||
<field name="srpt_quad" type="srpt_quad" multiValued="true" />
|
||||
<field name="srpt_packedquad" type="srpt_packedquad" multiValued="true" />
|
||||
<field name="stqpt_geohash" type="stqpt_geohash" multiValued="true" />
|
||||
<field name="pointvector" type="pointvector" />
|
||||
<field name="bbox" type="bbox" />
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ import com.spatial4j.core.context.SpatialContext;
|
|||
import com.spatial4j.core.distance.DistanceUtils;
|
||||
import com.spatial4j.core.shape.Point;
|
||||
import com.spatial4j.core.shape.Rectangle;
|
||||
|
||||
import org.apache.lucene.spatial.bbox.BBoxStrategy;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
|
@ -54,7 +53,7 @@ public class TestSolr4Spatial extends SolrTestCaseJ4 {
|
|||
@ParametersFactory
|
||||
public static Iterable<Object[]> parameters() {
|
||||
return Arrays.asList(new Object[][]{
|
||||
{"srpt_geohash"}, {"srpt_quad"}, {"stqpt_geohash"}, {"pointvector"}, {"bbox"}
|
||||
{"srpt_geohash"}, {"srpt_quad"}, {"srpt_packedquad"}, {"stqpt_geohash"}, {"pointvector"}, {"bbox"}
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue