LUCENE-6422: New spatial PackedQuadPrefixTree. Thanks Nick!

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1675538 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -73,6 +73,11 @@ New Features
 * LUCENE-6423: New LimitTokenOffsetFilter that limits tokens to those before
   a configured maximum start offset. (David Smiley)
 
+* LUCENE-6422: New spatial PackedQuadPrefixTree, a generally more efficient
+
  choice than QuadPrefixTree, especially for high precision shapes.
+
  When used, you should typically disable RPT's pruneLeafyBranches option.

+  (Nick Knize, David Smiley)
+
 Optimizations
 
 * LUCENE-6379: IndexWriter.deleteDocuments(Query...) now detects if

lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialDocMaker.java

@@ -33,6 +33,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.spatial.SpatialStrategy;
 import org.apache.lucene.spatial.composite.CompositeSpatialStrategy;
 import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
+import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTreeFactory;
 import org.apache.lucene.spatial.serialized.SerializedDVStrategy;
@@ -111,7 +112,13 @@ public class SpatialDocMaker extends DocMaker {
 
     RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategy(grid, spatialField);
     strategy.setPointsOnly(config.get("spatial.docPointsOnly", false));
-    strategy.setPruneLeafyBranches(config.get("spatial.pruneLeafyBranches", true));
+    final boolean pruneLeafyBranches = config.get("spatial.pruneLeafyBranches", true);
+    if (grid instanceof PackedQuadPrefixTree) {
+      ((PackedQuadPrefixTree) grid).setPruneLeafyBranches(pruneLeafyBranches);
+      strategy.setPruneLeafyBranches(false);//always leave it to packed grid, even though it isn't the same
+    } else {
+      strategy.setPruneLeafyBranches(pruneLeafyBranches);
+    }
 
     int prefixGridScanLevel = config.get("query.spatial.prefixGridScanLevel", -4);
     if (prefixGridScanLevel < 0)

lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java

@@ -92,9 +92,14 @@ public class RecursivePrefixTreeStrategy extends PrefixTreeStrategy {
     return pruneLeafyBranches;
   }
 
-  /** An optional hint affecting non-point shapes: it will
+  /**
-   * simplify/aggregate sets of complete leaves in a cell to its parent, resulting in ~20-25%
+   * An optional hint affecting non-point shapes: it will
-   * fewer indexed cells. However, it will likely be removed in the future. (default=true)
+   * prune away a complete set sibling leaves to their parent (recursively), resulting in ~20-50%
+   * fewer indexed cells, and consequently that much less disk and that much faster indexing.
+   * So if it's a quad tree and all 4 sub-cells are there marked as a leaf, then they will be
+   * removed (pruned) and the parent is marked as a leaf instead.  This occurs recursively on up.  Unfortunately, the
+   * current implementation will buffer all cells to do this, so consider disabling for high precision (low distErrPct)
+   * shapes. (default=true)
    */
   public void setPruneLeafyBranches(boolean pruneLeafyBranches) {
     this.pruneLeafyBranches = pruneLeafyBranches;

lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/CellIterator.java

@@ -65,7 +65,7 @@ public abstract class CellIterator implements Iterator<Cell> {
   }
 
   @Override
-  public final Cell next() {
+  public Cell next() {
     if (nextCell == null) {
       if (!hasNext())
         throw new NoSuchElementException();

lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/LegacyCell.java

@@ -36,9 +36,9 @@ public abstract class LegacyCell implements Cell {
   private static final byte LEAF_BYTE = '+';//NOTE: must sort before letters & numbers
 
   //Arguably we could simply use a BytesRef, using an extra Object.
-  private byte[] bytes;//generally bigger to potentially hold a leaf
+  protected byte[] bytes;//generally bigger to potentially hold a leaf
-  private int b_off;
+  protected int b_off;
-  private int b_len;//doesn't reflect leaf; same as getLevel()
+  protected int b_len;//doesn't reflect leaf; same as getLevel()
 
   protected boolean isLeaf;
 
@@ -68,7 +68,7 @@ public abstract class LegacyCell implements Cell {
     readLeafAdjust();
   }
 
-  private void readLeafAdjust() {
+  protected void readLeafAdjust() {
     isLeaf = (b_len > 0 && bytes[b_off + b_len - 1] == LEAF_BYTE);
     if (isLeaf)
       b_len--;
@@ -76,18 +76,6 @@ public abstract class LegacyCell implements Cell {
       isLeaf = true;
   }
 
-//  @Override
-//  public void copyFrom(Cell source) {
-//    LegacyCell src = (LegacyCell) source;
-//    shapeRel = src.shapeRel;
-//    shape = src.shape;
-//    isLeaf = src.isLeaf;
-//    //we don't actually copy the bytes because in LegacyCell the bytes aren't modified. (leaf byte doesn't count)
-//    bytes = src.bytes;
-//    b_off = src.b_off;
-//    b_len = src.b_len;
-//  }
-
   protected abstract SpatialPrefixTree getGrid();
 
   protected abstract int getMaxLevels();
@@ -214,7 +202,7 @@ public abstract class LegacyCell implements Cell {
 
   /** Copied from {@link BytesRef#compareTo(BytesRef)}.
    * This is to avoid creating a BytesRef. */
-  private static int compare(byte[] aBytes, int aUpto, int a_length, byte[] bBytes, int bUpto, int b_length) {
+  protected static int compare(byte[] aBytes, int aUpto, int a_length, byte[] bBytes, int bUpto, int b_length) {
     final int aStop = aUpto + Math.min(a_length, b_length);
     while(aUpto < aStop) {
       int aByte = aBytes[aUpto++] & 0xff;

lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/PackedQuadPrefixTree.java

@@ -0,0 +1,460 @@
+package org.apache.lucene.spatial.prefix.tree;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.shape.Point;
+import com.spatial4j.core.shape.Rectangle;
+import com.spatial4j.core.shape.Shape;
+import com.spatial4j.core.shape.SpatialRelation;
+import com.spatial4j.core.shape.impl.RectangleImpl;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Uses a compact binary representation of 8 bytes to encode a spatial quad trie.
+ *
+ * The binary representation is as follows:
+ * <pre>
+ * CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDL
+ *
+ * Where C = Cell bits (2 per quad)
+ *       D = Depth bits (5 with max of 29 levels)
+ *       L = isLeaf bit
+ * </pre>
+ *
+ * It includes a built-in "pruneLeafyBranches" setting (true by default) similar to
+ * {@link org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy#setPruneLeafyBranches(boolean)} although
+ * this one only prunes at the target detail level (where it has the most effect).  Usually you should disable RPT's
+ * prune, since it is very memory in-efficient.
+ *
+ * @lucene.experimental
+ */
+public class PackedQuadPrefixTree extends QuadPrefixTree {
+  public static final int MAX_LEVELS_POSSIBLE = 29;
+  protected static final byte[] QUAD = new byte[] {0x00, 0x01, 0x02, 0x03};
+
+  protected boolean leafyPrune = true;
+
+  /**
+   * Factory for creating {@link PackedQuadPrefixTree} instances with useful defaults.
+   */
+  public static class Factory extends QuadPrefixTree.Factory {
+    @Override
+    protected SpatialPrefixTree newSPT() {
+      return new PackedQuadPrefixTree(ctx, maxLevels != null ? maxLevels : MAX_LEVELS_POSSIBLE);
+    }
+  }
+
+  public PackedQuadPrefixTree(SpatialContext ctx, int maxLevels) {
+    super(ctx, maxLevels);
+    if (maxLevels > MAX_LEVELS_POSSIBLE) {
+      throw new IllegalArgumentException("maxLevels of " + maxLevels + " exceeds limit of " + MAX_LEVELS_POSSIBLE);
+    }
+  }
+
+  @Override
+  public String toString() {
+    return getClass().getSimpleName() + "(maxLevels:" + maxLevels + ",ctx:" + ctx + ",prune:" + leafyPrune + ")";
+  }
+
+  @Override
+  public Cell getWorldCell() {
+    return new PackedQuadCell(0x0L);
+  }
+
+  @Override
+  public Cell getCell(Point p, int level) {
+    List<Cell> cells = new ArrayList<>(1);
+    build(xmid, ymid, 0, cells, 0x0L, ctx.makePoint(p.getX(), p.getY()), level);
+    return cells.get(0);//note cells could be longer if p on edge
+  }
+
+  protected void build(double x, double y, int level, List<Cell> matches, long term, Shape shape, int maxLevel) {
+    double w = levelW[level] / 2;
+    double h = levelH[level] / 2;
+
+    // Z-Order
+    // http://en.wikipedia.org/wiki/Z-order_%28curve%29
+    checkBattenberg(QUAD[0], x - w, y + h, level, matches, term, shape, maxLevel);
+    checkBattenberg(QUAD[1], x + w, y + h, level, matches, term, shape, maxLevel);
+    checkBattenberg(QUAD[2], x - w, y - h, level, matches, term, shape, maxLevel);
+    checkBattenberg(QUAD[3], x + w, y - h, level, matches, term, shape, maxLevel);
+  }
+
+  protected void checkBattenberg(byte quad, double cx, double cy, int level, List<Cell> matches,
+                               long term, Shape shape, int maxLevel) {
+    // short-circuit if we find a match for the point (no need to continue recursion)
+    if (shape instanceof Point && !matches.isEmpty())
+      return;
+    double w = levelW[level] / 2;
+    double h = levelH[level] / 2;
+
+    SpatialRelation v = shape.relate(ctx.makeRectangle(cx - w, cx + w, cy - h, cy + h));
+
+    if (SpatialRelation.DISJOINT == v) {
+      return;
+    }
+
+    // set bits for next level
+    term |= (((long)(quad))<<(64-(++level<<1)));
+    // increment level
+    term = ((term>>>1)+1)<<1;
+
+    if (SpatialRelation.CONTAINS == v || (level >= maxLevel)) {
+      matches.add(new PackedQuadCell(term, v.transpose()));
+    } else {// SpatialRelation.WITHIN, SpatialRelation.INTERSECTS
+      build(cx, cy, level, matches, term, shape, maxLevel);
+    }
+  }
+
+  @Override
+  public Cell readCell(BytesRef term, Cell scratch) {
+    PackedQuadCell cell = (PackedQuadCell) scratch;
+    if (cell == null)
+      cell = (PackedQuadCell) getWorldCell();
+    cell.readCell(term);
+    return cell;
+  }
+
+  @Override
+  public CellIterator getTreeCellIterator(Shape shape, int detailLevel) {
+    if (detailLevel > maxLevels) {
+      throw new IllegalArgumentException("detailLevel:" + detailLevel +" exceed max: " + maxLevels);
+    }
+    return new PrefixTreeIterator(shape, (short) detailLevel);
+  }
+
+  public boolean isPruneLeafyBranches() {
+    return leafyPrune;
+  }
+
+  /** Like {@link org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy#setPruneLeafyBranches(boolean)}
+   * but more memory efficient and only applies to the detailLevel, where it has the most effect. */
+  public void setPruneLeafyBranches( boolean pruneLeafyBranches ) {
+    this.leafyPrune = pruneLeafyBranches;
+  }
+
+  /** See binary representation in the javadocs of {@link PackedQuadPrefixTree}. */
+  protected class PackedQuadCell extends QuadCell {
+    private long term;
+
+    PackedQuadCell(long term) {
+      super(null, 0, 0);
+      this.term = term;
+      this.b_off = 0;
+      this.bytes = longToByteArray(this.term);
+      this.b_len = 8;
+      readLeafAdjust();
+    }
+
+    PackedQuadCell(long term, SpatialRelation shapeRel) {
+      this(term);
+      this.shapeRel = shapeRel;
+    }
+
+    @Override
+    protected void readCell(BytesRef bytes) {
+      shapeRel = null;
+      shape = null;
+      this.bytes = bytes.bytes;
+      this.b_off = bytes.offset;
+      this.b_len = (short) bytes.length;
+      this.term = longFromByteArray(this.bytes, bytes.offset);
+      readLeafAdjust();
+    }
+
+    private final int getShiftForLevel(final int level) {
+      return 64 - (level<<1);
+    }
+
+    public boolean isEnd(final int level, final int shift) {
+      return (term != 0x0L && ((((0x1L<<(level<<1))-1)-(term>>>shift)) == 0x0L));
+    }
+
+    /**
+     * Get the next cell in the tree without using recursion. descend parameter requests traversal to the child nodes,
+     * setting this to false will step to the next sibling.
+     * Note: This complies with lexicographical ordering, once you've moved to the next sibling there is no backtracking.
+     */
+    public PackedQuadCell nextCell(boolean descend) {
+      final int level = getLevel();
+      final int shift = getShiftForLevel(level);
+      // base case: can't go further
+      if ( (!descend && isEnd(level, shift)) || isEnd(maxLevels, getShiftForLevel(maxLevels))) {
+        return null;
+      }
+      long newTerm;
+      final boolean isLeaf = (term&0x1L)==0x1L;
+      // if descend requested && we're not at the maxLevel
+      if ((descend && !isLeaf && (level != maxLevels)) || level == 0) {
+        // simple case: increment level bits (next level)
+        newTerm = ((term>>>1)+0x1L)<<1;
+      } else {  // we're not descending or we can't descend
+        newTerm = term + (0x1L<<shift);
+        // we're at the last sibling...force descend
+        if (((term>>>shift)&0x3L) == 0x3L) {
+          // adjust level for number popping up
+          newTerm = ((newTerm>>>1) - (Long.numberOfTrailingZeros(newTerm>>>shift)>>>1))<<1;
+        }
+      }
+      return new PackedQuadCell(newTerm);
+    }
+
+    @Override
+    protected void readLeafAdjust() {
+      isLeaf = ((0x1L)&term) == 0x1L;
+      if (getLevel() == getMaxLevels()) {
+        isLeaf = true;
+      }
+    }
+
+    @Override
+    public BytesRef getTokenBytesWithLeaf(BytesRef result) {
+      if (isLeaf) {
+        term |= 0x1L;
+      }
+      return getTokenBytesNoLeaf(result);
+    }
+
+    @Override
+    public BytesRef getTokenBytesNoLeaf(BytesRef result) {
+      if (result == null)
+        return new BytesRef(bytes, b_off, b_len);
+      result.bytes = longToByteArray(this.term);
+      result.offset = 0;
+      result.length = result.bytes.length;
+      return result;
+    }
+
+    @Override
+    public int compareToNoLeaf(Cell fromCell) {
+      PackedQuadCell b = (PackedQuadCell) fromCell;
+      final long thisTerm = (((0x1L)&term) == 0x1L) ? term-1 : term;
+      final long fromTerm = (((0x1L)&b.term) == 0x1L) ? b.term-1 : b.term;
+      final int result = Long.compareUnsigned(thisTerm, fromTerm);
+      assert Math.signum(result)
+          == Math.signum(compare(longToByteArray(thisTerm), 0, 8, longToByteArray(fromTerm), 0, 8)); // TODO remove
+      return result;
+    }
+
+    @Override
+    public int getLevel() {
+      int l = (int)((term >>> 1)&0x1FL);
+      return l;
+    }
+
+    @Override
+    protected Collection<Cell> getSubCells() {
+      List<Cell> cells = new ArrayList<>(4);
+      PackedQuadCell pqc = (new PackedQuadCell(((term&0x1)==0x1) ? this.term-1 : this.term))
+          .nextCell(true);
+      cells.add(pqc);
+      cells.add((pqc = pqc.nextCell(false)));
+      cells.add((pqc = pqc.nextCell(false)));
+      cells.add(pqc.nextCell(false));
+      return cells;
+    }
+
+    @Override
+    protected QuadCell getSubCell(Point p) {
+      return (PackedQuadCell) PackedQuadPrefixTree.this.getCell(p, getLevel() + 1);//not performant!
+    }
+
+    @Override
+    public boolean isPrefixOf(Cell c) {
+      PackedQuadCell cell = (PackedQuadCell)c;
+      return (this.term == 0x0L) || isInternalPrefix(cell);
+    }
+
+    protected boolean isInternalPrefix(PackedQuadCell c) {
+      final int shift = 64 - (getLevel()<<1);
+      return ((term>>>shift)-(c.term>>>shift)) == 0x0L;
+    }
+
+    protected long concat(byte postfix) {
+      // extra leaf bit
+      return this.term | (((long)(postfix))<<((getMaxLevels()-getLevel()<<1)+6));
+    }
+
+    /**
+     * Constructs a bounding box shape out of the encoded cell
+     */
+    @Override
+    protected Rectangle makeShape() {
+      double xmin = PackedQuadPrefixTree.this.xmin;
+      double ymin = PackedQuadPrefixTree.this.ymin;
+      int level = getLevel();
+
+      byte b;
+      for (short l=0, i=1; l<level; ++l, ++i) {
+        b = (byte) ((term>>>(64-(i<<1))) & 0x3L);
+
+        switch (b) {
+          case 0x00:
+            ymin += levelH[l];
+            break;
+          case 0x01:
+            xmin += levelW[l];
+            ymin += levelH[l];
+            break;
+          case 0x02:
+            break;//nothing really
+          case 0x03:
+            xmin += levelW[l];
+            break;
+          default:
+            throw new RuntimeException("unexpected quadrant");
+        }
+      }
+
+      double width, height;
+      if (level > 0) {
+        width = levelW[level - 1];
+        height = levelH[level - 1];
+      } else {
+        width = gridW;
+        height = gridH;
+      }
+      return new RectangleImpl(xmin, xmin + width, ymin, ymin + height, ctx);
+    }
+
+    private long fromBytes(byte b1, byte b2, byte b3, byte b4, byte b5, byte b6, byte b7, byte b8) {
+      return ((long)b1 & 255L) << 56 | ((long)b2 & 255L) << 48 | ((long)b3 & 255L) << 40
+          | ((long)b4 & 255L) << 32 | ((long)b5 & 255L) << 24 | ((long)b6 & 255L) << 16
+          | ((long)b7 & 255L) << 8 | (long)b8 & 255L;
+    }
+
+    private byte[] longToByteArray(long value) {
+      byte[] result = new byte[8];
+      for(int i = 7; i >= 0; --i) {
+        result[i] = (byte)((int)(value & 255L));
+        value >>= 8;
+      }
+      return result;
+    }
+
+    private long longFromByteArray(byte[] bytes, int ofs) {
+      assert bytes.length >= 8;
+      return fromBytes(bytes[0+ofs], bytes[1+ofs], bytes[2+ofs], bytes[3+ofs],
+          bytes[4+ofs], bytes[5+ofs], bytes[6+ofs], bytes[7+ofs]);
+    }
+
+    /**
+     * Used for debugging, this will print the bits of the cell
+     */
+    @Override
+    public String toString() {
+      StringBuilder s = new StringBuilder(64);
+      final int numberOfLeadingZeros = Long.numberOfLeadingZeros(term);
+      for (int i = 0; i < numberOfLeadingZeros; i++) {
+        s.append('0');
+      }
+      if (term != 0)
+        s.append(Long.toBinaryString(term));
+      return s.toString();
+    }
+  } // PackedQuadCell
+
+  /** This is a streamlined version of TreeCellIterator, with built-in support to prune at detailLevel
+   * (but not recursively upwards). */
+  protected class PrefixTreeIterator extends CellIterator {
+    private Shape shape;
+    private PackedQuadCell thisCell;
+    private PackedQuadCell nextCell;
+
+    private short level;
+    private final short detailLevel;
+    private CellIterator pruneIter;
+
+    PrefixTreeIterator(Shape shape, short detailLevel) {
+      this.shape = shape;
+      this.thisCell = ((PackedQuadCell)(getWorldCell())).nextCell(true);
+      this.detailLevel = detailLevel;
+      this.nextCell = null;
+    }
+
+    @Override
+    public boolean hasNext() {
+      if (nextCell != null) {
+        return true;
+      }
+      SpatialRelation rel;
+      // loop until we're at the end of the quad tree or we hit a relation
+      while (thisCell != null) {
+        rel = thisCell.getShape().relate(shape);
+        if (rel == SpatialRelation.DISJOINT) {
+          thisCell = thisCell.nextCell(false);
+        } else { // within || intersects || contains
+          thisCell.setShapeRel(rel);
+          nextCell = thisCell;
+          if (rel == SpatialRelation.WITHIN) {
+            thisCell.setLeaf();
+            thisCell = thisCell.nextCell(false);
+          } else {  // intersects || contains
+            level = (short) (thisCell.getLevel());
+            if (level == detailLevel || pruned(rel)) {
+              thisCell.setLeaf();
+              if (shape instanceof Point) {
+                thisCell.setShapeRel(SpatialRelation.WITHIN);
+                thisCell = null;
+              } else {
+                thisCell = thisCell.nextCell(false);
+              }
+              break;
+            }
+            thisCell = thisCell.nextCell(true);
+          }
+          break;
+        }
+      }
+      return nextCell != null;
+    }
+
+    private boolean pruned(SpatialRelation rel) {
+      int leaves;
+      if (rel == SpatialRelation.INTERSECTS && leafyPrune && level == detailLevel - 1) {
+        for (leaves=0, pruneIter=thisCell.getNextLevelCells(shape); pruneIter.hasNext(); pruneIter.next(), ++leaves);
+        return leaves == 4;
+      }
+      return false;
+    }
+
+    @Override
+    public Cell next() {
+      if (nextCell == null) {
+        if (!hasNext()) {
+          throw new NoSuchElementException();
+        }
+      }
+      // overriding since this implementation sets thisCell in hasNext
+      Cell temp = nextCell;
+      nextCell = null;
+      return temp;
+    }
+
+    @Override
+    public void remove() {
+      //no-op
+    }
+  }
+}

lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java

@@ -62,14 +62,14 @@ public class QuadPrefixTree extends LegacyPrefixTree {
   public static final int MAX_LEVELS_POSSIBLE = 50;//not really sure how big this should be
 
   public static final int DEFAULT_MAX_LEVELS = 12;
-  private final double xmin;
+  protected final double xmin;
-  private final double xmax;
+  protected final double xmax;
-  private final double ymin;
+  protected final double ymin;
-  private final double ymax;
+  protected final double ymax;
-  private final double xmid;
+  protected final double xmid;
-  private final double ymid;
+  protected final double ymid;
 
-  private final double gridW;
+  protected final double gridW;
   public final double gridH;
 
   final double[] levelW;
@@ -178,7 +178,7 @@ public class QuadPrefixTree extends LegacyPrefixTree {
     // if we actually use the range property in the query, this could be useful
   }
 
-  private void checkBattenberg(
+  protected void checkBattenberg(
       char c,
       double cx,
       double cy,
@@ -215,7 +215,7 @@ public class QuadPrefixTree extends LegacyPrefixTree {
     str.length = strlen;
   }
 
-  private class QuadCell extends LegacyCell {
+  protected class QuadCell extends LegacyCell {
 
     QuadCell(byte[] bytes, int off, int len) {
       super(bytes, off, len);
@@ -244,7 +244,7 @@ public class QuadPrefixTree extends LegacyPrefixTree {
       return cells;
     }
 
-    private BytesRef concat(BytesRef source, byte b) {
+    protected BytesRef concat(BytesRef source, byte b) {
       //+2 for new char + potential leaf
       final byte[] buffer = Arrays.copyOfRange(source.bytes, source.offset, source.offset + source.length + 2);
       BytesRef target = new BytesRef(buffer);
@@ -270,7 +270,7 @@ public class QuadPrefixTree extends LegacyPrefixTree {
       return shape;
     }
 
-    private Rectangle makeShape() {
+    protected Rectangle makeShape() {
       BytesRef token = getTokenBytesNoLeaf(null);
       double xmin = QuadPrefixTree.this.xmin;
       double ymin = QuadPrefixTree.this.ymin;

lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java

@@ -17,11 +17,11 @@
 
 package org.apache.lucene.spatial.prefix.tree;
 
+import java.util.Map;
+
 import com.spatial4j.core.context.SpatialContext;
 import com.spatial4j.core.distance.DistanceUtils;
 
-import java.util.Map;
-
 /**
  * Abstract Factory for creating {@link SpatialPrefixTree} instances with useful
  * defaults and passed on configurations defined in a Map.
@@ -52,6 +52,8 @@ public abstract class SpatialPrefixTreeFactory {
       instance = new GeohashPrefixTree.Factory();
     else if ("quad".equalsIgnoreCase(cname))
       instance = new QuadPrefixTree.Factory();
+    else if ("packedQuad".equalsIgnoreCase(cname))
+      instance = new PackedQuadPrefixTree.Factory();
     else {
       try {
         Class<?> c = classLoader.loadClass(cname);

lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java

@@ -22,22 +22,23 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.shape.Point;
+import com.spatial4j.core.shape.Shape;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.spatial.bbox.BBoxStrategy;
 import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
 import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy;
 import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
+import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
 import org.apache.lucene.spatial.serialized.SerializedDVStrategy;
 import org.apache.lucene.spatial.vector.PointVectorStrategy;
 import org.junit.Test;
-import com.carrotsearch.randomizedtesting.annotations.Name;
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.shape.Point;
-import com.spatial4j.core.shape.Shape;
 
 public class DistanceStrategyTest extends StrategyTestCase {
 
@@ -57,6 +58,10 @@ public class DistanceStrategyTest extends StrategyTestCase {
     strategy = new TermQueryPrefixTreeStrategy(grid, "termquery_geohash");
     ctorArgs.add(new Object[]{new Param(strategy)});
 
+    grid = new PackedQuadPrefixTree(ctx,25);
+    strategy = new RecursivePrefixTreeStrategy(grid, "recursive_packedquad");
+    ctorArgs.add(new Object[]{new Param(strategy)});
+
     strategy = new PointVectorStrategy(ctx, "pointvector");
     ctorArgs.add(new Object[]{new Param(strategy)});
 

lucene/spatial/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpFuzzyPrefixTreeTest.java

@@ -47,6 +47,7 @@ import org.apache.lucene.spatial.StrategyTestCase;
 import org.apache.lucene.spatial.prefix.tree.Cell;
 import org.apache.lucene.spatial.prefix.tree.CellIterator;
 import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
+import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
 import org.apache.lucene.spatial.query.SpatialArgs;
@@ -72,14 +73,17 @@ public class RandomSpatialOpFuzzyPrefixTreeTest extends StrategyTestCase {
 
   public void setupGrid(int maxLevels) throws IOException {
     if (randomBoolean())
-      setupQuadGrid(maxLevels);
+      setupQuadGrid(maxLevels, randomBoolean());
     else
       setupGeohashGrid(maxLevels);
     setupCtx2D(ctx);
 
-    //((PrefixTreeStrategy) strategy).setDistErrPct(0);//fully precise to grid
+    // set prune independently on strategy & grid randomly; should work
-
     ((RecursivePrefixTreeStrategy)strategy).setPruneLeafyBranches(randomBoolean());
+    if (this.grid instanceof PackedQuadPrefixTree) {
+      ((PackedQuadPrefixTree) this.grid).setPruneLeafyBranches(randomBoolean());
+    }
+
     if (maxLevels == -1 && rarely()) {
       ((PrefixTreeStrategy) strategy).setPointsOnly(true);
     }
@@ -97,7 +101,7 @@ public class RandomSpatialOpFuzzyPrefixTreeTest extends StrategyTestCase {
     ctx2D = ctxFactory.newSpatialContext();
   }
 
-  private void setupQuadGrid(int maxLevels) {
+  private void setupQuadGrid(int maxLevels, boolean packedQuadPrefixTree) {
     //non-geospatial makes this test a little easier (in gridSnap), and using boundary values 2^X raises
     // the prospect of edge conditions we want to test, plus makes for simpler numbers (no decimals).
     SpatialContextFactory factory = new SpatialContextFactory();
@@ -107,7 +111,11 @@ public class RandomSpatialOpFuzzyPrefixTreeTest extends StrategyTestCase {
     //A fairly shallow grid, and default 2.5% distErrPct
     if (maxLevels == -1)
       maxLevels = randomIntBetween(1, 8);//max 64k cells (4^8), also 256*256
-    this.grid = new QuadPrefixTree(ctx, maxLevels);
+    if (packedQuadPrefixTree) {
+      this.grid = new PackedQuadPrefixTree(ctx, maxLevels);
+    } else {
+      this.grid = new QuadPrefixTree(ctx, maxLevels);
+    }
     this.strategy = newRPT();
   }
 
@@ -148,7 +156,7 @@ public class RandomSpatialOpFuzzyPrefixTreeTest extends StrategyTestCase {
   /** See LUCENE-5062, {@link ContainsPrefixTreeFilter#multiOverlappingIndexedShapes}. */
   @Test
   public void testContainsPairOverlap() throws IOException {
-    setupQuadGrid(3);
+    setupQuadGrid(3, randomBoolean());
     adoc("0", new ShapePair(ctx.makeRectangle(0, 33, -128, 128), ctx.makeRectangle(33, 128, -128, 128), true));
     commit();
     Query query = strategy.makeQuery(new SpatialArgs(SpatialOperation.Contains,
@@ -159,7 +167,7 @@ public class RandomSpatialOpFuzzyPrefixTreeTest extends StrategyTestCase {
 
   @Test
   public void testWithinDisjointParts() throws IOException {
-    setupQuadGrid(7);
+    setupQuadGrid(7, randomBoolean());
     //one shape comprised of two parts, quite separated apart
     adoc("0", new ShapePair(ctx.makeRectangle(0, 10, -120, -100), ctx.makeRectangle(220, 240, 110, 125), false));
     commit();
@@ -173,7 +181,7 @@ public class RandomSpatialOpFuzzyPrefixTreeTest extends StrategyTestCase {
 
   @Test /** LUCENE-4916 */
   public void testWithinLeafApproxRule() throws IOException {
-    setupQuadGrid(2);//4x4 grid
+    setupQuadGrid(2, randomBoolean());//4x4 grid
     //indexed shape will simplify to entire right half (2 top cells)
     adoc("0", ctx.makeRectangle(192, 204, -128, 128));
     commit();

solr/core/src/java/org/apache/solr/schema/SpatialRecursivePrefixTreeFieldType.java

@@ -17,10 +17,11 @@ package org.apache.solr.schema;
  * limitations under the License.
  */
 
-import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
-
 import java.util.Map;
 
+import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
+import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree;
+
 /**
  * @see RecursivePrefixTreeStrategy
  * @lucene.experimental
@@ -45,6 +46,11 @@ public class SpatialRecursivePrefixTreeFieldType extends AbstractSpatialPrefixTr
     RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategy(grid, fieldName);
     if (prefixGridScanLevel != null)
       strategy.setPrefixGridScanLevel(prefixGridScanLevel);
+    if (grid instanceof PackedQuadPrefixTree) {
+      // This grid has a (usually) better prune leafy branch implementation
+      ((PackedQuadPrefixTree) grid).setPruneLeafyBranches(true);
+      strategy.setPruneLeafyBranches(false);
+    }
     return strategy;
   }
 }

solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml

@@ -35,6 +35,9 @@
     <fieldType name="srpt_quad"   class="solr.SpatialRecursivePrefixTreeFieldType"
               prefixTree="quad" distanceUnits="degrees"
         />
+    <fieldType name="srpt_packedquad"   class="solr.SpatialRecursivePrefixTreeFieldType"
+              prefixTree="packedQuad" distanceUnits="degrees"
+        />
     <fieldType name="srpt_100km"   class="solr.SpatialRecursivePrefixTreeFieldType"
               maxDistErr="100" distanceUnits="kilometers"
         />
@@ -58,6 +61,7 @@
 
     <field name="srpt_geohash" type="srpt_geohash" multiValued="true" />
     <field name="srpt_quad" type="srpt_quad" multiValued="true" />
+    <field name="srpt_packedquad" type="srpt_packedquad" multiValued="true" />
     <field name="stqpt_geohash" type="stqpt_geohash" multiValued="true" />
     <field name="pointvector" type="pointvector" />
     <field name="bbox" type="bbox" />

solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java

@@ -26,7 +26,6 @@ import com.spatial4j.core.context.SpatialContext;
 import com.spatial4j.core.distance.DistanceUtils;
 import com.spatial4j.core.shape.Point;
 import com.spatial4j.core.shape.Rectangle;
-
 import org.apache.lucene.spatial.bbox.BBoxStrategy;
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.common.SolrException;
@@ -54,7 +53,7 @@ public class TestSolr4Spatial extends SolrTestCaseJ4 {
   @ParametersFactory
   public static Iterable<Object[]> parameters() {
     return Arrays.asList(new Object[][]{
-        {"srpt_geohash"}, {"srpt_quad"}, {"stqpt_geohash"}, {"pointvector"}, {"bbox"}
+        {"srpt_geohash"}, {"srpt_quad"}, {"srpt_packedquad"}, {"stqpt_geohash"}, {"pointvector"}, {"bbox"}
     });
   }