LUCENE-4419: Test indexing non-point shapes; and add SpatialOperation.evaluate(s1,s2)

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1418007 13f79535-47bb-0310-9956-ffa450edef68

lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialOperation.java

@@ -17,6 +17,9 @@ package org.apache.lucene.spatial.query;
  * limitations under the License.
  */
 
+import com.spatial4j.core.shape.Shape;
+import com.spatial4j.core.shape.SpatialRelation;
+
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -25,14 +28,16 @@ import java.util.Locale;
 import java.util.Map;
 
 /**
- * A clause that compares a stored geometry to a supplied geometry.
+ * A clause that compares a stored geometry to a supplied geometry. For more
+ * explanation of each operation, consider looking at the source implementation
+ * of {@link #evaluate(com.spatial4j.core.shape.Shape, com.spatial4j.core.shape.Shape)}.
  *
  * @see <a href="http://edndoc.esri.com/arcsde/9.1/general_topics/understand_spatial_relations.htm">
  *   ESRIs docs on spatial relations</a>
  *
  * @lucene.experimental
  */
-public class SpatialOperation implements Serializable {
+public abstract class SpatialOperation implements Serializable {
   // Private registry
   private static final Map<String, SpatialOperation> registry = new HashMap<String, SpatialOperation>();
   private static final List<SpatialOperation> list = new ArrayList<SpatialOperation>();
@@ -40,15 +45,55 @@ public class SpatialOperation implements Serializable {
   // Geometry Operations
 
   /** Bounding box of the *indexed* shape. */
-  public static final SpatialOperation BBoxIntersects = new SpatialOperation("BBoxIntersects", true, false, false);
+  public static final SpatialOperation BBoxIntersects = new SpatialOperation("BBoxIntersects", true, false, false) {
+    @Override
+    public boolean evaluate(Shape indexedShape, Shape queryShape) {
+      return indexedShape.getBoundingBox().relate(queryShape).intersects();
+    }
+  };
   /** Bounding box of the *indexed* shape. */
-  public static final SpatialOperation BBoxWithin     = new SpatialOperation("BBoxWithin", true, false, false);
+  public static final SpatialOperation BBoxWithin     = new SpatialOperation("BBoxWithin", true, false, false) {
-  public static final SpatialOperation Contains       = new SpatialOperation("Contains", true, true, false);
+    @Override
-  public static final SpatialOperation Intersects     = new SpatialOperation("Intersects", true, false, false);
+    public boolean evaluate(Shape indexedShape, Shape queryShape) {
-  public static final SpatialOperation IsEqualTo      = new SpatialOperation("IsEqualTo", false, false, false);
+      return indexedShape.getBoundingBox().relate(queryShape) == SpatialRelation.WITHIN;
-  public static final SpatialOperation IsDisjointTo   = new SpatialOperation("IsDisjointTo", false, false, false);
+    }
-  public static final SpatialOperation IsWithin       = new SpatialOperation("IsWithin", true, false, true);
+  };
-  public static final SpatialOperation Overlaps       = new SpatialOperation("Overlaps", true, false, true);
+  public static final SpatialOperation Contains       = new SpatialOperation("Contains", true, true, false) {
+    @Override
+    public boolean evaluate(Shape indexedShape, Shape queryShape) {
+      return indexedShape.hasArea() && indexedShape.relate(queryShape) == SpatialRelation.CONTAINS;
+    }
+  };
+  public static final SpatialOperation Intersects     = new SpatialOperation("Intersects", true, false, false) {
+    @Override
+    public boolean evaluate(Shape indexedShape, Shape queryShape) {
+      return indexedShape.relate(queryShape).intersects();
+    }
+  };
+  public static final SpatialOperation IsEqualTo      = new SpatialOperation("IsEqualTo", false, false, false) {
+    @Override
+    public boolean evaluate(Shape indexedShape, Shape queryShape) {
+      return indexedShape.equals(queryShape);
+    }
+  };
+  public static final SpatialOperation IsDisjointTo   = new SpatialOperation("IsDisjointTo", false, false, false) {
+    @Override
+    public boolean evaluate(Shape indexedShape, Shape queryShape) {
+      return ! indexedShape.relate(queryShape).intersects();
+    }
+  };
+  public static final SpatialOperation IsWithin       = new SpatialOperation("IsWithin", true, false, true) {
+    @Override
+    public boolean evaluate(Shape indexedShape, Shape queryShape) {
+      return queryShape.hasArea() && indexedShape.relate(queryShape) == SpatialRelation.WITHIN;
+    }
+  };
+  public static final SpatialOperation Overlaps       = new SpatialOperation("Overlaps", true, false, true) {
+    @Override
+    public boolean evaluate(Shape indexedShape, Shape queryShape) {
+      return queryShape.hasArea() && indexedShape.relate(queryShape).intersects();
+    }
+  };
 
   // Member variables
   private final boolean scoreIsMeaningful;
@@ -90,6 +135,11 @@ public class SpatialOperation implements Serializable {
     return false;
   }
 
+  /**
+   * Returns whether the relationship between indexedShape and queryShape is
+   * satisfied by this operation.
+   */
+  public abstract boolean evaluate(Shape indexedShape, Shape queryShape);
 
   // ================================================= Getters / Setters =============================================
 

lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java

@@ -1,3 +1,5 @@
+package org.apache.lucene.spatial;
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -15,8 +17,9 @@
  * limitations under the License.
  */
 
-package org.apache.lucene.spatial;
+import com.spatial4j.core.context.SpatialContext;
-
+import com.spatial4j.core.shape.Point;
+import com.spatial4j.core.shape.Rectangle;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.RandomIndexWriter;
@@ -35,6 +38,10 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomGaussian;
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween;
+
+/** A base test class for spatial lucene. It's mostly Lucene generic. */
 public abstract class SpatialTestCase extends LuceneTestCase {
 
   private DirectoryReader indexReader;
@@ -42,6 +49,8 @@ public abstract class SpatialTestCase extends LuceneTestCase {
   private Directory directory;
   protected IndexSearcher indexSearcher;
 
+  protected SpatialContext ctx;//subclass must initialize
+
   @Override
   @Before
   public void setUp() throws Exception {
@@ -100,6 +109,63 @@ public abstract class SpatialTestCase extends LuceneTestCase {
     }
   }
 
+  protected Point randomPoint() {
+    final Rectangle WB = ctx.getWorldBounds();
+    return ctx.makePoint(
+        randomIntBetween((int) WB.getMinX(), (int) WB.getMaxX()),
+        randomIntBetween((int) WB.getMinY(), (int) WB.getMaxY()));
+  }
+
+  protected Rectangle randomRectangle() {
+    final Rectangle WB = ctx.getWorldBounds();
+    int rW = (int) randomGaussianMeanMax(10, WB.getWidth());
+    double xMin = randomIntBetween((int) WB.getMinX(), (int) WB.getMaxX() - rW);
+    double xMax = xMin + rW;
+
+    int yH = (int) randomGaussianMeanMax(Math.min(rW, WB.getHeight()), WB.getHeight());
+    double yMin = randomIntBetween((int) WB.getMinY(), (int) WB.getMaxY() - yH);
+    double yMax = yMin + yH;
+
+    return ctx.makeRectangle(xMin, xMax, yMin, yMax);
+  }
+
+  private double randomGaussianMinMeanMax(double min, double mean, double max) {
+    assert mean > min;
+    return randomGaussianMeanMax(mean - min, max - min) + min;
+  }
+
+  /**
+   * Within one standard deviation (68% of the time) the result is "close" to
+   * mean. By "close": when greater than mean, it's the lesser of 2*mean or half
+   * way to max, when lesser than mean, it's the greater of max-2*mean or half
+   * way to 0. The other 32% of the time it's in the rest of the range, touching
+   * either 0 or max but never exceeding.
+   */
+  private double randomGaussianMeanMax(double mean, double max) {
+    // DWS: I verified the results empirically
+    assert mean <= max && mean >= 0;
+    double g = randomGaussian();
+    double mean2 = mean;
+    double flip = 1;
+    if (g < 0) {
+      mean2 = max - mean;
+      flip = -1;
+      g *= -1;
+    }
+    // pivot is the distance from mean2 towards max where the boundary of
+    // 1 standard deviation alters the calculation
+    double pivotMax = max - mean2;
+    double pivot = Math.min(mean2, pivotMax / 2);//from 0 to max-mean2
+    assert pivot >= 0 && pivotMax >= pivot && g >= 0;
+    double pivotResult;
+    if (g <= 1)
+      pivotResult = pivot * g;
+    else
+      pivotResult = Math.min(pivotMax, (g - 1) * (pivotMax - pivot) + pivot);
+
+    return mean + flip * pivotResult;
+  }
+
   // ================================================= Inner Classes =================================================
 
   protected static class SearchResults {
@@ -116,7 +182,7 @@ public abstract class SpatialTestCase extends LuceneTestCase {
       StringBuilder str = new StringBuilder();
       str.append("found: ").append(numFound).append('[');
       for(SearchResult r : results) {
-        String id = r.document.get("id");
+        String id = r.getId();
         str.append(id).append(", ");
       }
       str.append(']');
@@ -139,6 +205,10 @@ public abstract class SpatialTestCase extends LuceneTestCase {
       this.document = storedDocument;
     }
 
+    public String getId() {
+      return document.get("id");
+    }
+    
     @Override
     public String toString() {
       return "["+score+"="+document+"]";

lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java

@@ -82,6 +82,8 @@ public class SpatialTestQuery {
 
   @Override
   public String toString() {
-    return line;
+    if (line != null)
+      return line;
+    return args.toString()+" "+ids;
   }
 }

lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java

@@ -31,8 +31,9 @@ import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.search.CheckHits;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.spatial.query.SpatialArgs;
 import org.apache.lucene.spatial.query.SpatialArgsParser;
-import org.junit.Assert;
+import org.apache.lucene.spatial.query.SpatialOperation;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -42,6 +43,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.logging.Logger;
 
@@ -64,7 +66,6 @@ public abstract class StrategyTestCase extends SpatialTestCase {
   protected final SpatialArgsParser argsParser = new SpatialArgsParser();
 
   protected SpatialStrategy strategy;
-  protected SpatialContext ctx;
   protected boolean storeShape = true;
 
   protected void executeQueries(SpatialMatchConcern concern, String... testQueryFile) throws IOException {
@@ -131,50 +132,52 @@ public abstract class StrategyTestCase extends SpatialTestCase {
       SpatialMatchConcern concern) {
     while (queries.hasNext()) {
       SpatialTestQuery q = queries.next();
+      runTestQuery(concern, q);
+    }
+  }
 
-      String msg = q.line; //"Query: " + q.args.toString(ctx);
+  public void runTestQuery(SpatialMatchConcern concern, SpatialTestQuery q) {
-      SearchResults got = executeQuery(strategy.makeQuery(q.args), 100);
+    String msg = q.toString(); //"Query: " + q.args.toString(ctx);
-      if (storeShape && got.numFound > 0) {
+    SearchResults got = executeQuery(strategy.makeQuery(q.args), Math.max(100, q.ids.size()+1));
-        //check stored value is there & parses
+    if (storeShape && got.numFound > 0) {
-        assertNotNull(ctx.readShape(got.results.get(0).document.get(strategy.getFieldName())));
+      //check stored value is there & parses
-      }
+      assertNotNull(ctx.readShape(got.results.get(0).document.get(strategy.getFieldName())));
-      if (concern.orderIsImportant) {
+    }
-        Iterator<String> ids = q.ids.iterator();
+    if (concern.orderIsImportant) {
-        for (SearchResult r : got.results) {
+      Iterator<String> ids = q.ids.iterator();
-          String id = r.document.get("id");
+      for (SearchResult r : got.results) {
-          if(!ids.hasNext()) {
+        String id = r.document.get("id");
-            Assert.fail(msg + " :: Did not get enough results.  Expect" + q.ids+", got: "+got.toDebugString());
+        if (!ids.hasNext()) {
-          }
+          fail(msg + " :: Did not get enough results.  Expect" + q.ids + ", got: " + got.toDebugString());
-          Assert.assertEquals( "out of order: " + msg, ids.next(), id);
         }
+        assertEquals("out of order: " + msg, ids.next(), id);
+      }
 
-        if (ids.hasNext()) {
+      if (ids.hasNext()) {
-          Assert.fail(msg + " :: expect more results then we got: " + ids.next());
+        fail(msg + " :: expect more results then we got: " + ids.next());
+      }
+    } else {
+      // We are looking at how the results overlap
+      if (concern.resultsAreSuperset) {
+        Set<String> found = new HashSet<String>();
+        for (SearchResult r : got.results) {
+          found.add(r.document.get("id"));
+        }
+        for (String s : q.ids) {
+          if (!found.contains(s)) {
+            fail("Results are mising id: " + s + " :: " + found);
+          }
         }
       } else {
-        // We are looking at how the results overlap
+        List<String> found = new ArrayList<String>();
-        if( concern.resultsAreSuperset ) {
+        for (SearchResult r : got.results) {
-          Set<String> found = new HashSet<String>();
+          found.add(r.document.get("id"));
-          for (SearchResult r : got.results) {
-            found.add(r.document.get("id"));
-          }
-          for( String s : q.ids ) {
-            if( !found.contains( s ) ) {
-              Assert.fail( "Results are mising id: "+s + " :: " + found );
-            }
-          }
         }
-        else {
-          List<String> found = new ArrayList<String>();
-          for (SearchResult r : got.results) {
-            found.add(r.document.get("id"));
-          }
 
-          // sort both so that the order is not important
+        // sort both so that the order is not important
-          Collections.sort(q.ids);
+        Collections.sort(q.ids);
-          Collections.sort(found);
+        Collections.sort(found);
-          Assert.assertEquals(msg, q.ids.toString(), found.toString());
+        assertEquals(msg, q.ids.toString(), found.toString());
-        }
       }
     }
   }
@@ -221,4 +224,19 @@ public abstract class StrategyTestCase extends SpatialTestCase {
     CheckHits.checkExplanations(q, "", indexSearcher);
   }
 
+  protected void assertOperation(Map<String,Shape> indexedDocs,
+                                 SpatialOperation operation, Shape queryShape) {
+    //Generate truth via brute force
+    Set<String> expectedIds = new HashSet<String>();
+    for (Map.Entry<String, Shape> stringShapeEntry : indexedDocs.entrySet()) {
+      if (operation.evaluate(stringShapeEntry.getValue(), queryShape))
+        expectedIds.add(stringShapeEntry.getKey());
+    }
+
+    SpatialTestQuery testQuery = new SpatialTestQuery();
+    testQuery.args = new SpatialArgs(operation, queryShape);
+    testQuery.ids = new ArrayList<String>(expectedIds);
+    runTestQuery(SpatialMatchConcern.FILTER, testQuery);
+  }
+
 }

lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java

@@ -0,0 +1,133 @@
+package org.apache.lucene.spatial.prefix;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.shape.Rectangle;
+import com.spatial4j.core.shape.Shape;
+import com.spatial4j.core.shape.impl.RectangleImpl;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.spatial.StrategyTestCase;
+import org.apache.lucene.spatial.prefix.tree.Node;
+import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
+import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
+import org.apache.lucene.spatial.query.SpatialArgs;
+import org.apache.lucene.spatial.query.SpatialOperation;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomInt;
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween;
+
+public class SpatialOpRecursivePrefixTreeTest extends StrategyTestCase {
+
+  private SpatialPrefixTree grid;
+
+  @Test
+  @Repeat(iterations = 20)
+  public void testIntersects() throws IOException {
+    //non-geospatial makes this test a little easier
+    this.ctx = new SpatialContext(false, null, new RectangleImpl(0, 256, -128, 128, null));
+    //A fairly shallow grid, and default 2.5% distErrPct
+    this.grid = new QuadPrefixTree(ctx, randomIntBetween(1, 8));
+    this.strategy = new RecursivePrefixTreeStrategy(grid, getClass().getSimpleName());
+    //((PrefixTreeStrategy) strategy).setDistErrPct(0);//fully precise to grid
+
+    deleteAll();
+
+    Map<String, Shape> indexedShapes = new LinkedHashMap<String, Shape>();
+    Map<String, Rectangle> indexedGriddedShapes = new LinkedHashMap<String, Rectangle>();
+    final int numIndexedShapes = randomIntBetween(1, 6);
+    for (int i = 1; i <= numIndexedShapes; i++) {
+      String id = "" + i;
+      Shape indexShape = randomRectangle();
+      Rectangle gridShape = gridSnapp(indexShape);
+      indexedShapes.put(id, indexShape);
+      indexedGriddedShapes.put(id, gridShape);
+      adoc(id, indexShape);
+    }
+
+    commit();
+
+    final int numQueryShapes = atLeast(10);
+    for (int i = 0; i < numQueryShapes; i++) {
+      int scanLevel = randomInt(grid.getMaxLevels());
+      ((RecursivePrefixTreeStrategy) strategy).setPrefixGridScanLevel(scanLevel);
+      Rectangle queryShape = randomRectangle();
+      Rectangle queryGridShape = gridSnapp(queryShape);
+
+      //Generate truth via brute force
+      final SpatialOperation operation = SpatialOperation.Intersects;
+      Set<String> expectedIds = new TreeSet<String>();
+      Set<String> optionalIds = new TreeSet<String>();
+      for (String id : indexedShapes.keySet()) {
+        Shape indexShape = indexedShapes.get(id);
+        Rectangle indexGridShape = indexedGriddedShapes.get(id);
+        if (operation.evaluate(indexShape, queryShape))
+          expectedIds.add(id);
+        else if (operation.evaluate(indexGridShape, queryGridShape))
+          optionalIds.add(id);
+      }
+
+      //Search and verify results
+      Query query = strategy.makeQuery(new SpatialArgs(operation, queryShape));
+      SearchResults got = executeQuery(query, 100);
+      Set<String> remainingExpectedIds = new TreeSet<String>(expectedIds);
+      String msg = queryShape.toString()+" Expect: "+expectedIds+" Opt: "+optionalIds;
+      for (SearchResult result : got.results) {
+        String id = result.getId();
+        Object removed = remainingExpectedIds.remove(id);
+        if (removed == null) {
+          assertTrue("Shouldn't match " + id + " in "+msg, optionalIds.contains(id));
+        }
+      }
+      assertTrue("Didn't match " + remainingExpectedIds + " in " + msg, remainingExpectedIds.isEmpty());
+    }
+
+  }
+
+  protected Rectangle gridSnapp(Shape snapMe) {
+    //The next 4 lines mimic PrefixTreeStrategy.createIndexableFields()
+    double distErrPct = ((PrefixTreeStrategy) strategy).getDistErrPct();
+    double distErr = SpatialArgs.calcDistanceFromErrPct(snapMe, distErrPct, ctx);
+    int detailLevel = grid.getLevelForDistance(distErr);
+    List<Node> cells = grid.getNodes(snapMe, detailLevel, false);
+
+    //calc bounding box of cells.
+    double minX = Double.POSITIVE_INFINITY, maxX = Double.NEGATIVE_INFINITY;
+    double minY = Double.POSITIVE_INFINITY, maxY = Double.NEGATIVE_INFINITY;
+    for (Node cell : cells) {
+      assert cell.getLevel() <= detailLevel;
+      Rectangle cellR = cell.getShape().getBoundingBox();
+
+      minX = Math.min(minX, cellR.getMinX());
+      maxX = Math.max(maxX, cellR.getMaxX());
+      minY = Math.min(minY, cellR.getMinY());
+      maxY = Math.max(maxY, cellR.getMaxY());
+    }
+    return ctx.makeRectangle(minX, maxX, minY, maxY);
+  }
+
+}