From e4071374f4a041d00faf7e9c0914a2e0ca78d4f5 Mon Sep 17 00:00:00 2001 From: David Wayne Smiley Date: Thu, 6 Dec 2012 17:19:23 +0000 Subject: [PATCH] LUCENE-4419: Test indexing non-point shapes; and add SpatialOperation.evaluate(s1,s2) git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1418007 13f79535-47bb-0310-9956-ffa450edef68 --- .../spatial/query/SpatialOperation.java | 70 +++++++-- .../lucene/spatial/SpatialTestCase.java | 76 +++++++++- .../lucene/spatial/SpatialTestQuery.java | 4 +- .../lucene/spatial/StrategyTestCase.java | 98 +++++++------ .../SpatialOpRecursivePrefixTreeTest.java | 133 ++++++++++++++++++ 5 files changed, 327 insertions(+), 54 deletions(-) create mode 100644 lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialOperation.java b/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialOperation.java index e786974401f..247f6ae6a9e 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialOperation.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialOperation.java @@ -17,6 +17,9 @@ package org.apache.lucene.spatial.query; * limitations under the License. */ +import com.spatial4j.core.shape.Shape; +import com.spatial4j.core.shape.SpatialRelation; + import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; @@ -25,14 +28,16 @@ import java.util.Locale; import java.util.Map; /** - * A clause that compares a stored geometry to a supplied geometry. + * A clause that compares a stored geometry to a supplied geometry. For more + * explanation of each operation, consider looking at the source implementation + * of {@link #evaluate(com.spatial4j.core.shape.Shape, com.spatial4j.core.shape.Shape)}. * * @see * ESRIs docs on spatial relations * * @lucene.experimental */ -public class SpatialOperation implements Serializable { +public abstract class SpatialOperation implements Serializable { // Private registry private static final Map registry = new HashMap(); private static final List list = new ArrayList(); @@ -40,15 +45,55 @@ public class SpatialOperation implements Serializable { // Geometry Operations /** Bounding box of the *indexed* shape. */ - public static final SpatialOperation BBoxIntersects = new SpatialOperation("BBoxIntersects", true, false, false); + public static final SpatialOperation BBoxIntersects = new SpatialOperation("BBoxIntersects", true, false, false) { + @Override + public boolean evaluate(Shape indexedShape, Shape queryShape) { + return indexedShape.getBoundingBox().relate(queryShape).intersects(); + } + }; /** Bounding box of the *indexed* shape. */ - public static final SpatialOperation BBoxWithin = new SpatialOperation("BBoxWithin", true, false, false); - public static final SpatialOperation Contains = new SpatialOperation("Contains", true, true, false); - public static final SpatialOperation Intersects = new SpatialOperation("Intersects", true, false, false); - public static final SpatialOperation IsEqualTo = new SpatialOperation("IsEqualTo", false, false, false); - public static final SpatialOperation IsDisjointTo = new SpatialOperation("IsDisjointTo", false, false, false); - public static final SpatialOperation IsWithin = new SpatialOperation("IsWithin", true, false, true); - public static final SpatialOperation Overlaps = new SpatialOperation("Overlaps", true, false, true); + public static final SpatialOperation BBoxWithin = new SpatialOperation("BBoxWithin", true, false, false) { + @Override + public boolean evaluate(Shape indexedShape, Shape queryShape) { + return indexedShape.getBoundingBox().relate(queryShape) == SpatialRelation.WITHIN; + } + }; + public static final SpatialOperation Contains = new SpatialOperation("Contains", true, true, false) { + @Override + public boolean evaluate(Shape indexedShape, Shape queryShape) { + return indexedShape.hasArea() && indexedShape.relate(queryShape) == SpatialRelation.CONTAINS; + } + }; + public static final SpatialOperation Intersects = new SpatialOperation("Intersects", true, false, false) { + @Override + public boolean evaluate(Shape indexedShape, Shape queryShape) { + return indexedShape.relate(queryShape).intersects(); + } + }; + public static final SpatialOperation IsEqualTo = new SpatialOperation("IsEqualTo", false, false, false) { + @Override + public boolean evaluate(Shape indexedShape, Shape queryShape) { + return indexedShape.equals(queryShape); + } + }; + public static final SpatialOperation IsDisjointTo = new SpatialOperation("IsDisjointTo", false, false, false) { + @Override + public boolean evaluate(Shape indexedShape, Shape queryShape) { + return ! indexedShape.relate(queryShape).intersects(); + } + }; + public static final SpatialOperation IsWithin = new SpatialOperation("IsWithin", true, false, true) { + @Override + public boolean evaluate(Shape indexedShape, Shape queryShape) { + return queryShape.hasArea() && indexedShape.relate(queryShape) == SpatialRelation.WITHIN; + } + }; + public static final SpatialOperation Overlaps = new SpatialOperation("Overlaps", true, false, true) { + @Override + public boolean evaluate(Shape indexedShape, Shape queryShape) { + return queryShape.hasArea() && indexedShape.relate(queryShape).intersects(); + } + }; // Member variables private final boolean scoreIsMeaningful; @@ -90,6 +135,11 @@ public class SpatialOperation implements Serializable { return false; } + /** + * Returns whether the relationship between indexedShape and queryShape is + * satisfied by this operation. + */ + public abstract boolean evaluate(Shape indexedShape, Shape queryShape); // ================================================= Getters / Setters ============================================= diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java index 13592ba7836..7e71f9c7a56 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java @@ -1,3 +1,5 @@ +package org.apache.lucene.spatial; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,8 +17,9 @@ * limitations under the License. */ -package org.apache.lucene.spatial; - +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.shape.Point; +import com.spatial4j.core.shape.Rectangle; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.RandomIndexWriter; @@ -35,6 +38,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomGaussian; +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween; + +/** A base test class for spatial lucene. It's mostly Lucene generic. */ public abstract class SpatialTestCase extends LuceneTestCase { private DirectoryReader indexReader; @@ -42,6 +49,8 @@ public abstract class SpatialTestCase extends LuceneTestCase { private Directory directory; protected IndexSearcher indexSearcher; + protected SpatialContext ctx;//subclass must initialize + @Override @Before public void setUp() throws Exception { @@ -100,6 +109,63 @@ public abstract class SpatialTestCase extends LuceneTestCase { } } + protected Point randomPoint() { + final Rectangle WB = ctx.getWorldBounds(); + return ctx.makePoint( + randomIntBetween((int) WB.getMinX(), (int) WB.getMaxX()), + randomIntBetween((int) WB.getMinY(), (int) WB.getMaxY())); + } + + protected Rectangle randomRectangle() { + final Rectangle WB = ctx.getWorldBounds(); + int rW = (int) randomGaussianMeanMax(10, WB.getWidth()); + double xMin = randomIntBetween((int) WB.getMinX(), (int) WB.getMaxX() - rW); + double xMax = xMin + rW; + + int yH = (int) randomGaussianMeanMax(Math.min(rW, WB.getHeight()), WB.getHeight()); + double yMin = randomIntBetween((int) WB.getMinY(), (int) WB.getMaxY() - yH); + double yMax = yMin + yH; + + return ctx.makeRectangle(xMin, xMax, yMin, yMax); + } + + private double randomGaussianMinMeanMax(double min, double mean, double max) { + assert mean > min; + return randomGaussianMeanMax(mean - min, max - min) + min; + } + + /** + * Within one standard deviation (68% of the time) the result is "close" to + * mean. By "close": when greater than mean, it's the lesser of 2*mean or half + * way to max, when lesser than mean, it's the greater of max-2*mean or half + * way to 0. The other 32% of the time it's in the rest of the range, touching + * either 0 or max but never exceeding. + */ + private double randomGaussianMeanMax(double mean, double max) { + // DWS: I verified the results empirically + assert mean <= max && mean >= 0; + double g = randomGaussian(); + double mean2 = mean; + double flip = 1; + if (g < 0) { + mean2 = max - mean; + flip = -1; + g *= -1; + } + // pivot is the distance from mean2 towards max where the boundary of + // 1 standard deviation alters the calculation + double pivotMax = max - mean2; + double pivot = Math.min(mean2, pivotMax / 2);//from 0 to max-mean2 + assert pivot >= 0 && pivotMax >= pivot && g >= 0; + double pivotResult; + if (g <= 1) + pivotResult = pivot * g; + else + pivotResult = Math.min(pivotMax, (g - 1) * (pivotMax - pivot) + pivot); + + return mean + flip * pivotResult; + } + // ================================================= Inner Classes ================================================= protected static class SearchResults { @@ -116,7 +182,7 @@ public abstract class SpatialTestCase extends LuceneTestCase { StringBuilder str = new StringBuilder(); str.append("found: ").append(numFound).append('['); for(SearchResult r : results) { - String id = r.document.get("id"); + String id = r.getId(); str.append(id).append(", "); } str.append(']'); @@ -138,6 +204,10 @@ public abstract class SpatialTestCase extends LuceneTestCase { this.score = score; this.document = storedDocument; } + + public String getId() { + return document.get("id"); + } @Override public String toString() { diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java index e165a7c10ca..4188983c9b5 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java @@ -82,6 +82,8 @@ public class SpatialTestQuery { @Override public String toString() { - return line; + if (line != null) + return line; + return args.toString()+" "+ids; } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java index 294240d899b..91ecf206ebb 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java @@ -31,8 +31,9 @@ import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.CheckHits; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.spatial.query.SpatialArgs; import org.apache.lucene.spatial.query.SpatialArgsParser; -import org.junit.Assert; +import org.apache.lucene.spatial.query.SpatialOperation; import java.io.FileNotFoundException; import java.io.IOException; @@ -42,6 +43,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.logging.Logger; @@ -64,7 +66,6 @@ public abstract class StrategyTestCase extends SpatialTestCase { protected final SpatialArgsParser argsParser = new SpatialArgsParser(); protected SpatialStrategy strategy; - protected SpatialContext ctx; protected boolean storeShape = true; protected void executeQueries(SpatialMatchConcern concern, String... testQueryFile) throws IOException { @@ -131,50 +132,52 @@ public abstract class StrategyTestCase extends SpatialTestCase { SpatialMatchConcern concern) { while (queries.hasNext()) { SpatialTestQuery q = queries.next(); + runTestQuery(concern, q); + } + } - String msg = q.line; //"Query: " + q.args.toString(ctx); - SearchResults got = executeQuery(strategy.makeQuery(q.args), 100); - if (storeShape && got.numFound > 0) { - //check stored value is there & parses - assertNotNull(ctx.readShape(got.results.get(0).document.get(strategy.getFieldName()))); - } - if (concern.orderIsImportant) { - Iterator ids = q.ids.iterator(); - for (SearchResult r : got.results) { - String id = r.document.get("id"); - if(!ids.hasNext()) { - Assert.fail(msg + " :: Did not get enough results. Expect" + q.ids+", got: "+got.toDebugString()); - } - Assert.assertEquals( "out of order: " + msg, ids.next(), id); + public void runTestQuery(SpatialMatchConcern concern, SpatialTestQuery q) { + String msg = q.toString(); //"Query: " + q.args.toString(ctx); + SearchResults got = executeQuery(strategy.makeQuery(q.args), Math.max(100, q.ids.size()+1)); + if (storeShape && got.numFound > 0) { + //check stored value is there & parses + assertNotNull(ctx.readShape(got.results.get(0).document.get(strategy.getFieldName()))); + } + if (concern.orderIsImportant) { + Iterator ids = q.ids.iterator(); + for (SearchResult r : got.results) { + String id = r.document.get("id"); + if (!ids.hasNext()) { + fail(msg + " :: Did not get enough results. Expect" + q.ids + ", got: " + got.toDebugString()); } - - if (ids.hasNext()) { - Assert.fail(msg + " :: expect more results then we got: " + ids.next()); + assertEquals("out of order: " + msg, ids.next(), id); + } + + if (ids.hasNext()) { + fail(msg + " :: expect more results then we got: " + ids.next()); + } + } else { + // We are looking at how the results overlap + if (concern.resultsAreSuperset) { + Set found = new HashSet(); + for (SearchResult r : got.results) { + found.add(r.document.get("id")); + } + for (String s : q.ids) { + if (!found.contains(s)) { + fail("Results are mising id: " + s + " :: " + found); + } } } else { - // We are looking at how the results overlap - if( concern.resultsAreSuperset ) { - Set found = new HashSet(); - for (SearchResult r : got.results) { - found.add(r.document.get("id")); - } - for( String s : q.ids ) { - if( !found.contains( s ) ) { - Assert.fail( "Results are mising id: "+s + " :: " + found ); - } - } + List found = new ArrayList(); + for (SearchResult r : got.results) { + found.add(r.document.get("id")); } - else { - List found = new ArrayList(); - for (SearchResult r : got.results) { - found.add(r.document.get("id")); - } - // sort both so that the order is not important - Collections.sort(q.ids); - Collections.sort(found); - Assert.assertEquals(msg, q.ids.toString(), found.toString()); - } + // sort both so that the order is not important + Collections.sort(q.ids); + Collections.sort(found); + assertEquals(msg, q.ids.toString(), found.toString()); } } } @@ -221,4 +224,19 @@ public abstract class StrategyTestCase extends SpatialTestCase { CheckHits.checkExplanations(q, "", indexSearcher); } + protected void assertOperation(Map indexedDocs, + SpatialOperation operation, Shape queryShape) { + //Generate truth via brute force + Set expectedIds = new HashSet(); + for (Map.Entry stringShapeEntry : indexedDocs.entrySet()) { + if (operation.evaluate(stringShapeEntry.getValue(), queryShape)) + expectedIds.add(stringShapeEntry.getKey()); + } + + SpatialTestQuery testQuery = new SpatialTestQuery(); + testQuery.args = new SpatialArgs(operation, queryShape); + testQuery.ids = new ArrayList(expectedIds); + runTestQuery(SpatialMatchConcern.FILTER, testQuery); + } + } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java b/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java new file mode 100644 index 00000000000..60ad4e760cf --- /dev/null +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java @@ -0,0 +1,133 @@ +package org.apache.lucene.spatial.prefix; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.carrotsearch.randomizedtesting.annotations.Repeat; +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.shape.Rectangle; +import com.spatial4j.core.shape.Shape; +import com.spatial4j.core.shape.impl.RectangleImpl; +import org.apache.lucene.search.Query; +import org.apache.lucene.spatial.StrategyTestCase; +import org.apache.lucene.spatial.prefix.tree.Node; +import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; +import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; +import org.apache.lucene.spatial.query.SpatialArgs; +import org.apache.lucene.spatial.query.SpatialOperation; +import org.junit.Test; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomInt; +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween; + +public class SpatialOpRecursivePrefixTreeTest extends StrategyTestCase { + + private SpatialPrefixTree grid; + + @Test + @Repeat(iterations = 20) + public void testIntersects() throws IOException { + //non-geospatial makes this test a little easier + this.ctx = new SpatialContext(false, null, new RectangleImpl(0, 256, -128, 128, null)); + //A fairly shallow grid, and default 2.5% distErrPct + this.grid = new QuadPrefixTree(ctx, randomIntBetween(1, 8)); + this.strategy = new RecursivePrefixTreeStrategy(grid, getClass().getSimpleName()); + //((PrefixTreeStrategy) strategy).setDistErrPct(0);//fully precise to grid + + deleteAll(); + + Map indexedShapes = new LinkedHashMap(); + Map indexedGriddedShapes = new LinkedHashMap(); + final int numIndexedShapes = randomIntBetween(1, 6); + for (int i = 1; i <= numIndexedShapes; i++) { + String id = "" + i; + Shape indexShape = randomRectangle(); + Rectangle gridShape = gridSnapp(indexShape); + indexedShapes.put(id, indexShape); + indexedGriddedShapes.put(id, gridShape); + adoc(id, indexShape); + } + + commit(); + + final int numQueryShapes = atLeast(10); + for (int i = 0; i < numQueryShapes; i++) { + int scanLevel = randomInt(grid.getMaxLevels()); + ((RecursivePrefixTreeStrategy) strategy).setPrefixGridScanLevel(scanLevel); + Rectangle queryShape = randomRectangle(); + Rectangle queryGridShape = gridSnapp(queryShape); + + //Generate truth via brute force + final SpatialOperation operation = SpatialOperation.Intersects; + Set expectedIds = new TreeSet(); + Set optionalIds = new TreeSet(); + for (String id : indexedShapes.keySet()) { + Shape indexShape = indexedShapes.get(id); + Rectangle indexGridShape = indexedGriddedShapes.get(id); + if (operation.evaluate(indexShape, queryShape)) + expectedIds.add(id); + else if (operation.evaluate(indexGridShape, queryGridShape)) + optionalIds.add(id); + } + + //Search and verify results + Query query = strategy.makeQuery(new SpatialArgs(operation, queryShape)); + SearchResults got = executeQuery(query, 100); + Set remainingExpectedIds = new TreeSet(expectedIds); + String msg = queryShape.toString()+" Expect: "+expectedIds+" Opt: "+optionalIds; + for (SearchResult result : got.results) { + String id = result.getId(); + Object removed = remainingExpectedIds.remove(id); + if (removed == null) { + assertTrue("Shouldn't match " + id + " in "+msg, optionalIds.contains(id)); + } + } + assertTrue("Didn't match " + remainingExpectedIds + " in " + msg, remainingExpectedIds.isEmpty()); + } + + } + + protected Rectangle gridSnapp(Shape snapMe) { + //The next 4 lines mimic PrefixTreeStrategy.createIndexableFields() + double distErrPct = ((PrefixTreeStrategy) strategy).getDistErrPct(); + double distErr = SpatialArgs.calcDistanceFromErrPct(snapMe, distErrPct, ctx); + int detailLevel = grid.getLevelForDistance(distErr); + List cells = grid.getNodes(snapMe, detailLevel, false); + + //calc bounding box of cells. + double minX = Double.POSITIVE_INFINITY, maxX = Double.NEGATIVE_INFINITY; + double minY = Double.POSITIVE_INFINITY, maxY = Double.NEGATIVE_INFINITY; + for (Node cell : cells) { + assert cell.getLevel() <= detailLevel; + Rectangle cellR = cell.getShape().getBoundingBox(); + + minX = Math.min(minX, cellR.getMinX()); + maxX = Math.max(maxX, cellR.getMaxX()); + minY = Math.min(minY, cellR.getMinY()); + maxY = Math.max(maxY, cellR.getMaxY()); + } + return ctx.makeRectangle(minX, maxX, minY, maxY); + } + +}