Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8440: Add support for indexing and searching Line and Point shapes using LatLonShape encoding

This commit is contained in:
Nicholas Knize 2018-07-31 17:45:12 -05:00
parent b5ed6350a0
commit a0e33a9bc8
9 changed files with 888 additions and 362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/CHANGES.txt
View File

@ -213,6 +213,8 @@ Changes in Runtime Behavior:
Improvements
* LUCENE-8440: Add support for indexing and searching Line and Point shapes using LatLonShape encoding (Nick Knize)
* LUCENE-8435: Add new LatLonShapePolygonQuery for querying indexed LatLonShape fields by arbitrary polygons (Nick Knize)
* LUCENE-8367: Make per-dimension drill down optional for each facet dimension (Mike McCandless)

2
lucene/core/src/java/org/apache/lucene/geo/Polygon.java
View File

@ -202,7 +202,7 @@ public final class Polygon {
return sb.toString();
}
private String verticesToGeoJSON(final double[] lats, final double[] lons) {
public static String verticesToGeoJSON(final double[] lats, final double[] lons) {
StringBuilder sb = new StringBuilder();
sb.append('[');
for (int i = 0; i < lats.length; i++) {

67
lucene/sandbox/src/java/org/apache/lucene/document/LatLonShape.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.document;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.geo.Line;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Tessellator;
import org.apache.lucene.geo.Tessellator.Triangle;
@ -27,6 +28,9 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
/**
* An indexed shape utility class.
* <p>
@ -62,16 +66,67 @@ public class LatLonShape {
private LatLonShape() {
}
/** the lionshare of the indexing is done by the tessellator */
/** create indexable fields for polygon geometry */
public static Field[] createIndexableFields(String fieldName, Polygon polygon) {
// the lionshare of the indexing is done by the tessellator
List<Triangle> tessellation = Tessellator.tessellate(polygon);
List<LatLonTriangle> fields = new ArrayList<>();
for (int i = 0; i < tessellation.size(); ++i) {
fields.add(new LatLonTriangle(fieldName, tessellation.get(i)));
for (Triangle t : tessellation) {
fields.add(new LatLonTriangle(fieldName, t.getEncodedX(0), t.getEncodedY(0),
t.getEncodedX(1), t.getEncodedY(1), t.getEncodedX(2), t.getEncodedY(2)));
}
return fields.toArray(new Field[fields.size()]);
}
/** create indexable fields for line geometry */
public static Field[] createIndexableFields(String fieldName, Line line) {
int numPoints = line.numPoints();
List<LatLonTriangle> fields = new ArrayList<>(numPoints - 1);
// encode the line vertices
int[] encodedLats = new int[numPoints];
int[] encodedLons = new int[numPoints];
for (int i = 0; i < numPoints; ++i) {
encodedLats[i] = encodeLatitude(line.getLat(i));
encodedLons[i] = encodeLongitude(line.getLon(i));
}
// create "flat" triangles
int aLat, bLat, aLon, bLon, temp;
for (int i = 0, j = 1; j < numPoints; ++i, ++j) {
aLat = encodedLats[i];
aLon = encodedLons[i];
bLat = encodedLats[j];
bLon = encodedLons[j];
if (aLat > bLat) {
temp = aLat;
aLat = bLat;
bLat = temp;
temp = aLon;
aLon = bLon;
bLon = temp;
} else if (aLat == bLat) {
if (aLon > bLon) {
temp = aLat;
aLat = bLat;
bLat = temp;
temp = aLon;
aLon = bLon;
bLon = temp;
}
}
fields.add(new LatLonTriangle(fieldName, aLon, aLat, bLon, bLat, aLon, aLat));
}
return fields.toArray(new Field[fields.size()]);
}
/** create indexable fields for point geometry */
public static Field[] createIndexableFields(String fieldName, double lat, double lon) {
final int encodedLat = encodeLatitude(lat);
final int encodedLon = encodeLongitude(lon);
return new Field[] {new LatLonTriangle(fieldName, encodedLon, encodedLat, encodedLon, encodedLat, encodedLon, encodedLat)};
}
/** create a query to find all polygons that intersect a defined bounding box
* note: does not currently support dateline crossing boxes
* todo split dateline crossing boxes into two queries like {@link LatLonPoint#newBoxQuery}
@ -89,11 +144,9 @@ public class LatLonShape {
*/
private static class LatLonTriangle extends Field {
public LatLonTriangle(String name, Triangle t) {
LatLonTriangle(String name, int ax, int ay, int bx, int by, int cx, int cy) {
super(name, TYPE);
setTriangleValue(t.getEncodedX(0), t.getEncodedY(0),
t.getEncodedX(1), t.getEncodedY(1),
t.getEncodedX(2), t.getEncodedY(2));
setTriangleValue(ax, ay, bx, by, cx, cy);
}
public void setTriangleValue(int aX, int aY, int bX, int bY, int cX, int cY) {

139
lucene/sandbox/src/java/org/apache/lucene/geo/Line.java Normal file
View File

@ -0,0 +1,139 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.geo;
import java.util.Arrays;
/**
* Represents a line on the earth's surface. You can construct the Line directly with {@code double[]}
* coordinates.
* <p>
* NOTES:
* <ol>
* <li>All latitude/longitude values must be in decimal degrees.
* <li>For more advanced GeoSpatial indexing and query operations see the {@code spatial-extras} module
* </ol>
* @lucene.experimental
*/
public class Line {
/** array of latitude coordinates */
private final double[] lats;
/** array of longitude coordinates */
private final double[] lons;
/** minimum latitude of this line's bounding box */
public final double minLat;
/** maximum latitude of this line's bounding box */
public final double maxLat;
/** minimum longitude of this line's bounding box */
public final double minLon;
/** maximum longitude of this line's bounding box */
public final double maxLon;
/**
* Creates a new Line from the supplied latitude/longitude array.
*/
public Line(double[] lats, double[] lons) {
if (lats == null) {
throw new IllegalArgumentException("lats must not be null");
}
if (lons == null) {
throw new IllegalArgumentException("lons must not be null");
}
if (lats.length != lons.length) {
throw new IllegalArgumentException("lats and lons must be equal length");
}
if (lats.length < 2) {
throw new IllegalArgumentException("at least 2 line points required");
}
// compute bounding box
double minLat = lats[0];
double minLon = lons[0];
double maxLat = lats[0];
double maxLon = lons[0];
for (int i = 0; i < lats.length; ++i) {
GeoUtils.checkLatitude(lats[i]);
GeoUtils.checkLongitude(lons[i]);
minLat = Math.min(lats[i], minLat);
minLon = Math.min(lons[i], minLon);
maxLat = Math.max(lats[i], maxLat);
maxLon = Math.max(lons[i], maxLon);
}
this.lats = lats.clone();
this.lons = lons.clone();
this.minLat = minLat;
this.maxLat = maxLat;
this.minLon = minLon;
this.maxLon = maxLon;
}
/** returns the number of vertex points */
public int numPoints() {
return lats.length;
}
/** Returns latitude value at given index */
public double getLat(int vertex) {
return lats[vertex];
}
/** Returns longitude value at given index */
public double getLon(int vertex) {
return lons[vertex];
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof Line)) return false;
Line line = (Line) o;
return Arrays.equals(lats, line.lats) && Arrays.equals(lons, line.lons);
}
@Override
public int hashCode() {
int result = Arrays.hashCode(lats);
result = 31 * result + Arrays.hashCode(lons);
return result;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("LINE(");
for (int i = 0; i < lats.length; i++) {
sb.append("[")
.append(lats[i])
.append(", ")
.append(lons[i])
.append("]");
}
sb.append(')');
return sb.toString();
}
/** prints polygons as geojson */
public String toGeoJSON() {
StringBuilder sb = new StringBuilder();
sb.append("[");
sb.append(Polygon.verticesToGeoJSON(lats, lons));
sb.append("]");
return sb.toString();
}
}

458
lucene/sandbox/src/test/org/apache/lucene/document/BaseLatLonShapeTestCase.java Normal file
View File

@ -0,0 +1,458 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.document;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.geo.GeoTestUtil;
import org.apache.lucene.geo.Line;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Polygon2D;
import org.apache.lucene.geo.Rectangle;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean;
import static com.carrotsearch.randomizedtesting.RandomizedTest.randomInt;
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitudeCeil;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitudeCeil;
import static org.apache.lucene.geo.GeoTestUtil.nextLatitude;
import static org.apache.lucene.geo.GeoTestUtil.nextLongitude;
public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
protected static final String FIELD_NAME = "shape";
protected abstract ShapeType getShapeType();
protected Object nextShape() {
return getShapeType().nextShape();
}
protected double quantizeLat(double rawLat) {
return decodeLatitude(encodeLatitude(rawLat));
}
protected double quantizeLatCeil(double rawLat) {
return decodeLatitude(encodeLatitudeCeil(rawLat));
}
protected double quantizeLon(double rawLon) {
return decodeLongitude(encodeLongitude(rawLon));
}
protected double quantizeLonCeil(double rawLon) {
return decodeLongitude(encodeLongitudeCeil(rawLon));
}
protected Polygon quantizePolygon(Polygon polygon) {
double[] lats = new double[polygon.numPoints()];
double[] lons = new double[polygon.numPoints()];
for (int i = 0; i < lats.length; ++i) {
lats[i] = quantizeLat(polygon.getPolyLat(i));
lons[i] = quantizeLon(polygon.getPolyLon(i));
}
return new Polygon(lats, lons);
}
protected abstract Field[] createIndexableFields(String field, Object shape);
private void addShapeToDoc(String field, Document doc, Object shape) {
Field[] fields = createIndexableFields(field, shape);
for (Field f : fields) {
doc.add(f);
}
}
protected Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon) {
return LatLonShape.newBoxQuery(field, minLat, maxLat, minLon, maxLon);
}
protected Query newPolygonQuery(String field, Polygon... polygons) {
return LatLonShape.newPolygonQuery(field, polygons);
}
public void testRandomTiny() throws Exception {
// Make sure single-leaf-node case is OK:
doTestRandom(10);
}
public void testRandomMedium() throws Exception {
doTestRandom(10000);
}
@Nightly
public void testRandomBig() throws Exception {
doTestRandom(50000);
}
private void doTestRandom(int count) throws Exception {
int numShapes = atLeast(count);
ShapeType type = getShapeType();
if (VERBOSE) {
System.out.println("TEST: number of " + type.name() + " shapes=" + numShapes);
}
Object[] shapes = new Object[numShapes];
for (int id = 0; id < numShapes; ++id) {
int x = randomInt(20);
if (x == 17) {
shapes[id] = null;
if (VERBOSE) {
System.out.println(" id=" + id + " is missing");
}
} else {
// create a new shape
shapes[id] = nextShape();
}
}
verify(shapes);
}
private void verify(Object... shapes) throws Exception {
IndexWriterConfig iwc = newIndexWriterConfig();
iwc.setMergeScheduler(new SerialMergeScheduler());
int mbd = iwc.getMaxBufferedDocs();
if (mbd != -1 && mbd < shapes.length / 100) {
iwc.setMaxBufferedDocs(shapes.length / 100);
}
Directory dir;
if (shapes.length > 1000) {
dir = newFSDirectory(createTempDir(getClass().getSimpleName()));
} else {
dir = newDirectory();
}
IndexWriter w = new IndexWriter(dir, iwc);
// index random polygons
indexRandomShapes(w, shapes);
// query testing
final IndexReader reader = DirectoryReader.open(w);
// test random bbox queries
verifyRandomBBoxQueries(reader, shapes);
// test random polygon queires
verifyRandomPolygonQueries(reader, shapes);
IOUtils.close(w, reader, dir);
}
protected void indexRandomShapes(IndexWriter w, Object... shapes) throws Exception {
Set<Integer> deleted = new HashSet<>();
for (int id = 0; id < shapes.length; ++id) {
Document doc = new Document();
doc.add(newStringField("id", "" + id, Field.Store.NO));
doc.add(new NumericDocValuesField("id", id));
if (shapes[id] != null) {
addShapeToDoc(FIELD_NAME, doc, shapes[id]);
}
w.addDocument(doc);
if (id > 0 && randomInt(100) == 42) {
int idToDelete = randomInt(id);
w.deleteDocuments(new Term("id", ""+idToDelete));
deleted.add(idToDelete);
if (VERBOSE) {
System.out.println(" delete id=" + idToDelete);
}
}
}
if (randomBoolean()) {
w.forceMerge(1);
}
}
protected void verifyRandomBBoxQueries(IndexReader reader, Object... shapes) throws Exception {
IndexSearcher s = newSearcher(reader);
final int iters = atLeast(75);
Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader());
int maxDoc = s.getIndexReader().maxDoc();
for (int iter = 0; iter < iters; ++iter) {
if (VERBOSE) {
System.out.println("\nTEST: iter=" + (iter+1) + " of " + iters + " s=" + s);
}
// BBox
Rectangle rect;
// quantizing the bbox may end up w/ bounding boxes crossing dateline...
// todo add support for bounding boxes crossing dateline
while (true) {
rect = GeoTestUtil.nextBoxNotCrossingDateline();
if (decodeLongitude(encodeLongitudeCeil(rect.minLon)) <= decodeLongitude(encodeLongitude(rect.maxLon)) &&
decodeLatitude(encodeLatitudeCeil(rect.minLat)) <= decodeLatitude(encodeLatitude(rect.maxLat))) {
break;
}
}
Query query = newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon);
if (VERBOSE) {
System.out.println(" query=" + query);
}
final FixedBitSet hits = new FixedBitSet(maxDoc);
s.search(query, new SimpleCollector() {
private int docBase;
@Override
public ScoreMode scoreMode() {
return ScoreMode.COMPLETE_NO_SCORES;
}
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
docBase = context.docBase;
}
@Override
public void collect(int doc) throws IOException {
hits.set(docBase+doc);
}
});
boolean fail = false;
NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
for (int docID = 0; docID < maxDoc; ++docID) {
assertEquals(docID, docIDToID.nextDoc());
int id = (int) docIDToID.longValue();
boolean expected;
if (liveDocs != null && liveDocs.get(docID) == false) {
// document is deleted
expected = false;
} else if (shapes[id] == null) {
expected = false;
} else {
// check quantized poly against quantized query
expected = getValidator().testBBoxQuery(quantizeLatCeil(rect.minLat), quantizeLat(rect.maxLat),
quantizeLonCeil(rect.minLon), quantizeLon(rect.maxLon), shapes[id]);
}
if (hits.get(docID) != expected) {
StringBuilder b = new StringBuilder();
if (expected) {
b.append("FAIL: id=" + id + " should match but did not\n");
} else {
b.append("FAIL: id=" + id + " should not match but did\n");
}
b.append(" query=" + query + " docID=" + docID + "\n");
b.append(" shape=" + shapes[id] + "\n");
b.append(" deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
b.append(" rect=Rectangle(" + quantizeLatCeil(rect.minLat) + " TO " + quantizeLat(rect.maxLat) + " lon=" + quantizeLonCeil(rect.minLon) + " TO " + quantizeLon(rect.maxLon) + ")");
if (true) {
fail("wrong hit (first of possibly more):\n\n" + b);
} else {
System.out.println(b.toString());
fail = true;
}
}
}
if (fail) {
fail("some hits were wrong");
}
}
}
protected void verifyRandomPolygonQueries(IndexReader reader, Object... shapes) throws Exception {
IndexSearcher s = newSearcher(reader);
final int iters = atLeast(75);
Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader());
int maxDoc = s.getIndexReader().maxDoc();
for (int iter = 0; iter < iters; ++iter) {
if (VERBOSE) {
System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
}
// Polygon
Polygon queryPolygon = GeoTestUtil.nextPolygon();
Polygon2D queryPoly2D = Polygon2D.create(queryPolygon);
Query query = newPolygonQuery(FIELD_NAME, queryPolygon);
if (VERBOSE) {
System.out.println(" query=" + query);
}
final FixedBitSet hits = new FixedBitSet(maxDoc);
s.search(query, new SimpleCollector() {
private int docBase;
@Override
public ScoreMode scoreMode() {
return ScoreMode.COMPLETE_NO_SCORES;
}
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
docBase = context.docBase;
}
@Override
public void collect(int doc) throws IOException {
hits.set(docBase+doc);
}
});
boolean fail = false;
NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
for (int docID = 0; docID < maxDoc; ++docID) {
assertEquals(docID, docIDToID.nextDoc());
int id = (int) docIDToID.longValue();
boolean expected;
if (liveDocs != null && liveDocs.get(docID) == false) {
// document is deleted
expected = false;
} else if (shapes[id] == null) {
expected = false;
} else {
expected = getValidator().testPolygonQuery(queryPoly2D, shapes[id]);
}
if (hits.get(docID) != expected) {
StringBuilder b = new StringBuilder();
if (expected) {
b.append("FAIL: id=" + id + " should match but did not\n");
} else {
b.append("FAIL: id=" + id + " should not match but did\n");
}
b.append(" query=" + query + " docID=" + docID + "\n");
b.append(" shape=" + shapes[id] + "\n");
b.append(" deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
b.append(" queryPolygon=" + queryPolygon.toGeoJSON());
if (true) {
fail("wrong hit (first of possibly more):\n\n" + b);
} else {
System.out.println(b.toString());
fail = true;
}
}
}
if (fail) {
fail("some hits were wrong");
}
}
}
protected abstract Validator getValidator();
/** internal point class for testing point shapes */
protected static class Point {
double lat;
double lon;
public Point(double lat, double lon) {
this.lat = lat;
this.lon = lon;
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("POINT(");
sb.append(lon);
sb.append(',');
sb.append(lat);
return sb.toString();
}
}
/** internal shape type for testing different shape types */
protected enum ShapeType {
POINT() {
public Point nextShape() {
return new Point(nextLatitude(), nextLongitude());
}
},
LINE() {
public Line nextShape() {
Polygon p = GeoTestUtil.nextPolygon();
double[] lats = new double[p.numPoints() - 1];
double[] lons = new double[lats.length];
for (int i = 0; i < lats.length; ++i) {
lats[i] = p.getPolyLat(i);
lons[i] = p.getPolyLon(i);
}
return new Line(lats, lons);
}
},
POLYGON() {
public Polygon nextShape() {
return GeoTestUtil.nextPolygon();
}
},
MIXED() {
public Object nextShape() {
return RandomPicks.randomFrom(random(), subList).nextShape();
}
};
static ShapeType[] subList;
static {
subList = new ShapeType[] {POINT, LINE, POLYGON};
}
public abstract Object nextShape();
static ShapeType fromObject(Object shape) {
if (shape instanceof Point) {
return POINT;
} else if (shape instanceof Line) {
return LINE;
} else if (shape instanceof Polygon) {
return POLYGON;
}
throw new IllegalArgumentException("invalid shape type from " + shape.toString());
}
}
protected interface Validator {
boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape);
boolean testPolygonQuery(Polygon2D poly2d, Object shape);
}
}

94
lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonLineShapeQueries.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.document;
import org.apache.lucene.geo.Line;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Polygon2D;
import org.apache.lucene.index.PointValues.Relation;
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
/** random bounding box and polygon query tests for random generated {@link Line} types */
public class TestLatLonLineShapeQueries extends BaseLatLonShapeTestCase {
protected final LineValidator VALIDATOR = new LineValidator();
@Override
protected ShapeType getShapeType() {
return ShapeType.LINE;
}
@Override
protected Field[] createIndexableFields(String field, Object line) {
return LatLonShape.createIndexableFields(field, (Line)line);
}
@Override
protected Validator getValidator() {
return VALIDATOR;
}
protected class LineValidator implements Validator {
@Override
public boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape) {
// to keep it simple we convert the bbox into a polygon and use poly2d
Polygon2D p = Polygon2D.create(new Polygon[] {new Polygon(new double[] {minLat, minLat, maxLat, maxLat, minLat},
new double[] {minLon, maxLon, maxLon, minLon, minLon})});
return testLine(p, (Line)shape);
}
@Override
public boolean testPolygonQuery(Polygon2D poly2d, Object shape) {
return testLine(poly2d, (Line) shape);
}
private boolean testLine(Polygon2D queryPoly, Line line) {
double ax, ay, bx, by, temp;
for (int i = 0, j = 1; j < line.numPoints(); ++i, ++j) {
ay = decodeLatitude(encodeLatitude(line.getLat(i)));
ax = decodeLongitude(encodeLongitude(line.getLon(i)));
by = decodeLatitude(encodeLatitude(line.getLat(j)));
bx = decodeLongitude(encodeLongitude(line.getLon(j)));
if (ay > by) {
temp = ay;
ay = by;
by = temp;
temp = ax;
ax = bx;
bx = temp;
} else if (ay == by) {
if (ax > bx) {
temp = ay;
ay = by;
by = temp;
temp = ax;
ax = bx;
bx = temp;
}
}
if (queryPoly.relateTriangle(ax, ay, bx, by, ax, ay) != Relation.CELL_OUTSIDE_QUERY) {
return true;
}
}
return false;
}
}
}

66
lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointShapeQueries.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.document;
import org.apache.lucene.geo.Polygon2D;
import org.apache.lucene.index.PointValues.Relation;
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
/** random bounding box and polygon query tests for random generated {@code latitude, longitude} points */
public class TestLatLonPointShapeQueries extends BaseLatLonShapeTestCase {
protected final PointValidator VALIDATOR = new PointValidator();
@Override
protected ShapeType getShapeType() {
return ShapeType.POINT;
}
@Override
protected Field[] createIndexableFields(String field, Object point) {
Point p = (Point)point;
return LatLonShape.createIndexableFields(field, p.lat, p.lon);
}
@Override
protected Validator getValidator() {
return VALIDATOR;
}
protected class PointValidator implements Validator {
@Override
public boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape) {
Point p = (Point)shape;
double lat = decodeLatitude(encodeLatitude(p.lat));
double lon = decodeLongitude(encodeLongitude(p.lon));
return (lat < minLat || lat > maxLat || lon < minLon || lon > maxLon) == false;
}
@Override
public boolean testPolygonQuery(Polygon2D poly2d, Object shape) {
Point p = (Point) shape;
double lat = decodeLatitude(encodeLatitude(p.lat));
double lon = decodeLongitude(encodeLongitude(p.lon));
// for consistency w/ the query we test the point as a triangle
return poly2d.relateTriangle(lon, lat, lon, lat, lon, lat) != Relation.CELL_OUTSIDE_QUERY;
}
}
}

395
lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
View File

@ -16,378 +16,67 @@
*/
package org.apache.lucene.document;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.lucene.geo.GeoTestUtil;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Polygon2D;
import org.apache.lucene.geo.Rectangle;
import org.apache.lucene.geo.Tessellator;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitudeCeil;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitudeCeil;
public class TestLatLonPolygonShapeQueries extends BaseLatLonShapeTestCase {
/** base Test case for {@link LatLonShape} indexing and search */
public class TestLatLonPolygonShapeQueries extends LuceneTestCase {
protected static final String FIELD_NAME = "shape";
protected final PolygonValidator VALIDATOR = new PolygonValidator();
private Polygon quantizePolygon(Polygon polygon) {
double[] lats = new double[polygon.numPoints()];
double[] lons = new double[polygon.numPoints()];
for (int i = 0; i < lats.length; ++i) {
lats[i] = quantizeLat(polygon.getPolyLat(i));
lons[i] = quantizeLon(polygon.getPolyLon(i));
}
return new Polygon(lats, lons);
@Override
protected ShapeType getShapeType() {
return ShapeType.POLYGON;
}
protected double quantizeLat(double rawLat) {
return decodeLatitude(encodeLatitude(rawLat));
}
protected double quantizeLatCeil(double rawLat) {
return decodeLatitude(encodeLatitudeCeil(rawLat));
}
protected double quantizeLon(double rawLon) {
return decodeLongitude(encodeLongitude(rawLon));
}
protected double quantizeLonCeil(double rawLon) {
return decodeLongitude(encodeLongitudeCeil(rawLon));
}
protected void addPolygonsToDoc(String field, Document doc, Polygon polygon) {
Field[] fields = LatLonShape.createIndexableFields(field, polygon);
for (Field f : fields) {
doc.add(f);
}
}
protected Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon) {
return LatLonShape.newBoxQuery(field, minLat, maxLat, minLon, maxLon);
}
protected Query newPolygonQuery(String field, Polygon... polygons) {
return LatLonShape.newPolygonQuery(field, polygons);
}
public void testRandomTiny() throws Exception {
// Make sure single-leaf-node case is OK:
doTestRandom(10);
}
public void testRandomMedium() throws Exception {
doTestRandom(10000);
}
@Nightly
public void testRandomBig() throws Exception {
doTestRandom(50000);
}
private void doTestRandom(int count) throws Exception {
int numPolygons = atLeast(count);
if (VERBOSE) {
System.out.println("TEST: numPolygons=" + numPolygons);
}
Polygon[] polygons = new Polygon[numPolygons];
for (int id = 0; id < numPolygons; ++id) {
int x = random().nextInt(20);
if (x == 17) {
polygons[id] = null;
if (VERBOSE) {
System.out.println(" id=" + id + " is missing");
}
} else {
// create a polygon that does not cross the dateline
polygons[id] = GeoTestUtil.nextPolygon();
}
}
verify(polygons);
}
private void verify(Polygon... polygons) throws Exception {
ArrayList<Polygon2D> poly2d = new ArrayList<>();
poly2d.ensureCapacity(polygons.length);
// index random polygons; poly2d will contain the Polygon2D objects needed for verification
IndexWriter w = indexRandomPolygons(poly2d, polygons);
Directory dir = w.getDirectory();
final IndexReader reader = DirectoryReader.open(w);
// test random bbox queries
verifyRandomBBoxQueries(reader, poly2d, polygons);
// test random polygon queires
verifyRandomPolygonQueries(reader, poly2d, polygons);
IOUtils.close(w, reader, dir);
}
protected IndexWriter indexRandomPolygons(List<Polygon2D> poly2d, Polygon... polygons) throws Exception {
IndexWriterConfig iwc = newIndexWriterConfig();
iwc.setMergeScheduler(new SerialMergeScheduler());
int mbd = iwc.getMaxBufferedDocs();
if (mbd != -1 && mbd < polygons.length / 100) {
iwc.setMaxBufferedDocs(polygons.length / 100);
}
Directory dir;
if (polygons.length > 1000) {
dir = newFSDirectory(createTempDir(getClass().getSimpleName()));
} else {
dir = newDirectory();
}
Set<Integer> deleted = new HashSet<>();
IndexWriter w = new IndexWriter(dir, iwc);
for (int id = 0; id < polygons.length; ++id) {
Document doc = new Document();
doc.add(newStringField("id", "" + id, Field.Store.NO));
doc.add(new NumericDocValuesField("id", id));
if (polygons[id] != null) {
@Override
protected Polygon nextShape() {
Polygon p;
while (true) {
// if we can't tessellate; then random polygon generator created a malformed shape
p = (Polygon)getShapeType().nextShape();
try {
addPolygonsToDoc(FIELD_NAME, doc, polygons[id]);
} catch (IllegalArgumentException e) {
// GeoTestUtil will occassionally create invalid polygons
// invalid polygons will not tessellate
// we skip those polygons that will not tessellate, relying on the TestTessellator class
// to ensure the Tessellator correctly identified a malformed shape and its not a bug
if (VERBOSE) {
System.out.println(" id=" + id + " could not tessellate. Malformed shape " + polygons[id] + " detected");
}
// remove and skip the malformed shape
polygons[id] = null;
poly2d.add(id, null);
continue;
}
poly2d.add(id, Polygon2D.create(quantizePolygon(polygons[id])));
} else {
poly2d.add(id, null);
}
w.addDocument(doc);
if (id > 0 && random().nextInt(100) == 42) {
int idToDelete = random().nextInt(id);
w.deleteDocuments(new Term("id", ""+idToDelete));
deleted.add(idToDelete);
if (VERBOSE) {
System.out.println(" delete id=" + idToDelete);
}
}
}
if (random().nextBoolean()) {
w.forceMerge(1);
}
return w;
}
protected void verifyRandomBBoxQueries(IndexReader reader, List<Polygon2D> poly2d, Polygon... polygons) throws Exception {
IndexSearcher s = newSearcher(reader);
final int iters = atLeast(75);
Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader());
int maxDoc = s.getIndexReader().maxDoc();
for (int iter = 0; iter < iters; ++iter) {
if (VERBOSE) {
System.out.println("\nTEST: iter=" + (iter+1) + " of " + iters + " s=" + s);
}
// BBox
Rectangle rect = GeoTestUtil.nextBoxNotCrossingDateline();
Query query = newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon);
if (VERBOSE) {
System.out.println(" query=" + query);
}
final FixedBitSet hits = new FixedBitSet(maxDoc);
s.search(query, new SimpleCollector() {
private int docBase;
@Override
public ScoreMode scoreMode() {
return ScoreMode.COMPLETE_NO_SCORES;
}
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
docBase = context.docBase;
}
@Override
public void collect(int doc) throws IOException {
hits.set(docBase+doc);
}
});
boolean fail = false;
NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
for (int docID = 0; docID < maxDoc; ++docID) {
assertEquals(docID, docIDToID.nextDoc());
int id = (int) docIDToID.longValue();
boolean expected;
if (liveDocs != null && liveDocs.get(docID) == false) {
// document is deleted
expected = false;
} else if (polygons[id] == null) {
expected = false;
} else {
// check quantized poly against quantized query
expected = poly2d.get(id).relate(quantizeLatCeil(rect.minLat), quantizeLat(rect.maxLat),
quantizeLonCeil(rect.minLon), quantizeLon(rect.maxLon)) != Relation.CELL_OUTSIDE_QUERY;
}
if (hits.get(docID) != expected) {
StringBuilder b = new StringBuilder();
if (expected) {
b.append("FAIL: id=" + id + " should match but did not\n");
} else {
b.append("FAIL: id=" + id + " should not match but did\n");
}
b.append(" query=" + query + " docID=" + docID + "\n");
b.append(" polygon=" + quantizePolygon(polygons[id]) + "\n");
b.append(" deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
b.append(" rect=Rectangle(" + quantizeLatCeil(rect.minLat) + " TO " + quantizeLat(rect.maxLat) + " lon=" + quantizeLonCeil(rect.minLon) + " TO " + quantizeLon(rect.maxLon) + ")");
if (true) {
fail("wrong hit (first of possibly more):\n\n" + b);
} else {
System.out.println(b.toString());
fail = true;
}
}
}
if (fail) {
fail("some hits were wrong");
}
}
}
protected void verifyRandomPolygonQueries(IndexReader reader, List<Polygon2D> poly2d, Polygon... polygons) throws Exception {
IndexSearcher s = newSearcher(reader);
final int iters = atLeast(75);
Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader());
int maxDoc = s.getIndexReader().maxDoc();
for (int iter = 0; iter < iters; ++iter) {
if (VERBOSE) {
System.out.println("\nTEST: iter=" + (iter+1) + " of " + iters + " s=" + s);
}
// Polygon
Polygon queryPolygon = GeoTestUtil.nextPolygon();
Polygon2D queryPoly2D = Polygon2D.create(queryPolygon);
Query query = newPolygonQuery(FIELD_NAME, queryPolygon);
if (VERBOSE) {
System.out.println(" query=" + query);
}
final FixedBitSet hits = new FixedBitSet(maxDoc);
s.search(query, new SimpleCollector() {
private int docBase;
@Override
public ScoreMode scoreMode() {
return ScoreMode.COMPLETE_NO_SCORES;
}
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
docBase = context.docBase;
}
@Override
public void collect(int doc) throws IOException {
hits.set(docBase+doc);
}
});
boolean fail = false;
NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
for (int docID = 0; docID < maxDoc; ++docID) {
assertEquals(docID, docIDToID.nextDoc());
int id = (int) docIDToID.longValue();
boolean expected;
if (liveDocs != null && liveDocs.get(docID) == false) {
// document is deleted
expected = false;
} else if (polygons[id] == null) {
expected = false;
} else {
expected = false;
try {
// check poly (quantized the same way as indexed) against query polygon
List<Tessellator.Triangle> tesselation = Tessellator.tessellate(quantizePolygon(polygons[id]));
for (Tessellator.Triangle t : tesselation) {
if (queryPoly2D.relateTriangle(t.getLon(0), t.getLat(0),
t.getLon(1), t.getLat(1), t.getLon(2), t.getLat(2)) != Relation.CELL_OUTSIDE_QUERY) {
expected = true;
break;
}
}
Tessellator.tessellate(p);
return p;
} catch (IllegalArgumentException e) {
continue;
}
}
}
if (hits.get(docID) != expected) {
StringBuilder b = new StringBuilder();
@Override
protected Field[] createIndexableFields(String field, Object polygon) {
return LatLonShape.createIndexableFields(field, (Polygon)polygon);
}
if (expected) {
b.append("FAIL: id=" + id + " should match but did not\n");
} else {
b.append("FAIL: id=" + id + " should not match but did\n");
}
b.append(" query=" + query + " docID=" + docID + "\n");
b.append(" polygon=" + quantizePolygon(polygons[id]).toGeoJSON() + "\n");
b.append(" deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
b.append(" queryPolygon=" + queryPolygon.toGeoJSON());
if (true) {
fail("wrong hit (first of possibly more):\n\n" + b);
} else {
System.out.println(b.toString());
fail = true;
}
}
}
if (fail) {
fail("some hits were wrong");
@Override
protected Validator getValidator() {
return VALIDATOR;
}
protected class PolygonValidator implements Validator {
@Override
public boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape) {
Polygon2D poly = Polygon2D.create(quantizePolygon((Polygon)shape));
return poly.relate(minLat, maxLat, minLon, maxLon) != Relation.CELL_OUTSIDE_QUERY;
}
@Override
public boolean testPolygonQuery(Polygon2D query, Object shape) {
List<Tessellator.Triangle> tessellation = Tessellator.tessellate((Polygon) shape);
for (Tessellator.Triangle t : tessellation) {
// we quantize the triangle for consistency with the index
if (query.relateTriangle(quantizeLon(t.getLon(0)), quantizeLat(t.getLat(0)),
quantizeLon(t.getLon(1)), quantizeLat(t.getLat(1)),
quantizeLon(t.getLon(2)), quantizeLat(t.getLat(2))) != Relation.CELL_OUTSIDE_QUERY) {
return true;
}
}
return false;
}
}
}

31
lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
View File

@ -18,6 +18,7 @@ package org.apache.lucene.document;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import org.apache.lucene.geo.GeoTestUtil;
import org.apache.lucene.geo.Line;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@ -43,6 +44,13 @@ public class TestLatLonShape extends LuceneTestCase {
}
}
protected void addLineToDoc(String field, Document doc, Line line) {
Field[] fields = LatLonShape.createIndexableFields(field, line);
for (Field f : fields) {
doc.add(f);
}
}
protected Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon) {
return LatLonShape.newBoxQuery(field, minLat, maxLat, minLon, maxLon);
}
@ -81,19 +89,36 @@ public class TestLatLonShape extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
// add a random polygon
// add a random polygon document
Polygon p = GeoTestUtil.createRegularPolygon(0, 90, atLeast(1000000), numVertices);
Document document = new Document();
addPolygonsToDoc(FIELDNAME, document, p);
writer.addDocument(document);
// add a line document
document = new Document();
// add a line string
double lats[] = new double[p.numPoints() - 1];
double lons[] = new double[p.numPoints() - 1];
for (int i = 0; i < lats.length; ++i) {
lats[i] = p.getPolyLat(i);
lons[i] = p.getPolyLon(i);
}
Line l = new Line(lats, lons);
addLineToDoc(FIELDNAME, document, l);
writer.addDocument(document);
////// search /////
// search an intersecting bbox
IndexReader reader = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(reader);
Query q = newRectQuery(FIELDNAME, -1d, 1d, p.minLon, p.maxLon);
assertEquals(1, searcher.count(q));
double minLat = Math.min(lats[0], lats[1]);
double minLon = Math.min(lons[0], lons[1]);
double maxLat = Math.max(lats[0], lats[1]);
double maxLon = Math.max(lons[0], lons[1]);
Query q = newRectQuery(FIELDNAME, minLat, maxLat, minLon, maxLon);
assertEquals(2, searcher.count(q));
// search a disjoint bbox
q = newRectQuery(FIELDNAME, p.minLat-1d, p.minLat+1, p.minLon-1d, p.minLon+1d);