diff --git a/lucene/sandbox/src/java/org/apache/lucene/util/GeoRect.java b/lucene/sandbox/src/java/org/apache/lucene/util/GeoRect.java new file mode 100644 index 00000000000..1e9317c9238 --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/util/GeoRect.java @@ -0,0 +1,67 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Represents a lat/lon rectangle. */ +public class GeoRect { + public final double minLon; + public final double maxLon; + public final double minLat; + public final double maxLat; + + public GeoRect(double minLon, double maxLon, double minLat, double maxLat) { + if (GeoUtils.isValidLon(minLon) == false) { + throw new IllegalArgumentException("invalid minLon " + minLon); + } + if (GeoUtils.isValidLon(maxLon) == false) { + throw new IllegalArgumentException("invalid maxLon " + maxLon); + } + if (GeoUtils.isValidLat(minLat) == false) { + throw new IllegalArgumentException("invalid minLat " + minLat); + } + if (GeoUtils.isValidLat(maxLat) == false) { + throw new IllegalArgumentException("invalid maxLat " + maxLat); + } + this.minLon = minLon; + this.maxLon = maxLon; + this.minLat = minLat; + this.maxLat = maxLat; + assert maxLat >= minLat; + + // NOTE: cannot assert maxLon >= minLon since this rect could cross the dateline + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append("GeoRect(lon="); + b.append(minLon); + b.append(" TO "); + b.append(maxLon); + if (maxLon < minLon) { + b.append(" (crosses dateline!)"); + } + b.append(" lat="); + b.append(minLat); + b.append(" TO "); + b.append(maxLat); + b.append(")"); + + return b.toString(); + } +} diff --git a/lucene/sandbox/src/test/org/apache/lucene/util/BaseGeoPointTestCase.java b/lucene/sandbox/src/test/org/apache/lucene/util/BaseGeoPointTestCase.java new file mode 100644 index 00000000000..3a11d7b46ae --- /dev/null +++ b/lucene/sandbox/src/test/org/apache/lucene/util/BaseGeoPointTestCase.java @@ -0,0 +1,759 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiDocValues; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SimpleCollector; +import org.apache.lucene.store.Directory; +import org.junit.BeforeClass; + +// TODO: cutover TestGeoUtils too? + +public abstract class BaseGeoPointTestCase extends LuceneTestCase { + + protected static final String FIELD_NAME = "point"; + + private static final double LON_SCALE = (0x1L< 0 && x == 14 && haveRealDoc) { + int oldDocID; + while (true) { + oldDocID = random().nextInt(docID); + if (Double.isNaN(lats[oldDocID]) == false) { + break; + } + } + + // Fully identical point: + lons[docID] = lons[oldDocID]; + if (VERBOSE) { + System.out.println(" doc=" + docID + " lat=" + lat + " lon=" + lons[docID] + " (same lat/lon as doc=" + oldDocID + ")"); + } + } else { + lons[docID] = randomLon(small); + haveRealDoc = true; + if (VERBOSE) { + System.out.println(" doc=" + docID + " lat=" + lat + " lon=" + lons[docID]); + } + } + lats[docID] = lat; + } + + verify(small, lats, lons); + } + + @Nightly + public void testAllLonEqual() throws Exception { + int numPoints = atLeast(10000); + // TODO: GeoUtils are potentially slow if we use small=false with heavy testing + // boolean small = random().nextBoolean(); + boolean small = true; + double theLon = randomLon(small); + double[] lats = new double[numPoints]; + double[] lons = new double[numPoints]; + + boolean haveRealDoc = false; + + //System.out.println("theLon=" + theLon); + + for(int docID=0;docID 0 && x == 14 && haveRealDoc) { + int oldDocID; + while (true) { + oldDocID = random().nextInt(docID); + if (Double.isNaN(lats[oldDocID]) == false) { + break; + } + } + + // Fully identical point: + lats[docID] = lats[oldDocID]; + if (VERBOSE) { + System.out.println(" doc=" + docID + " lat=" + lats[docID] + " lon=" + theLon + " (same lat/lon as doc=" + oldDocID + ")"); + } + } else { + lats[docID] = randomLat(small); + haveRealDoc = true; + if (VERBOSE) { + System.out.println(" doc=" + docID + " lat=" + lats[docID] + " lon=" + theLon); + } + } + lons[docID] = theLon; + } + + verify(small, lats, lons); + } + + @Nightly + public void testMultiValued() throws Exception { + int numPoints = atLeast(10000); + // Every doc has 2 points: + double[] lats = new double[2*numPoints]; + double[] lons = new double[2*numPoints]; + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(); + initIndexWriterConfig(FIELD_NAME, iwc); + + // We rely on docID order: + iwc.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + + // TODO: GeoUtils are potentially slow if we use small=false with heavy testing + boolean small = random().nextBoolean(); + //boolean small = true; + + for (int id=0;id 0 && x < 3 && haveRealDoc) { + int oldID; + while (true) { + oldID = random().nextInt(id); + if (Double.isNaN(lats[oldID]) == false) { + break; + } + } + + if (x == 0) { + // Identical lat to old point + lats[id] = lats[oldID]; + lons[id] = randomLon(small); + if (VERBOSE) { + System.out.println(" id=" + id + " lat=" + lats[id] + " lon=" + lons[id] + " (same lat as doc=" + oldID + ")"); + } + } else if (x == 1) { + // Identical lon to old point + lats[id] = randomLat(small); + lons[id] = lons[oldID]; + if (VERBOSE) { + System.out.println(" id=" + id + " lat=" + lats[id] + " lon=" + lons[id] + " (same lon as doc=" + oldID + ")"); + } + } else { + assert x == 2; + // Fully identical point: + lats[id] = lats[oldID]; + lons[id] = lons[oldID]; + if (VERBOSE) { + System.out.println(" id=" + id + " lat=" + lats[id] + " lon=" + lons[id] + " (same lat/lon as doc=" + oldID + ")"); + } + } + } else { + lats[id] = randomLat(small); + lons[id] = randomLon(small); + haveRealDoc = true; + if (VERBOSE) { + System.out.println(" id=" + id + " lat=" + lats[id] + " lon=" + lons[id]); + } + } + } + + verify(small, lats, lons); + } + + public long scaleLon(final double val) { + return (long) ((val-GeoUtils.MIN_LON_INCL) * LON_SCALE); + } + + public long scaleLat(final double val) { + return (long) ((val-GeoUtils.MIN_LAT_INCL) * LAT_SCALE); + } + + public double unscaleLon(final long val) { + return (val / LON_SCALE) + GeoUtils.MIN_LON_INCL; + } + + public double unscaleLat(final long val) { + return (val / LAT_SCALE) + GeoUtils.MIN_LAT_INCL; + } + + public double randomLat(boolean small) { + double result; + if (small) { + result = GeoUtils.normalizeLat(originLat + latRange * (random().nextDouble() - 0.5)); + } else { + result = -90 + 180.0 * random().nextDouble(); + } + return unscaleLat(scaleLat(result)); + } + + public double randomLon(boolean small) { + double result; + if (small) { + result = GeoUtils.normalizeLon(originLon + lonRange * (random().nextDouble() - 0.5)); + } else { + result = -180 + 360.0 * random().nextDouble(); + } + return unscaleLon(scaleLon(result)); + } + + protected GeoRect randomRect(boolean small, boolean canCrossDateLine) { + double lat0 = randomLat(small); + double lat1 = randomLat(small); + double lon0 = randomLon(small); + double lon1 = randomLon(small); + + if (lat1 < lat0) { + double x = lat0; + lat0 = lat1; + lat1 = x; + } + + if (canCrossDateLine == false && lon1 < lon0) { + double x = lon0; + lon0 = lon1; + lon1 = x; + } + + return new GeoRect(lon0, lon1, lat0, lat1); + } + + protected void initIndexWriterConfig(String field, IndexWriterConfig iwc) { + } + + protected abstract void addPointToDoc(String field, Document doc, double lat, double lon); + + protected abstract Query newBBoxQuery(String field, GeoRect bbox); + + protected abstract Query newDistanceQuery(String field, double centerLat, double centerLon, double radiusMeters); + + protected abstract Query newDistanceRangeQuery(String field, double centerLat, double centerLon, double minRadiusMeters, double radiusMeters); + + protected abstract Query newPolygonQuery(String field, double[] lats, double[] lons); + + /** Returns null if it's borderline case */ + protected abstract Boolean rectContainsPoint(GeoRect rect, double pointLat, double pointLon); + + /** Returns null if it's borderline case */ + protected abstract Boolean polyRectContainsPoint(GeoRect rect, double pointLat, double pointLon); + + /** Returns null if it's borderline case */ + protected abstract Boolean circleContainsPoint(double centerLat, double centerLon, double radiusMeters, double pointLat, double pointLon); + + protected abstract Boolean distanceRangeContainsPoint(double centerLat, double centerLon, double minRadiusMeters, double radiusMeters, double pointLat, double pointLon); + + private static abstract class VerifyHits { + + public void test(boolean small, IndexSearcher s, NumericDocValues docIDToID, Set deleted, Query query, double[] lats, double[] lons) throws Exception { + int maxDoc = s.getIndexReader().maxDoc(); + final FixedBitSet hits = new FixedBitSet(maxDoc); + s.search(query, new SimpleCollector() { + + private int docBase; + + @Override + public boolean needsScores() { + return false; + } + + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + docBase = context.docBase; + } + + @Override + public void collect(int doc) { + hits.set(docBase+doc); + } + }); + + boolean fail = false; + + for(int docID=0;docID 100000) { + dir = newFSDirectory(createTempDir(getClass().getSimpleName())); + } else { + dir = newDirectory(); + } + + Set deleted = new HashSet<>(); + // RandomIndexWriter is too slow here: + IndexWriter w = new IndexWriter(dir, iwc); + for(int id=0;id 0 && random().nextInt(100) == 42) { + int idToDelete = random().nextInt(id); + w.deleteDocuments(new Term("id", ""+idToDelete)); + deleted.add(idToDelete); + if (VERBOSE) { + System.out.println(" delete id=" + idToDelete); + } + } + } + + if (random().nextBoolean()) { + w.forceMerge(1); + } + final IndexReader r = DirectoryReader.open(w, true); + w.close(); + + // We can't wrap with "exotic" readers because the BKD query must see the BKDDVFormat: + IndexSearcher s = newSearcher(r, false); + + // Make sure queries are thread safe: + int numThreads = TestUtil.nextInt(random(), 2, 5); + + List threads = new ArrayList<>(); + final int iters = atLeast(75); + + final CountDownLatch startingGun = new CountDownLatch(1); + final AtomicBoolean failed = new AtomicBoolean(); + + for(int i=0;i