diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java index b4435e7c140..c2e2c7d6bcc 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java @@ -115,8 +115,8 @@ public class TestFieldMaskingSpanQuery extends LuceneTestCase { field("gender", "male"), field("first", "bubba"), field("last", "jones") })); - reader = writer.getReader(); writer.forceMerge(1); + reader = writer.getReader(); writer.close(); searcher = new IndexSearcher(getOnlyLeafReader(reader)); } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index 0e5d898e41b..cd38b0ebd9f 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.spatial.util.GeoTestUtil; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; @@ -182,8 +183,8 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase { doc.add(new StoredField("id", i)); doc.add(new NumericDocValuesField("id", i)); if (random().nextInt(10) > 7) { - double latRaw = -90 + 180.0 * random().nextDouble(); - double lonRaw = -180 + 360.0 * random().nextDouble(); + double latRaw = GeoTestUtil.nextLatitude(); + double lonRaw = GeoTestUtil.nextLongitude(); // pre-normalize up front, so we can just use quantized value for testing and do simple exact comparisons double lat = LatLonPoint.decodeLatitude(LatLonPoint.encodeLatitude(latRaw)); double lon = LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(lonRaw)); @@ -198,8 +199,8 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase { IndexSearcher searcher = newSearcher(reader); for (int i = 0; i < numQueries; i++) { - double lat = -90 + 180.0 * random().nextDouble(); - double lon = -180 + 360.0 * random().nextDouble(); + double lat = GeoTestUtil.nextLatitude(); + double lon = GeoTestUtil.nextLongitude(); double missingValue = Double.POSITIVE_INFINITY; Result expected[] = new Result[reader.maxDoc()]; diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index e404032afed..595891046b8 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -19,13 +19,11 @@ package org.apache.lucene.spatial.util; import java.io.IOException; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; -import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.HashSet; import java.util.Locale; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.FilterCodec; @@ -45,6 +43,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.SegmentReadState; @@ -57,6 +56,7 @@ import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; @@ -82,41 +82,11 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase { private static double originLat; private static double originLon; - private static double lonRange; - private static double latRange; @BeforeClass public static void beforeClassBase() throws Exception { - // Between 1.0 and 3.0: - lonRange = 2 * (random().nextDouble() + 0.5); - latRange = 2 * (random().nextDouble() + 0.5); - - originLon = normalizeLon(GeoUtils.MIN_LON_INCL + lonRange + (GeoUtils.MAX_LON_INCL - GeoUtils.MIN_LON_INCL - 2 * lonRange) * random().nextDouble()); - originLat = normalizeLat(GeoUtils.MIN_LAT_INCL + latRange + (GeoUtils.MAX_LAT_INCL - GeoUtils.MIN_LAT_INCL - 2 * latRange) * random().nextDouble()); - } - - /** Puts longitude in range of -180 to +180. */ - public static double normalizeLon(double lon_deg) { - if (lon_deg >= -180 && lon_deg <= 180) { - return lon_deg; //common case, and avoids slight double precision shifting - } - double off = (lon_deg + 180) % 360; - if (off < 0) { - return 180 + off; - } else if (off == 0 && lon_deg > 0) { - return 180; - } else { - return -180 + off; - } - } - - /** Puts latitude in range of -90 to 90. */ - public static double normalizeLat(double lat_deg) { - if (lat_deg >= -90 && lat_deg <= 90) { - return lat_deg; //common case, and avoids slight double precision shifting - } - double off = Math.abs((lat_deg + 90) % 360); - return (off <= 180 ? off : 360-off) - 90; + originLon = GeoTestUtil.nextLongitude(); + originLat = GeoTestUtil.nextLatitude(); } /** Valid values that should not cause exception */ @@ -538,12 +508,11 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase { IndexReader r = w.getReader(); w.close(); - // We can't wrap with "exotic" readers because the BKD query must see the BKDDVFormat: - IndexSearcher s = newSearcher(r, false); + IndexSearcher s = newSearcher(r); - int iters = atLeast(75); + int iters = atLeast(25); for (int iter=0;iter lats = new ArrayList<>(); - ArrayList lons = new ArrayList<>(); - double angle = 0.0; - while (true) { - angle += random().nextDouble()*40.0; - //System.out.println(" angle " + angle); - if (angle > 360) { - break; - } - double len = radius * (1.0 - radiusDelta + radiusDelta * random().nextDouble()); - //System.out.println(" len=" + len); - double lat = centerLat + len * Math.cos(Math.toRadians(angle)); - double lon = centerLon + len * Math.sin(Math.toRadians(angle)); - if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { - // cannot cross dateline: try again! - continue newPoly; - } - if (lat > 90) { - // cross the north pole - lat = 180 - lat; - lon = 180 - lon; - } else if (lat < -90) { - // cross the south pole - lat = -180 - lat; - lon = 180 - lon; - } - if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { - // cannot cross dateline: try again! - continue newPoly; - } - lats.add(lat); - lons.add(lon); - - //System.out.println(" lat=" + lats.get(lats.size()-1) + " lon=" + lons.get(lons.size()-1)); - } - - // close it - lats.add(lats.get(0)); - lons.add(lons.get(0)); - - double[] latsArray = new double[lats.size()]; - double[] lonsArray = new double[lons.size()]; - for(int i=0;i 100000) { + dir = newFSDirectory(createTempDir(getClass().getSimpleName())); + } else { + dir = newDirectory(); + } + + Set deleted = new HashSet<>(); + // RandomIndexWriter is too slow here: + IndexWriter w = new IndexWriter(dir, iwc); + for(int id=0;id 0 && random().nextInt(100) == 42) { + int idToDelete = random().nextInt(id); + w.deleteDocuments(new Term("id", ""+idToDelete)); + deleted.add(idToDelete); if (VERBOSE) { - final DecimalFormat df = new DecimalFormat("#,###.00", DecimalFormatSymbols.getInstance(Locale.ENGLISH)); - System.out.println(" radiusMeters = " + df.format(radiusMeters)); + System.out.println(" delete id=" + idToDelete); } + } + } - query = newDistanceQuery(FIELD_NAME, centerLat, centerLon, radiusMeters); + if (random().nextBoolean()) { + w.forceMerge(1); + } + final IndexReader r = DirectoryReader.open(w); + w.close(); - verifyHits = new VerifyHits() { - @Override - protected boolean shouldMatch(double pointLat, double pointLon) { - return circleContainsPoint(centerLat, centerLon, radiusMeters, pointLat, pointLon); - } + IndexSearcher s = newSearcher(r); - @Override - protected void describe(int docID, double pointLat, double pointLon) { - double distanceMeters = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); - System.out.println(" docID=" + docID + " centerLat=" + centerLat + " centerLon=" + centerLon - + " pointLat=" + pointLat + " pointLon=" + pointLon + " distanceMeters=" + distanceMeters - + " vs radiusMeters=" + radiusMeters); - } - }; + int iters = atLeast(25); - // TODO: get poly query working with dateline crossing too (how?)! - } else { + NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id"); - // TODO: poly query can't handle dateline crossing yet: - final GeoRect bbox = randomRect(small, false); + Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader()); + int maxDoc = s.getIndexReader().maxDoc(); - // Polygon - final double[] polyLats; - final double[] polyLons; - // TODO: factor this out, maybe if we add Polygon class? - switch (random().nextInt(3)) { - case 0: - // box - polyLats = new double[5]; - polyLons = new double[5]; - polyLats[0] = bbox.minLat; - polyLons[0] = bbox.minLon; - polyLats[1] = bbox.maxLat; - polyLons[1] = bbox.minLon; - polyLats[2] = bbox.maxLat; - polyLons[2] = bbox.maxLon; - polyLats[3] = bbox.minLat; - polyLons[3] = bbox.maxLon; - polyLats[4] = bbox.minLat; - polyLons[4] = bbox.minLon; - break; - case 1: - // right triangle - polyLats = new double[4]; - polyLons = new double[4]; - polyLats[0] = bbox.minLat; - polyLons[0] = bbox.minLon; - polyLats[1] = bbox.maxLat; - polyLons[1] = bbox.minLon; - polyLats[2] = bbox.maxLat; - polyLons[2] = bbox.maxLon; - polyLats[3] = bbox.minLat; - polyLons[3] = bbox.minLon; - break; - default: - // surprise me! - double[][] res = surpriseMePolygon(); - polyLats = res[0]; - polyLons = res[1]; - break; - } - query = newPolygonQuery(FIELD_NAME, polyLats, polyLons); + for (int iter=0;iter 100000) { + dir = newFSDirectory(createTempDir(getClass().getSimpleName())); + } else { + dir = newDirectory(); + } + + Set deleted = new HashSet<>(); + // RandomIndexWriter is too slow here: + IndexWriter w = new IndexWriter(dir, iwc); + for(int id=0;id 0 && random().nextInt(100) == 42) { + int idToDelete = random().nextInt(id); + w.deleteDocuments(new Term("id", ""+idToDelete)); + deleted.add(idToDelete); + if (VERBOSE) { + System.out.println(" delete id=" + idToDelete); + } + } + } + + if (random().nextBoolean()) { + w.forceMerge(1); + } + final IndexReader r = DirectoryReader.open(w); + w.close(); + + // We can't wrap with "exotic" readers because points needs to work: + IndexSearcher s = newSearcher(r); + + final int iters = atLeast(75); + + NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id"); + + Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader()); + int maxDoc = s.getIndexReader().maxDoc(); + + for (int iter=0;iterNOTE:minLatitude/maxLatitude are merely guidelines. the returned value is sometimes + * outside of that range! this is to facilitate edge testing. + */ + public static double nextLatitudeAround(double minLatitude, double maxLatitude) { + GeoUtils.checkLatitude(minLatitude); + GeoUtils.checkLatitude(maxLatitude); + return normalizeLatitude(randomRangeMaybeSlightlyOutside(minLatitude, maxLatitude)); + } + + /** + * returns next pseudorandom longitude, kinda close to {@code minLongitude/maxLongitude} + * NOTE:minLongitude/maxLongitude are merely guidelines. the returned value is sometimes + * outside of that range! this is to facilitate edge testing. + */ + public static double nextLongitudeAround(double minLongitude, double maxLongitude) { + GeoUtils.checkLongitude(minLongitude); + GeoUtils.checkLongitude(maxLongitude); + return normalizeLongitude(randomRangeMaybeSlightlyOutside(minLongitude, maxLongitude)); + } + + /** returns next pseudorandom box: can cross the 180th meridian */ + public static GeoRect nextBox() { + return nextBoxInternal(nextLatitude(), nextLatitude(), nextLongitude(), nextLongitude(), true); + } + + /** returns next pseudorandom box, can cross the 180th meridian, kinda close to {@code otherLatitude} and {@code otherLongitude} */ + public static GeoRect nextBoxNear(double otherLatitude, double otherLongitude) { + GeoUtils.checkLongitude(otherLongitude); + GeoUtils.checkLongitude(otherLongitude); + return nextBoxInternal(nextLatitudeNear(otherLatitude), nextLatitudeNear(otherLatitude), + nextLongitudeNear(otherLongitude), nextLongitudeNear(otherLongitude), true); + } + + /** returns next pseudorandom polygon */ + public static double[][] nextPolygon() { + if (random().nextBoolean()) { + return surpriseMePolygon(null, null); + } + + GeoRect box = nextBoxInternal(nextLatitude(), nextLatitude(), nextLongitude(), nextLongitude(), false); + if (random().nextBoolean()) { + // box + return boxPolygon(box); + } else { + // triangle + return trianglePolygon(box); + } + } + + /** returns next pseudorandom polygon, kinda close to {@code otherLatitude} and {@code otherLongitude} */ + public static double[][] nextPolygonNear(double otherLatitude, double otherLongitude) { + if (random().nextBoolean()) { + return surpriseMePolygon(otherLatitude, otherLongitude); + } + + GeoRect box = nextBoxInternal(nextLatitudeNear(otherLatitude), nextLatitudeNear(otherLatitude), + nextLongitudeNear(otherLongitude), nextLongitudeNear(otherLongitude), false); + if (random().nextBoolean()) { + // box + return boxPolygon(box); + } else { + // triangle + return trianglePolygon(box); + } + } + + private static GeoRect nextBoxInternal(double lat0, double lat1, double lon0, double lon1, boolean canCrossDateLine) { + if (lat1 < lat0) { + double x = lat0; + lat0 = lat1; + lat1 = x; + } + + if (canCrossDateLine == false && lon1 < lon0) { + double x = lon0; + lon0 = lon1; + lon1 = x; + } + + return new GeoRect(lat0, lat1, lon0, lon1); + } + + private static double[][] boxPolygon(GeoRect box) { + assert box.crossesDateline() == false; + final double[] polyLats = new double[5]; + final double[] polyLons = new double[5]; + polyLats[0] = box.minLat; + polyLons[0] = box.minLon; + polyLats[1] = box.maxLat; + polyLons[1] = box.minLon; + polyLats[2] = box.maxLat; + polyLons[2] = box.maxLon; + polyLats[3] = box.minLat; + polyLons[3] = box.maxLon; + polyLats[4] = box.minLat; + polyLons[4] = box.minLon; + return new double[][] { polyLats, polyLons }; + } + + private static double[][] trianglePolygon(GeoRect box) { + assert box.crossesDateline() == false; + final double[] polyLats = new double[4]; + final double[] polyLons = new double[4]; + polyLats[0] = box.minLat; + polyLons[0] = box.minLon; + polyLats[1] = box.maxLat; + polyLons[1] = box.minLon; + polyLats[2] = box.maxLat; + polyLons[2] = box.maxLon; + polyLats[3] = box.minLat; + polyLons[3] = box.minLon; + return new double[][] { polyLats, polyLons }; + } + + /** Returns {polyLats, polyLons} double[] array */ + private static double[][] surpriseMePolygon(Double otherLatitude, Double otherLongitude) { + // repeat until we get a poly that doesn't cross dateline: + newPoly: + while (true) { + //System.out.println("\nPOLY ITER"); + final double centerLat; + final double centerLon; + if (otherLatitude == null) { + centerLat = nextLatitude(); + centerLon = nextLongitude(); + } else { + GeoUtils.checkLatitude(otherLatitude); + GeoUtils.checkLongitude(otherLongitude); + centerLat = nextLatitudeNear(otherLatitude); + centerLon = nextLongitudeNear(otherLongitude); + } + + double radius = 0.1 + 20 * random().nextDouble(); + double radiusDelta = random().nextDouble(); + + ArrayList lats = new ArrayList<>(); + ArrayList lons = new ArrayList<>(); + double angle = 0.0; + while (true) { + angle += random().nextDouble()*40.0; + //System.out.println(" angle " + angle); + if (angle > 360) { + break; + } + double len = radius * (1.0 - radiusDelta + radiusDelta * random().nextDouble()); + //System.out.println(" len=" + len); + double lat = centerLat + len * Math.cos(Math.toRadians(angle)); + double lon = centerLon + len * Math.sin(Math.toRadians(angle)); + if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { + // cannot cross dateline: try again! + continue newPoly; + } + if (lat > 90) { + // cross the north pole + lat = 180 - lat; + lon = 180 - lon; + } else if (lat < -90) { + // cross the south pole + lat = -180 - lat; + lon = 180 - lon; + } + if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { + // cannot cross dateline: try again! + continue newPoly; + } + lats.add(lat); + lons.add(lon); + + //System.out.println(" lat=" + lats.get(lats.size()-1) + " lon=" + lons.get(lons.size()-1)); + } + + // close it + lats.add(lats.get(0)); + lons.add(lons.get(0)); + + double[] latsArray = new double[lats.size()]; + double[] lonsArray = new double[lons.size()]; + for(int i=0;i= -90 && latitude <= 90) { + return latitude; //common case, and avoids slight double precision shifting + } + double off = Math.abs((latitude + 90) % 360); + return (off <= 180 ? off : 360-off) - 90; + } + + /** Puts longitude in range of -180 to +180. */ + private static double normalizeLongitude(double longitude) { + if (longitude >= -180 && longitude <= 180) { + return longitude; //common case, and avoids slight double precision shifting + } + double off = (longitude + 180) % 360; + if (off < 0) { + return 180 + off; + } else if (off == 0 && longitude > 0) { + return 180; + } else { + return -180 + off; + } + } + + /** Keep it simple, we don't need to take arbitrary Random for geo tests */ + private static Random random() { + return RandomizedContext.current().getRandom(); + } +} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java index 5378debf6a1..f6fb139a7bd 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java @@ -28,54 +28,22 @@ import org.junit.BeforeClass; */ public class TestGeoUtils extends LuceneTestCase { - private static final double LON_SCALE = (0x1L<= box.minLat && lat <= box.maxLat); @@ -264,8 +227,8 @@ public class TestGeoUtils extends LuceneTestCase { // test we can use haversinSortKey() for distance queries. public void testHaversinOpto() { for (int i = 0; i < 1000; i++) { - double lat = -90 + 180.0 * random().nextDouble(); - double lon = -180 + 360.0 * random().nextDouble(); + double lat = GeoTestUtil.nextLatitude(); + double lon = GeoTestUtil.nextLongitude(); double radius = 50000000 * random().nextDouble(); GeoRect box = GeoUtils.circleToBBox(lat, lon, radius); @@ -274,8 +237,8 @@ public class TestGeoUtils extends LuceneTestCase { SloppyMath.haversinSortKey(lat, lon, box.maxLat, lon)); for (int j = 0; j < 10000; j++) { - double lat2 = -90 + 180.0 * random().nextDouble(); - double lon2 = -180 + 360.0 * random().nextDouble(); + double lat2 = GeoTestUtil.nextLatitude(); + double lon2 = GeoTestUtil.nextLongitude(); // if the point is within radius, then it should be <= our sort key if (SloppyMath.haversinMeters(lat, lon, lat2, lon2) <= radius) { assertTrue(SloppyMath.haversinSortKey(lat, lon, lat2, lon2) <= minPartialDistance); @@ -288,8 +251,8 @@ public class TestGeoUtils extends LuceneTestCase { /** Test infinite radius covers whole earth */ public void testInfiniteRect() { for (int i = 0; i < 1000; i++) { - double centerLat = -90 + 180.0 * random().nextDouble(); - double centerLon = -180 + 360.0 * random().nextDouble(); + double centerLat = GeoTestUtil.nextLatitude(); + double centerLon = GeoTestUtil.nextLongitude(); GeoRect rect = GeoUtils.circleToBBox(centerLat, centerLon, Double.POSITIVE_INFINITY); assertEquals(-180.0, rect.minLon, 0.0D); assertEquals(180.0, rect.maxLon, 0.0D);