From 296d96e4c609d145f621a76d9f2a93b9e11375eb Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Thu, 14 Apr 2016 15:37:41 +0100 Subject: [PATCH 1/4] LUCENE-7210: Make TestCore*Parser's analyzer choice override-able. (Christine Poerschke, Daniel Collins) --- lucene/CHANGES.txt | 3 + .../xml/CoreParserTestIndexData.java | 74 +++++++++++++++ .../queryparser/xml/TestCoreParser.java | 89 ++++++++----------- .../xml/TestCorePlusExtensionsParser.java | 17 +--- .../xml/TestCorePlusQueriesParser.java | 17 +--- 5 files changed, 124 insertions(+), 76 deletions(-) create mode 100644 lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 495e8cccbd1..dbffdc4e236 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -84,6 +84,9 @@ Other * LUCENE-7205: Remove repeated nl.getLength() calls in (Boolean|DisjunctionMax|FuzzyLikeThis)QueryBuilder. (Christine Poerschke) +* LUCENE-7210: Make TestCore*Parser's analyzer choice override-able + (Christine Poerschke, Daniel Collins) + ======================= Lucene 6.0.0 ======================= System Requirements diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java new file mode 100644 index 00000000000..71b627e74cd --- /dev/null +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queryparser.xml; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LegacyIntField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +class CoreParserTestIndexData implements Closeable { + + final Directory dir; + final IndexReader reader; + final IndexSearcher searcher; + + CoreParserTestIndexData(Analyzer analyzer) throws Exception { + BufferedReader d = new BufferedReader(new InputStreamReader( + TestCoreParser.class.getResourceAsStream("reuters21578.txt"), StandardCharsets.US_ASCII)); + dir = LuceneTestCase.newDirectory(); + IndexWriter writer = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(analyzer)); + String line = d.readLine(); + while (line != null) { + int endOfDate = line.indexOf('\t'); + String date = line.substring(0, endOfDate).trim(); + String content = line.substring(endOfDate).trim(); + Document doc = new Document(); + doc.add(LuceneTestCase.newTextField("date", date, Field.Store.YES)); + doc.add(LuceneTestCase.newTextField("contents", content, Field.Store.YES)); + doc.add(new LegacyIntField("date2", Integer.valueOf(date), Field.Store.NO)); + doc.add(new IntPoint("date3", Integer.valueOf(date))); + writer.addDocument(doc); + line = d.readLine(); + } + d.close(); + writer.close(); + reader = DirectoryReader.open(dir); + searcher = LuceneTestCase.newSearcher(reader, false); + } + + @Override + public void close() throws IOException { + reader.close(); + dir.close(); + } + +} + diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java index 04faa7d5f1c..9b18c81848e 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java @@ -21,78 +21,41 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenFilter; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.junit.AfterClass; -import org.junit.BeforeClass; -import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; - public class TestCoreParser extends LuceneTestCase { final private static String defaultField = "contents"; + private static Analyzer analyzer; private static CoreParser coreParser; - private static Directory dir; - private static IndexReader reader; - private static IndexSearcher searcher; - @BeforeClass - public static void beforeClass() throws Exception { + private static CoreParserTestIndexData indexData; + + protected Analyzer newAnalyzer() { // TODO: rewrite test (this needs to set QueryParser.enablePositionIncrements, too, for work with CURRENT): - analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET); - //initialize the parser - coreParser = new CoreParser(defaultField, analyzer); - - BufferedReader d = new BufferedReader(new InputStreamReader( - TestCoreParser.class.getResourceAsStream("reuters21578.txt"), StandardCharsets.US_ASCII)); - dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(analyzer)); - String line = d.readLine(); - while (line != null) { - int endOfDate = line.indexOf('\t'); - String date = line.substring(0, endOfDate).trim(); - String content = line.substring(endOfDate).trim(); - Document doc = new Document(); - doc.add(newTextField("date", date, Field.Store.YES)); - doc.add(newTextField("contents", content, Field.Store.YES)); - doc.add(new LegacyIntField("date2", Integer.valueOf(date), Field.Store.NO)); - doc.add(new IntPoint("date3", Integer.valueOf(date))); - writer.addDocument(doc); - line = d.readLine(); - } - d.close(); - writer.close(); - reader = DirectoryReader.open(dir); - searcher = newSearcher(reader, false); + return new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET); + } + protected CoreParser newCoreParser(String defaultField, Analyzer analyzer) { + return new CoreParser(defaultField, analyzer); } @AfterClass public static void afterClass() throws Exception { - reader.close(); - dir.close(); - reader = null; - searcher = null; - dir = null; - coreParser = null; - analyzer = null; + if (indexData != null) { + indexData.close(); + } } public void testTermQueryXML() throws ParserException, IOException { @@ -133,7 +96,7 @@ public class TestCoreParser extends LuceneTestCase { public void testCustomFieldUserQueryXML() throws ParserException, IOException { Query q = parse("UserInputQueryCustomField.xml"); - int h = searcher.search(q, 1000).totalHits; + int h = searcher().search(q, 1000).totalHits; assertEquals("UserInputQueryCustomField should produce 0 result ", 0, h); } @@ -179,13 +142,38 @@ public class TestCoreParser extends LuceneTestCase { } protected Analyzer analyzer() { + if (analyzer == null) { + analyzer = newAnalyzer(); + } return analyzer; } protected CoreParser coreParser() { + if (coreParser == null) { + coreParser = newCoreParser(defaultField, analyzer()); + } return coreParser; } + private CoreParserTestIndexData indexData() { + if (indexData == null) { + try { + indexData = new CoreParserTestIndexData(analyzer()); + } catch (Exception e) { + fail("caught Exception "+e); + } + } + return indexData; + } + + protected IndexReader reader() { + return indexData().reader; + } + + protected IndexSearcher searcher() { + return indexData().searcher; + } + protected Query parse(String xmlFileName) throws ParserException, IOException { try (InputStream xmlStream = TestCoreParser.class.getResourceAsStream(xmlFileName)) { assertNotNull("Test XML file " + xmlFileName + " cannot be found", xmlStream); @@ -195,13 +183,14 @@ public class TestCoreParser extends LuceneTestCase { } protected Query rewrite(Query q) throws IOException { - return q.rewrite(reader); + return q.rewrite(reader()); } protected void dumpResults(String qType, Query q, int numDocs) throws IOException { if (VERBOSE) { System.out.println("TEST: qType=" + qType + " query=" + q + " numDocs=" + numDocs); } + final IndexSearcher searcher = searcher(); TopDocs hits = searcher.search(q, numDocs); assertTrue(qType + " should produce results ", hits.totalHits > 0); if (VERBOSE) { diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java index 23cfb5057a0..35f28ef1103 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java @@ -16,11 +16,14 @@ */ package org.apache.lucene.queryparser.xml; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.search.Query; public class TestCorePlusExtensionsParser extends TestCorePlusQueriesParser { - private CoreParser corePlusExtensionsParser; + protected CoreParser newCoreParser(String defaultField, Analyzer analyzer) { + return new CorePlusExtensionsParser(defaultField, analyzer); + } public void testFuzzyLikeThisQueryXML() throws Exception { Query q = parse("FuzzyLikeThisQuery.xml"); @@ -31,16 +34,4 @@ public class TestCorePlusExtensionsParser extends TestCorePlusQueriesParser { dumpResults("FuzzyLikeThis", q, 5); } - //================= Helper methods =================================== - - @Override - protected CoreParser coreParser() { - if (corePlusExtensionsParser == null) { - corePlusExtensionsParser = new CorePlusExtensionsParser( - super.defaultField(), - super.analyzer()); - } - return corePlusExtensionsParser; - } - } diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java index d87d40b7701..7e58c47c24b 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java @@ -16,11 +16,14 @@ */ package org.apache.lucene.queryparser.xml; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.search.Query; public class TestCorePlusQueriesParser extends TestCoreParser { - private CoreParser corePlusQueriesParser; + protected CoreParser newCoreParser(String defaultField, Analyzer analyzer) { + return new CorePlusQueriesParser(defaultField, analyzer); + } public void testLikeThisQueryXML() throws Exception { Query q = parse("LikeThisQuery.xml"); @@ -32,16 +35,4 @@ public class TestCorePlusQueriesParser extends TestCoreParser { dumpResults("boosting ", q, 5); } - //================= Helper methods =================================== - - @Override - protected CoreParser coreParser() { - if (corePlusQueriesParser == null) { - corePlusQueriesParser = new CorePlusQueriesParser( - super.defaultField(), - super.analyzer()); - } - return corePlusQueriesParser; - } - } From 22df9fc3b72b8d1918aa8580d32fc657757df1ef Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Thu, 14 Apr 2016 19:38:43 +0100 Subject: [PATCH 2/4] LUCENE-7210: Add missing @Override to TestCorePlus(Queries|Extensions)Parser's newCoreParser method. --- .../lucene/queryparser/xml/TestCorePlusExtensionsParser.java | 1 + .../apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java | 1 + 2 files changed, 2 insertions(+) diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java index 35f28ef1103..8b096170625 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java @@ -21,6 +21,7 @@ import org.apache.lucene.search.Query; public class TestCorePlusExtensionsParser extends TestCorePlusQueriesParser { + @Override protected CoreParser newCoreParser(String defaultField, Analyzer analyzer) { return new CorePlusExtensionsParser(defaultField, analyzer); } diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java index 7e58c47c24b..a91800fd763 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java @@ -21,6 +21,7 @@ import org.apache.lucene.search.Query; public class TestCorePlusQueriesParser extends TestCoreParser { + @Override protected CoreParser newCoreParser(String defaultField, Analyzer analyzer) { return new CorePlusQueriesParser(defaultField, analyzer); } From 6ec124446790d81f79bf76f0f589a9bf81de08e0 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Fri, 15 Apr 2016 07:11:15 -0400 Subject: [PATCH 3/4] LUCENE-7221: Do not attempt to carve out holes from bounds. --- .../spatial3d/geom/GeoConcavePolygon.java | 50 +++++++++++++++---- .../spatial3d/geom/GeoConvexPolygon.java | 40 +++++++++++++-- 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java index b28ebd27f08..124b46bf2ae 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java @@ -258,15 +258,7 @@ class GeoConcavePolygon extends GeoBasePolygon { @Override public boolean isWithin(final double x, final double y, final double z) { - // If present within *any* plane, then it is a member, except where there are holes. - boolean isMember = false; - for (final SidedPlane edge : edges) { - if (edge.isWithin(x, y, z)) { - isMember = true; - break; - } - } - if (isMember == false) { + if (!localIsWithin(x, y, z)) { return false; } if (holes != null) { @@ -279,6 +271,22 @@ class GeoConcavePolygon extends GeoBasePolygon { return true; } + protected boolean localIsWithin(final Vector v) { + return localIsWithin(v.x, v.y, v.z); + } + + protected boolean localIsWithin(final double x, final double y, final double z) { + // If present within *any* plane, then it is a member, except where there are holes. + boolean isMember = false; + for (final SidedPlane edge : edges) { + if (edge.isWithin(x, y, z)) { + isMember = true; + break; + } + } + return isMember; + } + @Override public GeoPoint[] getEdgePoints() { return edgePoints; @@ -341,7 +349,28 @@ class GeoConcavePolygon extends GeoBasePolygon { @Override public void getBounds(Bounds bounds) { - super.getBounds(bounds); + // Because of holes, we don't want to use superclass method + if (localIsWithin(planetModel.NORTH_POLE)) { + bounds.noTopLatitudeBound().noLongitudeBound() + .addPoint(planetModel.NORTH_POLE); + } + if (localIsWithin(planetModel.SOUTH_POLE)) { + bounds.noBottomLatitudeBound().noLongitudeBound() + .addPoint(planetModel.SOUTH_POLE); + } + if (localIsWithin(planetModel.MIN_X_POLE)) { + bounds.addPoint(planetModel.MIN_X_POLE); + } + if (localIsWithin(planetModel.MAX_X_POLE)) { + bounds.addPoint(planetModel.MAX_X_POLE); + } + if (localIsWithin(planetModel.MIN_Y_POLE)) { + bounds.addPoint(planetModel.MIN_Y_POLE); + } + if (localIsWithin(planetModel.MAX_Y_POLE)) { + bounds.addPoint(planetModel.MAX_Y_POLE); + } + bounds.isWide(); // Add all the points @@ -353,6 +382,7 @@ class GeoConcavePolygon extends GeoBasePolygon { for (final SidedPlane edge : edges) { bounds.addPlane(planetModel, edge, eitherBounds.get(edge)); } + } @Override diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java index 890df30d09f..c51ae828fbc 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java @@ -253,9 +253,8 @@ class GeoConvexPolygon extends GeoBasePolygon { @Override public boolean isWithin(final double x, final double y, final double z) { - for (final SidedPlane edge : edges) { - if (!edge.isWithin(x, y, z)) - return false; + if (!localIsWithin(x, y, z)) { + return false; } if (holes != null) { for (final GeoPolygon polygon : holes) { @@ -266,7 +265,19 @@ class GeoConvexPolygon extends GeoBasePolygon { } return true; } + + protected boolean localIsWithin(final Vector v) { + return localIsWithin(v.x, v.y, v.z); + } + protected boolean localIsWithin(final double x, final double y, final double z) { + for (final SidedPlane edge : edges) { + if (!edge.isWithin(x, y, z)) + return false; + } + return true; + } + @Override public GeoPoint[] getEdgePoints() { return edgePoints; @@ -328,7 +339,27 @@ class GeoConvexPolygon extends GeoBasePolygon { @Override public void getBounds(Bounds bounds) { - super.getBounds(bounds); + // Because of holes, we don't want to use superclass method + if (localIsWithin(planetModel.NORTH_POLE)) { + bounds.noTopLatitudeBound().noLongitudeBound() + .addPoint(planetModel.NORTH_POLE); + } + if (localIsWithin(planetModel.SOUTH_POLE)) { + bounds.noBottomLatitudeBound().noLongitudeBound() + .addPoint(planetModel.SOUTH_POLE); + } + if (localIsWithin(planetModel.MIN_X_POLE)) { + bounds.addPoint(planetModel.MIN_X_POLE); + } + if (localIsWithin(planetModel.MAX_X_POLE)) { + bounds.addPoint(planetModel.MAX_X_POLE); + } + if (localIsWithin(planetModel.MIN_Y_POLE)) { + bounds.addPoint(planetModel.MIN_Y_POLE); + } + if (localIsWithin(planetModel.MAX_Y_POLE)) { + bounds.addPoint(planetModel.MAX_Y_POLE); + } // Add all the points for (final GeoPoint point : points) { @@ -339,6 +370,7 @@ class GeoConvexPolygon extends GeoBasePolygon { for (final SidedPlane edge : edges) { bounds.addPlane(planetModel, edge, eitherBounds.get(edge)); } + } @Override From e6bcb131a7f0ddb41c526f97a0327a6fb96a0be0 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Fri, 15 Apr 2016 12:11:56 +0100 Subject: [PATCH 4/4] LUCENE-7210: TestCoreParser.afterClass now resets (static) indexData/coreParser/analyzer to null. --- .../test/org/apache/lucene/queryparser/xml/TestCoreParser.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java index 9b18c81848e..82426d06076 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java @@ -55,7 +55,10 @@ public class TestCoreParser extends LuceneTestCase { public static void afterClass() throws Exception { if (indexData != null) { indexData.close(); + indexData = null; } + coreParser = null; + analyzer = null; } public void testTermQueryXML() throws ParserException, IOException {