diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java index 4896fa8b9c6..20a9e7af2ec 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java @@ -138,7 +138,7 @@ public class SynonymFilterFactory extends TokenFilterFactory implements Resource }; } - try { + try (Analyzer a = analyzer) { String formatClass = format; if (format == null || format.equals("solr")) { formatClass = SolrSynonymParser.class.getName(); @@ -146,7 +146,7 @@ public class SynonymFilterFactory extends TokenFilterFactory implements Resource formatClass = WordnetSynonymParser.class.getName(); } // TODO: expose dedup as a parameter? - map = loadSynonyms(loader, formatClass, true, analyzer); + map = loadSynonyms(loader, formatClass, true, a); } catch (ParseException e) { throw new IOException("Error parsing synonyms file:", e); } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java index 49275c9328c..95a636baad2 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java @@ -31,7 +31,7 @@ public class TestArabicAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new ArabicAnalyzer(); + new ArabicAnalyzer().close(); } /** @@ -53,6 +53,7 @@ public class TestArabicAnalyzer extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "ما ملكت أيمانكم", new String[] { "ملكت", "ايمانكم"}); assertAnalyzesTo(a, "الذين ملكت أيمانكم", new String[] { "ملكت", "ايمانكم" }); // stopwords + a.close(); } /** @@ -62,14 +63,17 @@ public class TestArabicAnalyzer extends BaseTokenStreamTestCase { ArabicAnalyzer a = new ArabicAnalyzer(); assertAnalyzesTo(a, "كبير", new String[] { "كبير" }); assertAnalyzesTo(a, "كبيرة", new String[] { "كبير" }); // feminine marker + a.close(); } /** * Non-arabic text gets treated in a similar way as SimpleAnalyzer. */ public void testEnglishInput() throws Exception { - assertAnalyzesTo(new ArabicAnalyzer(), "English text.", new String[] { + ArabicAnalyzer a = new ArabicAnalyzer(); + assertAnalyzesTo(a, "English text.", new String[] { "english", "text" }); + a.close(); } /** @@ -80,6 +84,7 @@ public class TestArabicAnalyzer extends BaseTokenStreamTestCase { ArabicAnalyzer a = new ArabicAnalyzer(set); assertAnalyzesTo(a, "The quick brown fox.", new String[] { "quick", "brown", "fox" }); + a.close(); } public void testWithStemExclusionSet() throws IOException { @@ -87,15 +92,18 @@ public class TestArabicAnalyzer extends BaseTokenStreamTestCase { ArabicAnalyzer a = new ArabicAnalyzer(CharArraySet.EMPTY_SET, set); assertAnalyzesTo(a, "كبيرة the quick ساهدهات", new String[] { "كبير","the", "quick", "ساهدهات" }); assertAnalyzesTo(a, "كبيرة the quick ساهدهات", new String[] { "كبير","the", "quick", "ساهدهات" }); - + a.close(); a = new ArabicAnalyzer(CharArraySet.EMPTY_SET, CharArraySet.EMPTY_SET); assertAnalyzesTo(a, "كبيرة the quick ساهدهات", new String[] { "كبير","the", "quick", "ساهد" }); assertAnalyzesTo(a, "كبيرة the quick ساهدهات", new String[] { "كبير","the", "quick", "ساهد" }); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new ArabicAnalyzer(), 1000*RANDOM_MULTIPLIER); + ArabicAnalyzer a = new ArabicAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java index 49a749471de..b3ddfb0069b 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java @@ -104,6 +104,7 @@ public class TestArabicNormalizationFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java index 8768e290350..09169bfc0e1 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java @@ -141,5 +141,6 @@ public class TestArabicStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianAnalyzer.java index dd50be254a9..a5b82a87c04 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianAnalyzer.java @@ -32,24 +32,27 @@ public class TestBulgarianAnalyzer extends BaseTokenStreamTestCase { * This test fails with NPE when the stopwords file is missing in classpath */ public void testResourcesAvailable() { - new BulgarianAnalyzer(); + new BulgarianAnalyzer().close(); } public void testStopwords() throws IOException { Analyzer a = new BulgarianAnalyzer(); assertAnalyzesTo(a, "Как се казваш?", new String[] {"казваш"}); + a.close(); } public void testCustomStopwords() throws IOException { Analyzer a = new BulgarianAnalyzer(CharArraySet.EMPTY_SET); assertAnalyzesTo(a, "Как се казваш?", new String[] {"как", "се", "казваш"}); + a.close(); } public void testReusableTokenStream() throws IOException { Analyzer a = new BulgarianAnalyzer(); assertAnalyzesTo(a, "документи", new String[] {"документ"}); assertAnalyzesTo(a, "документ", new String[] {"документ"}); + a.close(); } /** @@ -64,6 +67,7 @@ public class TestBulgarianAnalyzer extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "компютър", new String[] {"компютр"}); assertAnalyzesTo(a, "градове", new String[] {"град"}); + a.close(); } public void testWithStemExclusionSet() throws IOException { @@ -71,10 +75,13 @@ public class TestBulgarianAnalyzer extends BaseTokenStreamTestCase { set.add("строеве"); Analyzer a = new BulgarianAnalyzer(CharArraySet.EMPTY_SET, set); assertAnalyzesTo(a, "строевете строеве", new String[] { "строй", "строеве" }); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new BulgarianAnalyzer(), 1000*RANDOM_MULTIPLIER); + BulgarianAnalyzer a = new BulgarianAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianStemmer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianStemmer.java index 5054ff5e074..476c156900b 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianStemmer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianStemmer.java @@ -97,6 +97,8 @@ public class TestBulgarianStemmer extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "братя", new String[] {"брат"}); assertAnalyzesTo(a, "братята", new String[] {"брат"}); assertAnalyzesTo(a, "брате", new String[] {"брат"}); + + a.close(); } /** @@ -109,6 +111,8 @@ public class TestBulgarianStemmer extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "вестта", new String[] {"вест"}); assertAnalyzesTo(a, "вести", new String[] {"вест"}); assertAnalyzesTo(a, "вестите", new String[] {"вест"}); + + a.close(); } /** @@ -138,6 +142,8 @@ public class TestBulgarianStemmer extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "изключенията", new String[] {"изключени"}); /* note the below form in this example does not conflate with the rest */ assertAnalyzesTo(a, "изключения", new String[] {"изключн"}); + + a.close(); } /** @@ -154,6 +160,7 @@ public class TestBulgarianStemmer extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "красивото", new String[] {"красив"}); assertAnalyzesTo(a, "красиви", new String[] {"красив"}); assertAnalyzesTo(a, "красивите", new String[] {"красив"}); + a.close(); } /** @@ -212,6 +219,8 @@ public class TestBulgarianStemmer extends BaseTokenStreamTestCase { /* note the below forms conflate with each other, but not the rest */ assertAnalyzesTo(a, "строя", new String[] {"стр"}); assertAnalyzesTo(a, "строят", new String[] {"стр"}); + + a.close(); } public void testWithKeywordAttribute() throws IOException { @@ -234,5 +243,6 @@ public class TestBulgarianStemmer extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianAnalyzer.java index a96cbfb8393..23500d30d23 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianAnalyzer.java @@ -135,12 +135,14 @@ public class TestBrazilianAnalyzer extends BaseTokenStreamTestCase { checkReuse(a, "boainain", "boainain"); checkReuse(a, "boas", "boas"); checkReuse(a, "bôas", "boas"); // removes diacritic: different from snowball portugese + a.close(); } public void testStemExclusionTable() throws Exception { BrazilianAnalyzer a = new BrazilianAnalyzer( CharArraySet.EMPTY_SET, new CharArraySet(asSet("quintessência"), false)); checkReuse(a, "quintessência", "quintessência"); // excluded words will be completely unchanged. + a.close(); } public void testWithKeywordAttribute() throws IOException { @@ -154,7 +156,9 @@ public class TestBrazilianAnalyzer extends BaseTokenStreamTestCase { } private void check(final String input, final String expected) throws Exception { - checkOneTerm(new BrazilianAnalyzer(), input, expected); + BrazilianAnalyzer a = new BrazilianAnalyzer(); + checkOneTerm(a, input, expected); + a.close(); } private void checkReuse(Analyzer a, String input, String expected) throws Exception { @@ -163,7 +167,9 @@ public class TestBrazilianAnalyzer extends BaseTokenStreamTestCase { /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new BrazilianAnalyzer(), 1000*RANDOM_MULTIPLIER); + BrazilianAnalyzer a = new BrazilianAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } public void testEmptyTerm() throws IOException { @@ -175,5 +181,6 @@ public class TestBrazilianAnalyzer extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ca/TestCatalanAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ca/TestCatalanAnalyzer.java index bc14adc7897..661f49aba2c 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ca/TestCatalanAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ca/TestCatalanAnalyzer.java @@ -27,7 +27,7 @@ public class TestCatalanAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new CatalanAnalyzer(); + new CatalanAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestCatalanAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "llengua", "llengu"); // stopword assertAnalyzesTo(a, "un", new String[] { }); + a.close(); } /** test use of elisionfilter */ @@ -45,6 +46,7 @@ public class TestCatalanAnalyzer extends BaseTokenStreamTestCase { Analyzer a = new CatalanAnalyzer(); assertAnalyzesTo(a, "Diccionari de l'Institut d'Estudis Catalans", new String[] { "diccion", "inst", "estud", "catalan" }); + a.close(); } /** test use of exclusion set */ @@ -53,10 +55,13 @@ public class TestCatalanAnalyzer extends BaseTokenStreamTestCase { Analyzer a = new CatalanAnalyzer(CatalanAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "llengües", "llengües"); checkOneTerm(a, "llengua", "llengu"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new CatalanAnalyzer(), 1000*RANDOM_MULTIPLIER); + CatalanAnalyzer a = new CatalanAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java index ce1429a8f30..fb07ceea3d5 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java @@ -25,7 +25,6 @@ import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashSet; -import java.util.List; import java.util.Set; import org.apache.lucene.analysis.Analyzer; @@ -403,16 +402,22 @@ public class HTMLStripCharFilterTest extends BaseTokenStreamTestCase { public void testRandom() throws Exception { int numRounds = RANDOM_MULTIPLIER * 1000; - checkRandomData(random(), newTestAnalyzer(), numRounds); + Analyzer a = newTestAnalyzer(); + checkRandomData(random(), a, numRounds); + a.close(); } public void testRandomHugeStrings() throws Exception { int numRounds = RANDOM_MULTIPLIER * 100; - checkRandomData(random(), newTestAnalyzer(), numRounds, 8192); + Analyzer a = newTestAnalyzer(); + checkRandomData(random(), a, numRounds, 8192); + a.close(); } public void testCloseBR() throws Exception { - checkAnalysisConsistency(random(), newTestAnalyzer(), random().nextBoolean(), " Secretary)
[[M"); + Analyzer a = newTestAnalyzer(); + checkAnalysisConsistency(random(), a, random().nextBoolean(), " Secretary)
[[M"); + a.close(); } public void testServerSideIncludes() throws Exception { @@ -549,7 +554,9 @@ public class HTMLStripCharFilterTest extends BaseTokenStreamTestCase { public void testRandomBrokenHTML() throws Exception { int maxNumElements = 10000; String text = TestUtil.randomHtmlishString(random(), maxNumElements); - checkAnalysisConsistency(random(), newTestAnalyzer(), random().nextBoolean(), text); + Analyzer a = newTestAnalyzer(); + checkAnalysisConsistency(random(), a, random().nextBoolean(), text); + a.close(); } public void testRandomText() throws Exception { @@ -617,6 +624,7 @@ public class HTMLStripCharFilterTest extends BaseTokenStreamTestCase { assertAnalyzesTo(analyzer, " �", new String[] { "\uFFFD" } ); assertAnalyzesTo(analyzer, " �", new String[] { "\uFFFD" } ); assertAnalyzesTo(analyzer, " �
", new String[] { "�" } ); + analyzer.close(); } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java index e6f5e95c847..40a845c060a 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java @@ -216,6 +216,7 @@ public class TestMappingCharFilter extends BaseTokenStreamTestCase { int numRounds = RANDOM_MULTIPLIER * 10000; checkRandomData(random(), analyzer, numRounds); + analyzer.close(); } //@Ignore("wrong finalOffset: https://issues.apache.org/jira/browse/LUCENE-3971") @@ -242,6 +243,7 @@ public class TestMappingCharFilter extends BaseTokenStreamTestCase { String text = "gzw f quaxot"; checkAnalysisConsistency(random(), analyzer, false, text); + analyzer.close(); } //@Ignore("wrong finalOffset: https://issues.apache.org/jira/browse/LUCENE-3971") @@ -263,6 +265,7 @@ public class TestMappingCharFilter extends BaseTokenStreamTestCase { }; int numRounds = 100; checkRandomData(random(), analyzer, numRounds); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java index fc25c5496c0..71a1a698fe0 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java @@ -19,7 +19,6 @@ package org.apache.lucene.analysis.cjk; import java.io.IOException; import java.io.Reader; -import java.util.Random; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -39,7 +38,19 @@ import org.apache.lucene.analysis.util.CharArraySet; * Most tests adopted from TestCJKTokenizer */ public class TestCJKAnalyzer extends BaseTokenStreamTestCase { - private Analyzer analyzer = new CJKAnalyzer(); + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new CJKAnalyzer(); + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } public void testJa1() throws IOException { assertAnalyzesTo(analyzer, "一二三四五六七八九十", @@ -228,6 +239,8 @@ public class TestCJKAnalyzer extends BaseTokenStreamTestCase { // before bigramming, the 4 tokens look like: // { 0, 0, 1, 1 }, // { 0, 1, 1, 2 } + + analyzer.close(); } private static class FakeStandardTokenizer extends TokenFilter { @@ -267,17 +280,21 @@ public class TestCJKAnalyzer extends BaseTokenStreamTestCase { new int[] { 1 }, new String[] { "" }, new int[] { 1 }); + analyzer.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new CJKAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer a = new CJKAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomHugeStrings() throws Exception { - Random random = random(); - checkRandomData(random, new CJKAnalyzer(), 100*RANDOM_MULTIPLIER, 8192); + Analyzer a = new CJKAnalyzer(); + checkRandomData(random(), a, 100*RANDOM_MULTIPLIER, 8192); + a.close(); } public void testEmptyTerm() throws IOException { @@ -289,5 +306,6 @@ public class TestCJKAnalyzer extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKBigramFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKBigramFilter.java index 08684009d8e..8f81756c7ee 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKBigramFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKBigramFilter.java @@ -17,31 +17,42 @@ package org.apache.lucene.analysis.cjk; * limitations under the License. */ -import java.io.Reader; import java.util.Random; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.standard.StandardTokenizer; +import org.apache.lucene.util.IOUtils; public class TestCJKBigramFilter extends BaseTokenStreamTestCase { - Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer t = new StandardTokenizer(); - return new TokenStreamComponents(t, new CJKBigramFilter(t)); - } - }; + Analyzer analyzer, unibiAnalyzer; - Analyzer unibiAnalyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer t = new StandardTokenizer(); - return new TokenStreamComponents(t, - new CJKBigramFilter(t, 0xff, true)); - } - }; + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer t = new StandardTokenizer(); + return new TokenStreamComponents(t, new CJKBigramFilter(t)); + } + }; + unibiAnalyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer t = new StandardTokenizer(); + return new TokenStreamComponents(t, + new CJKBigramFilter(t, 0xff, true)); + } + }; + } + + @Override + public void tearDown() throws Exception { + IOUtils.close(analyzer, unibiAnalyzer); + super.tearDown(); + } public void testHuge() throws Exception { assertAnalyzesTo(analyzer, "多くの学生が試験に落ちた" + "多くの学生が試験に落ちた" + "多くの学生が試験に落ちた" @@ -79,6 +90,7 @@ public class TestCJKBigramFilter extends BaseTokenStreamTestCase { "", "", "", "", "" }, new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }); + a.close(); } public void testAllScripts() throws Exception { @@ -92,6 +104,7 @@ public class TestCJKBigramFilter extends BaseTokenStreamTestCase { }; assertAnalyzesTo(a, "多くの学生が試験に落ちた。", new String[] { "多く", "くの", "の学", "学生", "生が", "が試", "試験", "験に", "に落", "落ち", "ちた" }); + a.close(); } public void testUnigramsAndBigramsAllScripts() throws Exception { @@ -132,6 +145,7 @@ public class TestCJKBigramFilter extends BaseTokenStreamTestCase { "", "", "", "", "" }, new int[] { 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1 }, new int[] { 1, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1 }); + a.close(); } public void testUnigramsAndBigramsHuge() throws Exception { diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKWidthFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKWidthFilter.java index 733c4e1ae79..0e6f2907cea 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKWidthFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKWidthFilter.java @@ -29,13 +29,25 @@ import org.apache.lucene.analysis.core.KeywordTokenizer; * Tests for {@link CJKWidthFilter} */ public class TestCJKWidthFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new CJKWidthFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new CJKWidthFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** * Full-width ASCII forms normalized to half-width (basic latin) @@ -74,5 +86,6 @@ public class TestCJKWidthFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniAnalyzer.java index 9a2c9d9969a..e3942c14368 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniAnalyzer.java @@ -32,24 +32,27 @@ public class TestSoraniAnalyzer extends BaseTokenStreamTestCase { * This test fails with NPE when the stopwords file is missing in classpath */ public void testResourcesAvailable() { - new SoraniAnalyzer(); + new SoraniAnalyzer().close(); } public void testStopwords() throws IOException { Analyzer a = new SoraniAnalyzer(); assertAnalyzesTo(a, "ئەم پیاوە", new String[] {"پیاو"}); + a.close(); } public void testCustomStopwords() throws IOException { Analyzer a = new SoraniAnalyzer(CharArraySet.EMPTY_SET); assertAnalyzesTo(a, "ئەم پیاوە", new String[] {"ئەم", "پیاو"}); + a.close(); } public void testReusableTokenStream() throws IOException { Analyzer a = new SoraniAnalyzer(); assertAnalyzesTo(a, "پیاوە", new String[] {"پیاو"}); assertAnalyzesTo(a, "پیاو", new String[] {"پیاو"}); + a.close(); } public void testWithStemExclusionSet() throws IOException { @@ -57,10 +60,13 @@ public class TestSoraniAnalyzer extends BaseTokenStreamTestCase { set.add("پیاوە"); Analyzer a = new SoraniAnalyzer(CharArraySet.EMPTY_SET, set); assertAnalyzesTo(a, "پیاوە", new String[] { "پیاوە" }); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new SoraniAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer a = new SoraniAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilter.java index a91b0d5eba9..4e8f3529694 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.ckb; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -30,13 +29,25 @@ import org.apache.lucene.analysis.core.KeywordTokenizer; * Tests normalization for Sorani (this is more critical than stemming...) */ public class TestSoraniNormalizationFilter extends BaseTokenStreamTestCase { - Analyzer a = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false); - return new TokenStreamComponents(tokenizer, new SoraniNormalizationFilter(tokenizer)); - } - }; + Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false); + return new TokenStreamComponents(tokenizer, new SoraniNormalizationFilter(tokenizer)); + } + }; + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } public void testY() throws Exception { checkOneTerm(a, "\u064A", "\u06CC"); @@ -96,5 +107,6 @@ public class TestSoraniNormalizationFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniStemFilter.java index bf98fa659b7..9b867e57e22 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniStemFilter.java @@ -20,7 +20,6 @@ package org.apache.lucene.analysis.ckb; import static org.apache.lucene.analysis.VocabularyAssert.assertVocabulary; import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -31,7 +30,19 @@ import org.apache.lucene.analysis.core.KeywordTokenizer; * Test the Sorani Stemmer. */ public class TestSoraniStemFilter extends BaseTokenStreamTestCase { - SoraniAnalyzer a = new SoraniAnalyzer(); + Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new SoraniAnalyzer(); + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } public void testIndefiniteSingular() throws Exception { checkOneTerm(a, "پیاوێک", "پیاو"); // -ek @@ -90,6 +101,7 @@ public class TestSoraniStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } /** test against a basic vocabulary file */ diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java index e8afb5eaaf2..409adf743cb 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java @@ -156,6 +156,7 @@ public class CommonGramsFilterTest extends BaseTokenStreamTestCase { new String[] { "s_s", "s_s" }); assertAnalyzesTo(a, "of the of", new String[] { "of_the", "the_of" }); + a.close(); } public void testCommonGramsFilter() throws Exception { @@ -242,6 +243,7 @@ public class CommonGramsFilterTest extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "of the of", new String[] { "of", "of_the", "the", "the_of", "of" }, new int[] { 1, 0, 1, 0, 1 }); + a.close(); } /** @@ -330,6 +332,7 @@ public class CommonGramsFilterTest extends BaseTokenStreamTestCase { }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); Analyzer b = new Analyzer() { @@ -342,5 +345,6 @@ public class CommonGramsFilterTest extends BaseTokenStreamTestCase { }; checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER); + b.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java index d547e634351..d9d2f01f678 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java @@ -336,6 +336,7 @@ public class TestCompoundWordTokenFilter extends BaseTokenStreamTestCase { new String[] { "bankueberfall", "fall" }, new int[] { 0, 0 }, new int[] { 12, 12 }); + analyzer.close(); } /** blast some random strings through the analyzer */ @@ -350,6 +351,7 @@ public class TestCompoundWordTokenFilter extends BaseTokenStreamTestCase { } }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); InputSource is = new InputSource(getClass().getResource("da_UTF8.xml").toExternalForm()); final HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter.getHyphenationTree(is); @@ -363,6 +365,7 @@ public class TestCompoundWordTokenFilter extends BaseTokenStreamTestCase { } }; checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER); + b.close(); } public void testEmptyTerm() throws Exception { @@ -376,6 +379,7 @@ public class TestCompoundWordTokenFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); InputSource is = new InputSource(getClass().getResource("da_UTF8.xml").toExternalForm()); final HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter.getHyphenationTree(is); @@ -389,5 +393,6 @@ public class TestCompoundWordTokenFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(b, "", ""); + b.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java index 24b9629e527..8381e27d0c2 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java @@ -18,15 +18,18 @@ package org.apache.lucene.analysis.core; */ import java.io.IOException; -import java.io.Reader; import java.io.StringReader; -import java.util.Random; -import org.apache.lucene.analysis.*; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.standard.StandardTokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; public class TestAnalyzers extends BaseTokenStreamTestCase { @@ -48,6 +51,7 @@ public class TestAnalyzers extends BaseTokenStreamTestCase { new String[] { "b" }); assertAnalyzesTo(a, "\"QUOTED\" word", new String[] { "quoted", "word" }); + a.close(); } public void testNull() throws Exception { @@ -68,6 +72,7 @@ public class TestAnalyzers extends BaseTokenStreamTestCase { new String[] { "2B" }); assertAnalyzesTo(a, "\"QUOTED\" word", new String[] { "\"QUOTED\"", "word" }); + a.close(); } public void testStop() throws Exception { @@ -76,6 +81,7 @@ public class TestAnalyzers extends BaseTokenStreamTestCase { new String[] { "foo", "bar", "foo", "bar" }); assertAnalyzesTo(a, "foo a bar such FOO THESE BAR", new String[] { "foo", "bar", "foo", "bar" }); + a.close(); } void verifyPayload(TokenStream ts) throws IOException { @@ -159,6 +165,7 @@ public class TestAnalyzers extends BaseTokenStreamTestCase { // unpaired trail surrogate assertAnalyzesTo(a, "AbaC\uDC16AdaBa", new String [] { "abac\uDC16adaba" }); + a.close(); } /** @@ -179,9 +186,9 @@ public class TestAnalyzers extends BaseTokenStreamTestCase { // unpaired trail surrogate assertAnalyzesTo(a, "AbaC\uDC16AdaBa", new String [] { "ABAC\uDC16ADABA" }); + a.close(); } - /** * Test that LowercaseFilter handles the lowercasing correctly if the term * buffer has a trailing surrogate character leftover and the current term in @@ -223,17 +230,20 @@ public class TestAnalyzers extends BaseTokenStreamTestCase { /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new WhitespaceAnalyzer(), 1000*RANDOM_MULTIPLIER); - checkRandomData(random(), new SimpleAnalyzer(), 1000*RANDOM_MULTIPLIER); - checkRandomData(random(), new StopAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer() }; + for (Analyzer analyzer : analyzers) { + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + } + IOUtils.close(analyzers); } /** blast some random large strings through the analyzer */ public void testRandomHugeStrings() throws Exception { - Random random = random(); - checkRandomData(random, new WhitespaceAnalyzer(), 100*RANDOM_MULTIPLIER, 8192); - checkRandomData(random, new SimpleAnalyzer(), 100*RANDOM_MULTIPLIER, 8192); - checkRandomData(random, new StopAnalyzer(), 100*RANDOM_MULTIPLIER, 8192); + Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer() }; + for (Analyzer analyzer : analyzers) { + checkRandomData(random(), analyzer, 100*RANDOM_MULTIPLIER, 8192); + } + IOUtils.close(analyzers); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java index 5e354d2ac5f..32d2c78834d 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java @@ -75,6 +75,7 @@ public class TestBugInSomething extends BaseTokenStreamTestCase { } }; checkAnalysisConsistency(random(), a, false, "wmgddzunizdomqyj"); + a.close(); } CharFilter wrappedStream = new CharFilter(new StringReader("bogus")) { @@ -261,6 +262,7 @@ public class TestBugInSomething extends BaseTokenStreamTestCase { } }; checkRandomData(random(), analyzer, 2000); + analyzer.close(); } public void testCuriousWikipediaString() throws Exception { @@ -285,5 +287,6 @@ public class TestBugInSomething extends BaseTokenStreamTestCase { } }; checkAnalysisConsistency(random(), a, false, "B\u28c3\ue0f8[ \ud800\udfc2

jb"); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java index 97cb8b0758f..b76465e84f4 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java @@ -30,6 +30,7 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.CharacterRunAutomaton; @@ -78,6 +79,7 @@ public class TestDuelingAnalyzers extends BaseTokenStreamTestCase { assertEquals(s, left.tokenStream("foo", newStringReader(s)), right.tokenStream("foo", newStringReader(s))); } + IOUtils.close(left, right); } // not so useful since it's all one token?! @@ -99,6 +101,7 @@ public class TestDuelingAnalyzers extends BaseTokenStreamTestCase { assertEquals(s, left.tokenStream("foo", newStringReader(s)), right.tokenStream("foo", newStringReader(s))); } + IOUtils.close(left, right); } public void testLetterHtmlish() throws Exception { @@ -116,6 +119,7 @@ public class TestDuelingAnalyzers extends BaseTokenStreamTestCase { assertEquals(s, left.tokenStream("foo", newStringReader(s)), right.tokenStream("foo", newStringReader(s))); } + IOUtils.close(left, right); } public void testLetterHtmlishHuge() throws Exception { @@ -136,6 +140,7 @@ public class TestDuelingAnalyzers extends BaseTokenStreamTestCase { assertEquals(s, left.tokenStream("foo", newStringReader(s)), right.tokenStream("foo", newStringReader(s))); } + IOUtils.close(left, right); } public void testLetterUnicode() throws Exception { @@ -153,6 +158,7 @@ public class TestDuelingAnalyzers extends BaseTokenStreamTestCase { assertEquals(s, left.tokenStream("foo", newStringReader(s)), right.tokenStream("foo", newStringReader(s))); } + IOUtils.close(left, right); } public void testLetterUnicodeHuge() throws Exception { @@ -173,6 +179,7 @@ public class TestDuelingAnalyzers extends BaseTokenStreamTestCase { assertEquals(s, left.tokenStream("foo", newStringReader(s)), right.tokenStream("foo", newStringReader(s))); } + IOUtils.close(left, right); } // we only check a few core attributes here. diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java index ccdfc49f1fa..8150fda6e36 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java @@ -46,6 +46,8 @@ import org.apache.lucene.util.Version; // TODO: move this, TestRandomChains, and TestAllAnalyzersHaveFactories // to an integration test module that sucks in all analysis modules. // currently the only way to do this is via eclipse etc (LUCENE-3974) + +// TODO: fix this to use CustomAnalyzer instead of its own FactoryAnalyzer public class TestFactories extends BaseTokenStreamTestCase { public void test() throws IOException { for (String tokenizer : TokenizerFactory.availableTokenizers()) { @@ -77,7 +79,9 @@ public class TestFactories extends BaseTokenStreamTestCase { // beast it just a little, it shouldnt throw exceptions: // (it should have thrown them in initialize) - checkRandomData(random(), new FactoryAnalyzer(factory, null, null), 20, 20, false, false); + Analyzer a = new FactoryAnalyzer(factory, null, null); + checkRandomData(random(), a, 20, 20, false, false); + a.close(); } } @@ -97,7 +101,9 @@ public class TestFactories extends BaseTokenStreamTestCase { // beast it just a little, it shouldnt throw exceptions: // (it should have thrown them in initialize) - checkRandomData(random(), new FactoryAnalyzer(assertingTokenizer, factory, null), 20, 20, false, false); + Analyzer a = new FactoryAnalyzer(assertingTokenizer, factory, null); + checkRandomData(random(), a, 20, 20, false, false); + a.close(); } } @@ -117,7 +123,9 @@ public class TestFactories extends BaseTokenStreamTestCase { // beast it just a little, it shouldnt throw exceptions: // (it should have thrown them in initialize) - checkRandomData(random(), new FactoryAnalyzer(assertingTokenizer, null, factory), 20, 20, false, false); + Analyzer a = new FactoryAnalyzer(assertingTokenizer, null, factory); + checkRandomData(random(), a, 20, 20, false, false); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java index e3f88802633..03746bb3608 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java @@ -19,6 +19,7 @@ package org.apache.lucene.analysis.core; import java.io.StringReader; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; @@ -33,23 +34,24 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiFields; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.TestUtil; public class TestKeywordAnalyzer extends BaseTokenStreamTestCase { private Directory directory; - private IndexSearcher searcher; private IndexReader reader; + private Analyzer analyzer; @Override public void setUp() throws Exception { super.setUp(); directory = newDirectory(); - IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(new SimpleAnalyzer())); + analyzer = new SimpleAnalyzer(); + IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(analyzer)); Document doc = new Document(); doc.add(new StringField("partnum", "Q36", Field.Store.YES)); @@ -59,13 +61,11 @@ public class TestKeywordAnalyzer extends BaseTokenStreamTestCase { writer.close(); reader = DirectoryReader.open(directory); - searcher = newSearcher(reader); } @Override public void tearDown() throws Exception { - reader.close(); - directory.close(); + IOUtils.close(analyzer, reader, directory); super.tearDown(); } @@ -86,7 +86,8 @@ public class TestKeywordAnalyzer extends BaseTokenStreamTestCase { public void testMutipleDocument() throws Exception { RAMDirectory dir = new RAMDirectory(); - IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new KeywordAnalyzer())); + Analyzer analyzer = new KeywordAnalyzer(); + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzer)); Document doc = new Document(); doc.add(new TextField("partnum", "Q36", Field.Store.YES)); writer.addDocument(doc); @@ -112,11 +113,13 @@ public class TestKeywordAnalyzer extends BaseTokenStreamTestCase { null, 0); assertTrue(td.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); + analyzer.close(); } // LUCENE-1441 public void testOffsets() throws Exception { - try (TokenStream stream = new KeywordAnalyzer().tokenStream("field", new StringReader("abcd"))) { + try (Analyzer analyzer = new KeywordAnalyzer(); + TokenStream stream = analyzer.tokenStream("field", new StringReader("abcd"))) { OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class); stream.reset(); assertTrue(stream.incrementToken()); @@ -129,6 +132,8 @@ public class TestKeywordAnalyzer extends BaseTokenStreamTestCase { /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new KeywordAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new KeywordAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java index ced8b729f96..7b6ee171bcf 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java @@ -901,16 +901,17 @@ public class TestRandomChains extends BaseTokenStreamTestCase { int numIterations = TEST_NIGHTLY ? atLeast(20) : 3; Random random = random(); for (int i = 0; i < numIterations; i++) { - MockRandomAnalyzer a = new MockRandomAnalyzer(random.nextLong()); - if (VERBOSE) { - System.out.println("Creating random analyzer:" + a); - } - try { - checkRandomData(random, a, 500*RANDOM_MULTIPLIER, 20, false, - false /* We already validate our own offsets... */); - } catch (Throwable e) { - System.err.println("Exception from random analyzer: " + a); - throw e; + try (MockRandomAnalyzer a = new MockRandomAnalyzer(random.nextLong())) { + if (VERBOSE) { + System.out.println("Creating random analyzer:" + a); + } + try { + checkRandomData(random, a, 500*RANDOM_MULTIPLIER, 20, false, + false /* We already validate our own offsets... */); + } catch (Throwable e) { + System.err.println("Exception from random analyzer: " + a); + throw e; + } } } } @@ -920,16 +921,17 @@ public class TestRandomChains extends BaseTokenStreamTestCase { int numIterations = TEST_NIGHTLY ? atLeast(20) : 3; Random random = random(); for (int i = 0; i < numIterations; i++) { - MockRandomAnalyzer a = new MockRandomAnalyzer(random.nextLong()); - if (VERBOSE) { - System.out.println("Creating random analyzer:" + a); - } - try { - checkRandomData(random, a, 50*RANDOM_MULTIPLIER, 80, false, - false /* We already validate our own offsets... */); - } catch (Throwable e) { - System.err.println("Exception from random analyzer: " + a); - throw e; + try (MockRandomAnalyzer a = new MockRandomAnalyzer(random.nextLong())) { + if (VERBOSE) { + System.out.println("Creating random analyzer:" + a); + } + try { + checkRandomData(random, a, 50*RANDOM_MULTIPLIER, 80, false, + false /* We already validate our own offsets... */); + } catch (Throwable e) { + System.err.println("Exception from random analyzer: " + a); + throw e; + } } } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java index de8b061f070..36b5d416f2d 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java @@ -30,7 +30,7 @@ import java.util.HashSet; public class TestStopAnalyzer extends BaseTokenStreamTestCase { - private StopAnalyzer stop = new StopAnalyzer(); + private StopAnalyzer stop; private Set inValidTokens = new HashSet<>(); @Override @@ -41,6 +41,13 @@ public class TestStopAnalyzer extends BaseTokenStreamTestCase { while(it.hasNext()) { inValidTokens.add(it.next()); } + stop = new StopAnalyzer(); + } + + @Override + public void tearDown() throws Exception { + stop.close(); + super.tearDown(); } public void testDefaults() throws IOException { @@ -71,6 +78,7 @@ public class TestStopAnalyzer extends BaseTokenStreamTestCase { } stream.end(); } + newStop.close(); } public void testStopListPositions() throws IOException { @@ -92,6 +100,7 @@ public class TestStopAnalyzer extends BaseTokenStreamTestCase { } stream.end(); } + newStop.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java index 005c990b739..b20595efe36 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java @@ -62,6 +62,7 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "föó bär FÖÖ BAR", new String[] { "foo", "föó", "bar", "bär", "foo", "föö", "bar" }, new int[] { 1, 0, 1, 0, 1, 0, 1}); + a.close(); } public void testHtmlStripClassicFolding() throws Exception { @@ -93,6 +94,7 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "

föó bär FÖÖ BAR

", new String[] { "foo", "föó", "bar", "bär", "foo", "föö", "bar" }, new int[] { 1, 0, 1, 0, 1, 0, 1}); + a.close(); } public void testStopWordsFromClasspath() throws Exception { @@ -114,6 +116,7 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase { assertSame(Version.LATEST, a.getVersion()); assertAnalyzesTo(a, "foo Foo Bar", new String[0]); + a.close(); } public void testStopWordsFromClasspathWithMap() throws Exception { @@ -141,6 +144,7 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase { } catch (IllegalArgumentException | UnsupportedOperationException e) { // pass } + a.close(); } public void testStopWordsFromFile() throws Exception { @@ -152,6 +156,7 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase { "format", "wordset") .build(); assertAnalyzesTo(a, "foo Foo Bar", new String[0]); + a.close(); } public void testStopWordsFromFileAbsolute() throws Exception { @@ -163,6 +168,7 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase { "format", "wordset") .build(); assertAnalyzesTo(a, "foo Foo Bar", new String[0]); + a.close(); } // Now test misconfigurations: diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java index 91b0be7c3f8..a8505cb71d4 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java @@ -18,6 +18,7 @@ package org.apache.lucene.analysis.cz; */ import java.io.IOException; + import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.util.CharArraySet; @@ -31,15 +32,24 @@ import org.apache.lucene.analysis.util.CharArraySet; */ public class TestCzechAnalyzer extends BaseTokenStreamTestCase { + /** This test fails with NPE when the + * stopwords file is missing in classpath */ + public void testResourcesAvailable() { + new CzechAnalyzer().close(); + } + public void testStopWord() throws Exception { - assertAnalyzesTo(new CzechAnalyzer(), "Pokud mluvime o volnem", + Analyzer analyzer = new CzechAnalyzer(); + assertAnalyzesTo(analyzer, "Pokud mluvime o volnem", new String[] { "mluvim", "voln" }); + analyzer.close(); } public void testReusableTokenStream() throws Exception { Analyzer analyzer = new CzechAnalyzer(); assertAnalyzesTo(analyzer, "Pokud mluvime o volnem", new String[] { "mluvim", "voln" }); assertAnalyzesTo(analyzer, "Česká Republika", new String[] { "česk", "republik" }); + analyzer.close(); } public void testWithStemExclusionSet() throws IOException{ @@ -47,10 +57,13 @@ public class TestCzechAnalyzer extends BaseTokenStreamTestCase { set.add("hole"); CzechAnalyzer cz = new CzechAnalyzer(CharArraySet.EMPTY_SET, set); assertAnalyzesTo(cz, "hole desek", new String[] {"hole", "desk"}); + cz.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new CzechAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new CzechAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemmer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemmer.java index a0eede07029..259e948bfe5 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemmer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemmer.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.cz; */ import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; @@ -103,6 +102,8 @@ public class TestCzechStemmer extends BaseTokenStreamTestCase { assertAnalyzesTo(cz, "soudcům", new String[] { "soudk" }); assertAnalyzesTo(cz, "soudcích", new String[] { "soudk" }); assertAnalyzesTo(cz, "soudcem", new String[] { "soudk" }); + + cz.close(); } /** @@ -147,6 +148,8 @@ public class TestCzechStemmer extends BaseTokenStreamTestCase { assertAnalyzesTo(cz, "ženách", new String[] { "žn" }); assertAnalyzesTo(cz, "ženou", new String[] { "žn" }); assertAnalyzesTo(cz, "ženami", new String[] { "žn" }); + + cz.close(); } /** @@ -189,7 +192,9 @@ public class TestCzechStemmer extends BaseTokenStreamTestCase { assertAnalyzesTo(cz, "stavení", new String[] { "stavn" }); assertAnalyzesTo(cz, "stavením", new String[] { "stavn" }); assertAnalyzesTo(cz, "staveních", new String[] { "stavn" }); - assertAnalyzesTo(cz, "staveními", new String[] { "stavn" }); + assertAnalyzesTo(cz, "staveními", new String[] { "stavn" }); + + cz.close(); } /** @@ -218,6 +223,8 @@ public class TestCzechStemmer extends BaseTokenStreamTestCase { assertAnalyzesTo(cz, "jarnímu", new String[] { "jarn" }); assertAnalyzesTo(cz, "jarním", new String[] { "jarn" }); assertAnalyzesTo(cz, "jarními", new String[] { "jarn" }); + + cz.close(); } /** @@ -227,6 +234,7 @@ public class TestCzechStemmer extends BaseTokenStreamTestCase { CzechAnalyzer cz = new CzechAnalyzer(); assertAnalyzesTo(cz, "Karlův", new String[] { "karl" }); assertAnalyzesTo(cz, "jazykový", new String[] { "jazyk" }); + cz.close(); } /** @@ -267,6 +275,8 @@ public class TestCzechStemmer extends BaseTokenStreamTestCase { /* rewrite of e* -> * */ assertAnalyzesTo(cz, "deska", new String[] { "desk" }); assertAnalyzesTo(cz, "desek", new String[] { "desk" }); + + cz.close(); } /** @@ -276,6 +286,7 @@ public class TestCzechStemmer extends BaseTokenStreamTestCase { CzechAnalyzer cz = new CzechAnalyzer(); assertAnalyzesTo(cz, "e", new String[] { "e" }); assertAnalyzesTo(cz, "zi", new String[] { "zi" }); + cz.close(); } public void testWithKeywordAttribute() throws IOException { @@ -297,6 +308,7 @@ public class TestCzechStemmer extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/da/TestDanishAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/da/TestDanishAnalyzer.java index a0a591060b5..3df580020f5 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/da/TestDanishAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/da/TestDanishAnalyzer.java @@ -27,7 +27,7 @@ public class TestDanishAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new DanishAnalyzer(); + new DanishAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestDanishAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "undersøgelse", "undersøg"); // stopword assertAnalyzesTo(a, "på", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestDanishAnalyzer extends BaseTokenStreamTestCase { DanishAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "undersøgelse", "undersøgelse"); checkOneTerm(a, "undersøg", "undersøg"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new DanishAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new DanishAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java index a9095781422..2a4917a2e9f 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java @@ -32,6 +32,7 @@ public class TestGermanAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "Tisch", "tisch"); checkOneTerm(a, "Tische", "tisch"); checkOneTerm(a, "Tischen", "tisch"); + a.close(); } public void testWithKeywordAttribute() throws IOException { @@ -48,6 +49,7 @@ public class TestGermanAnalyzer extends BaseTokenStreamTestCase { GermanAnalyzer a = new GermanAnalyzer( CharArraySet.EMPTY_SET, new CharArraySet( asSet("tischen"), false)); checkOneTerm(a, "tischen", "tischen"); + a.close(); } /** test some features of the new snowball filter @@ -58,10 +60,13 @@ public class TestGermanAnalyzer extends BaseTokenStreamTestCase { // a/o/u + e is equivalent to the umlaut form checkOneTerm(a, "Schaltflächen", "schaltflach"); checkOneTerm(a, "Schaltflaechen", "schaltflach"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new GermanAnalyzer(), 1000*RANDOM_MULTIPLIER); + GermanAnalyzer a = new GermanAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java index 5aac3b34ded..f248b2ebb96 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.de; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -35,13 +34,25 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Simple tests for {@link GermanLightStemFilter} */ public class TestGermanLightStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new GermanLightStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new GermanLightStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test against a vocabulary from the reference impl */ public void testVocabulary() throws IOException { @@ -59,6 +70,7 @@ public class TestGermanLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "sängerinnen", "sängerinnen"); + a.close(); } /** blast some random strings through the analyzer */ @@ -75,5 +87,6 @@ public class TestGermanLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java index dd0f3200489..3900aa910a6 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.de; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -35,13 +34,25 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Simple tests for {@link GermanMinimalStemFilter} */ public class TestGermanMinimalStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new GermanMinimalStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new GermanMinimalStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test some examples from the paper */ public void testExamples() throws IOException { @@ -66,6 +77,7 @@ public class TestGermanMinimalStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "sängerinnen", "sängerinnen"); + a.close(); } /** Test against a vocabulary from the reference impl */ @@ -87,5 +99,6 @@ public class TestGermanMinimalStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilter.java index be590eb5722..910bc5046b9 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.de; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -31,14 +30,26 @@ import org.apache.lucene.analysis.core.KeywordTokenizer; * Tests {@link GermanNormalizationFilter} */ public class TestGermanNormalizationFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String field) { - final Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); - final TokenStream stream = new GermanNormalizationFilter(tokenizer); - return new TokenStreamComponents(tokenizer, stream); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String field) { + final Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); + final TokenStream stream = new GermanNormalizationFilter(tokenizer); + return new TokenStreamComponents(tokenizer, stream); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** * Tests that a/o/u + e is equivalent to the umlaut form @@ -76,5 +87,6 @@ public class TestGermanNormalizationFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java index 1b49c409ccd..975d1ce35fb 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java @@ -25,6 +25,7 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.Analyzer.TokenStreamComponents; import org.apache.lucene.analysis.core.KeywordTokenizer; import org.apache.lucene.analysis.core.LowerCaseFilter; import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter; @@ -39,14 +40,26 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * */ public class TestGermanStemFilter extends BaseTokenStreamTestCase { - Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer t = new MockTokenizer(MockTokenizer.KEYWORD, false); - return new TokenStreamComponents(t, - new GermanStemFilter(new LowerCaseFilter(t))); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer t = new MockTokenizer(MockTokenizer.KEYWORD, false); + return new TokenStreamComponents(t, + new GermanStemFilter(new LowerCaseFilter(t))); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } public void testStemming() throws Exception { InputStream vocOut = getClass().getResourceAsStream("data.txt"); @@ -65,6 +78,7 @@ public class TestGermanStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "sängerinnen", "sängerinnen"); + a.close(); } /** blast some random strings through the analyzer */ @@ -81,5 +95,6 @@ public class TestGermanStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/GreekAnalyzerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/GreekAnalyzerTest.java index d416898cb08..6939d2018df 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/GreekAnalyzerTest.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/GreekAnalyzerTest.java @@ -45,6 +45,7 @@ public class GreekAnalyzerTest extends BaseTokenStreamTestCase { // as well as the elimination of stop words assertAnalyzesTo(a, "ΠΡΟΫΠΟΘΕΣΕΙΣ Άψογος, ο μεστός και οι άλλοι", new String[] { "προυποθεσ", "αψογ", "μεστ", "αλλ" }); + a.close(); } public void testReusableTokenStream() throws Exception { @@ -62,10 +63,13 @@ public class GreekAnalyzerTest extends BaseTokenStreamTestCase { // as well as the elimination of stop words assertAnalyzesTo(a, "ΠΡΟΫΠΟΘΕΣΕΙΣ Άψογος, ο μεστός και οι άλλοι", new String[] { "προυποθεσ", "αψογ", "μεστ", "αλλ" }); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new GreekAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer a = new GreekAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemmer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemmer.java index cd5a2c1b105..16bb0f376f6 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemmer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemmer.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.el; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -26,7 +25,19 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; public class TestGreekStemmer extends BaseTokenStreamTestCase { - Analyzer a = new GreekAnalyzer(); + private Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new GreekAnalyzer(); + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } public void testMasculineNouns() throws Exception { // -ος @@ -537,5 +548,6 @@ public class TestGreekStemmer extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishAnalyzer.java index 3844cbd7960..be0a36b1d59 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishAnalyzer.java @@ -27,7 +27,7 @@ public class TestEnglishAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new EnglishAnalyzer(); + new EnglishAnalyzer().close(); } /** test stopwords and stemming */ @@ -42,6 +42,7 @@ public class TestEnglishAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "steven's", "steven"); checkOneTerm(a, "steven\u2019s", "steven"); checkOneTerm(a, "steven\uFF07s", "steven"); + a.close(); } /** test use of exclusion set */ @@ -51,10 +52,13 @@ public class TestEnglishAnalyzer extends BaseTokenStreamTestCase { EnglishAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "books", "books"); checkOneTerm(a, "book", "book"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new EnglishAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer a = new EnglishAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java index dbb71449f54..64b37307dab 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.en; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -30,14 +29,26 @@ import org.apache.lucene.analysis.core.KeywordTokenizer; * Simple tests for {@link EnglishMinimalStemFilter} */ public class TestEnglishMinimalStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new EnglishMinimalStemFilter(source)); - } - }; + private Analyzer analyzer; + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new EnglishMinimalStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } + /** Test some examples from various papers about this technique */ public void testExamples() throws IOException { checkOneTerm(analyzer, "queries", "query"); @@ -65,5 +76,6 @@ public class TestEnglishMinimalStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java index 755354fc8c0..9339aec2a0e 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java @@ -20,7 +20,6 @@ package org.apache.lucene.analysis.en; import static org.apache.lucene.analysis.VocabularyAssert.assertVocabulary; import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -32,13 +31,25 @@ import org.apache.lucene.analysis.core.KeywordTokenizer; * Tests for {@link KStemmer} */ public class TestKStemmer extends BaseTokenStreamTestCase { - Analyzer a = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true); - return new TokenStreamComponents(tokenizer, new KStemFilter(tokenizer)); - } - }; + private Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true); + return new TokenStreamComponents(tokenizer, new KStemFilter(tokenizer)); + } + }; + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { @@ -63,6 +74,7 @@ public class TestKStemmer extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } /****** requires original java kstem source code to create map diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java index db7fc328c4f..8627c3b8f33 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.en; */ import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -36,13 +35,25 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Test the PorterStemFilter with Martin Porter's test data. */ public class TestPorterStemFilter extends BaseTokenStreamTestCase { - Analyzer a = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer t = new MockTokenizer( MockTokenizer.KEYWORD, false); - return new TokenStreamComponents(t, new PorterStemFilter(t)); - } - }; + private Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer t = new MockTokenizer( MockTokenizer.KEYWORD, false); + return new TokenStreamComponents(t, new PorterStemFilter(t)); + } + }; + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } /** * Run the stemmer against all strings in voc.txt @@ -75,5 +86,6 @@ public class TestPorterStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishAnalyzer.java index 9a6c06f0d31..366d9798fe2 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishAnalyzer.java @@ -27,7 +27,7 @@ public class TestSpanishAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new SpanishAnalyzer(); + new SpanishAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestSpanishAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "chicano", "chican"); // stopword assertAnalyzesTo(a, "los", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestSpanishAnalyzer extends BaseTokenStreamTestCase { SpanishAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "chicana", "chican"); checkOneTerm(a, "chicano", "chicano"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new SpanishAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer a = new SpanishAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java index 81a3b90be41..fc25b69c63e 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.es; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -32,14 +31,26 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Simple tests for {@link SpanishLightStemFilter} */ public class TestSpanishLightStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new SpanishLightStemFilter(source)); - } - }; + private Analyzer analyzer; + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new SpanishLightStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } + /** Test against a vocabulary from the reference impl */ public void testVocabulary() throws IOException { assertVocabulary(analyzer, getDataPath("eslighttestdata.zip"), "eslight.txt"); @@ -59,5 +70,6 @@ public class TestSpanishLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/eu/TestBasqueAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/eu/TestBasqueAnalyzer.java index d398ec9d437..4e3ee219d4d 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/eu/TestBasqueAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/eu/TestBasqueAnalyzer.java @@ -27,7 +27,7 @@ public class TestBasqueAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new BasqueAnalyzer(); + new BasqueAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestBasqueAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "zaldiak", "zaldi"); // stopword assertAnalyzesTo(a, "izan", new String[] { }); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestBasqueAnalyzer extends BaseTokenStreamTestCase { BasqueAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "zaldiak", "zaldiak"); checkOneTerm(a, "mendiari", "mendi"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new BasqueAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer a = new BasqueAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java index 67dace3253c..a142dca1565 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java @@ -31,7 +31,7 @@ public class TestPersianAnalyzer extends BaseTokenStreamTestCase { * This test fails with NPE when the stopwords file is missing in classpath */ public void testResourcesAvailable() { - new PersianAnalyzer(); + new PersianAnalyzer().close(); } /** @@ -105,6 +105,7 @@ public class TestPersianAnalyzer extends BaseTokenStreamTestCase { // active present subjunctive assertAnalyzesTo(a, "بخورد", new String[] { "بخورد" }); + a.close(); } /** @@ -181,6 +182,7 @@ public class TestPersianAnalyzer extends BaseTokenStreamTestCase { // active present subjunctive assertAnalyzesTo(a, "بخورد", new String[] { "بخورد" }); + a.close(); } /** @@ -192,6 +194,7 @@ public class TestPersianAnalyzer extends BaseTokenStreamTestCase { Analyzer a = new PersianAnalyzer(); assertAnalyzesTo(a, "برگ ها", new String[] { "برگ" }); assertAnalyzesTo(a, "برگ‌ها", new String[] { "برگ" }); + a.close(); } /** @@ -201,6 +204,7 @@ public class TestPersianAnalyzer extends BaseTokenStreamTestCase { public void testBehaviorNonPersian() throws Exception { Analyzer a = new PersianAnalyzer(); assertAnalyzesTo(a, "English test.", new String[] { "english", "test" }); + a.close(); } /** @@ -210,6 +214,7 @@ public class TestPersianAnalyzer extends BaseTokenStreamTestCase { Analyzer a = new PersianAnalyzer(); assertAnalyzesTo(a, "خورده مي شده بوده باشد", new String[] { "خورده" }); assertAnalyzesTo(a, "برگ‌ها", new String[] { "برگ" }); + a.close(); } /** @@ -220,10 +225,13 @@ public class TestPersianAnalyzer extends BaseTokenStreamTestCase { new CharArraySet( asSet("the", "and", "a"), false)); assertAnalyzesTo(a, "The quick brown fox.", new String[] { "quick", "brown", "fox" }); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new PersianAnalyzer(), 1000*RANDOM_MULTIPLIER); + PersianAnalyzer a = new PersianAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java index 4d48ab0bc7d..0d10b936c04 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java @@ -24,17 +24,29 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.MockTokenizer; public class TestPersianCharFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - return new TokenStreamComponents(new MockTokenizer()); - } + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + return new TokenStreamComponents(new MockTokenizer()); + } - @Override - protected Reader initReader(String fieldName, Reader reader) { - return new PersianCharFilter(reader); - } - }; + @Override + protected Reader initReader(String fieldName, Reader reader) { + return new PersianCharFilter(reader); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } public void testBasics() throws Exception { assertAnalyzesTo(analyzer, "this is a\u200Ctest", diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java index 31d172f17bc..d8d504eb981 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java @@ -18,8 +18,6 @@ package org.apache.lucene.analysis.fa; */ import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -73,6 +71,7 @@ public class TestPersianNormalizationFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishAnalyzer.java index 3fb7ce85369..659bed06854 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishAnalyzer.java @@ -27,7 +27,7 @@ public class TestFinnishAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new FinnishAnalyzer(); + new FinnishAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestFinnishAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "edeltäjistään", "edeltäj"); // stopword assertAnalyzesTo(a, "olla", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestFinnishAnalyzer extends BaseTokenStreamTestCase { FinnishAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "edeltäjiinsä", "edeltäj"); checkOneTerm(a, "edeltäjistään", "edeltäjistään"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new FinnishAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer a = new FinnishAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java index f183f5bb721..05bba6a737d 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java @@ -34,13 +34,25 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Simple tests for {@link FinnishLightStemFilter} */ public class TestFinnishLightStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new FinnishLightStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new FinnishLightStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test against a vocabulary from the reference impl */ public void testVocabulary() throws IOException { @@ -58,6 +70,7 @@ public class TestFinnishLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "edeltäjistään", "edeltäjistään"); + a.close(); } /** blast some random strings through the analyzer */ @@ -74,5 +87,6 @@ public class TestFinnishLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java index 112573b358c..a441dd618d5 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java @@ -110,7 +110,7 @@ public class TestFrenchAnalyzer extends BaseTokenStreamTestCase { fa, "33Bis 1940-1945 1940:1945 (---i+++)*", new String[] { "33bi", "1940", "1945", "1940", "1945", "i" }); - + fa.close(); } public void testReusableTokenStream() throws Exception { @@ -132,6 +132,7 @@ public class TestFrenchAnalyzer extends BaseTokenStreamTestCase { "chist", "element", "captif" }); + fa.close(); } public void testExclusionTableViaCtor() throws Exception { @@ -141,15 +142,18 @@ public class TestFrenchAnalyzer extends BaseTokenStreamTestCase { CharArraySet.EMPTY_SET, set); assertAnalyzesTo(fa, "habitable chiste", new String[] { "habitable", "chist" }); + fa.close(); fa = new FrenchAnalyzer( CharArraySet.EMPTY_SET, set); assertAnalyzesTo(fa, "habitable chiste", new String[] { "habitable", "chist" }); + fa.close(); } public void testElision() throws Exception { FrenchAnalyzer fa = new FrenchAnalyzer(); assertAnalyzesTo(fa, "voir l'embrouille", new String[] { "voir", "embrouil" }); + fa.close(); } /** @@ -158,11 +162,14 @@ public class TestFrenchAnalyzer extends BaseTokenStreamTestCase { public void testStopwordsCasing() throws IOException { FrenchAnalyzer a = new FrenchAnalyzer(); assertAnalyzesTo(a, "Votre", new String[] { }); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new FrenchAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer a = new FrenchAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } /** test accent-insensitive */ @@ -170,5 +177,6 @@ public class TestFrenchAnalyzer extends BaseTokenStreamTestCase { Analyzer a = new FrenchAnalyzer(); checkOneTerm(a, "sécuritaires", "securitair"); checkOneTerm(a, "securitaires", "securitair"); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java index 743962b88a1..cc3a5a1f7f7 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.fr; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -35,13 +34,25 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Simple tests for {@link FrenchLightStemFilter} */ public class TestFrenchLightStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer( MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new FrenchLightStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer( MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new FrenchLightStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test some examples from the paper */ public void testExamples() throws IOException { @@ -189,6 +200,7 @@ public class TestFrenchLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "chevaux", "chevaux"); + a.close(); } /** blast some random strings through the analyzer */ @@ -205,5 +217,6 @@ public class TestFrenchLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java index 7f44a7f710e..24f8a8221d0 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.fr; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -35,13 +34,25 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Simple tests for {@link FrenchMinimalStemFilter} */ public class TestFrenchMinimalStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new FrenchMinimalStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new FrenchMinimalStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test some examples from the paper */ public void testExamples() throws IOException { @@ -68,6 +79,7 @@ public class TestFrenchMinimalStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "chevaux", "chevaux"); + a.close(); } /** Test against a vocabulary from the reference impl */ @@ -89,5 +101,6 @@ public class TestFrenchMinimalStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java index 994dff5adda..c979722e595 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java @@ -27,7 +27,7 @@ public class TestIrishAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new IrishAnalyzer(); + new IrishAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestIrishAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "síceapatacha", "síceapaite"); // stopword assertAnalyzesTo(a, "le", new String[] { }); + a.close(); } /** test use of elisionfilter */ @@ -45,6 +46,7 @@ public class TestIrishAnalyzer extends BaseTokenStreamTestCase { Analyzer a = new IrishAnalyzer(); assertAnalyzesTo(a, "b'fhearr m'athair", new String[] { "fearr", "athair" }); + a.close(); } /** test use of exclusion set */ @@ -54,6 +56,7 @@ public class TestIrishAnalyzer extends BaseTokenStreamTestCase { IrishAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "feirmeoireacht", "feirmeoireacht"); checkOneTerm(a, "siopadóireacht", "siopadóir"); + a.close(); } /** test special hyphen handling */ @@ -62,10 +65,13 @@ public class TestIrishAnalyzer extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "n-athair", new String[] { "athair" }, new int[] { 2 }); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new IrishAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer a = new IrishAnalyzer(); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java index 22fd555c70e..3f68e1d3720 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java @@ -48,5 +48,6 @@ public class TestIrishLowerCaseFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianAnalyzer.java index 3d5e47e08d5..cbbe396fcb4 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianAnalyzer.java @@ -27,7 +27,7 @@ public class TestGalicianAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new GalicianAnalyzer(); + new GalicianAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestGalicianAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "corresponderá", "correspond"); // stopword assertAnalyzesTo(a, "e", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestGalicianAnalyzer extends BaseTokenStreamTestCase { GalicianAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "correspondente", "correspondente"); checkOneTerm(a, "corresponderá", "correspond"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new GalicianAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new GalicianAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java index b309bb422a5..7b0503c5399 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.gl; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -33,13 +32,25 @@ import org.apache.lucene.analysis.util.CharArraySet; * Simple tests for {@link GalicianMinimalStemmer} */ public class TestGalicianMinimalStemFilter extends BaseTokenStreamTestCase { - Analyzer a = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(tokenizer, new GalicianMinimalStemFilter(tokenizer)); - } - }; + private Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(tokenizer, new GalicianMinimalStemFilter(tokenizer)); + } + }; + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } public void testPlural() throws Exception { checkOneTerm(a, "elefantes", "elefante"); @@ -64,6 +75,7 @@ public class TestGalicianMinimalStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "elefantes", "elefantes"); + a.close(); } /** blast some random strings through the analyzer */ @@ -80,5 +92,6 @@ public class TestGalicianMinimalStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java index 6b0c9447cba..5f8116bab2b 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java @@ -20,29 +20,36 @@ package org.apache.lucene.analysis.gl; import static org.apache.lucene.analysis.VocabularyAssert.assertVocabulary; import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; -import org.apache.lucene.analysis.core.LowerCaseFilter; -import org.apache.lucene.analysis.standard.StandardTokenizer; /** * Simple tests for {@link GalicianStemFilter} */ public class TestGalicianStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new GalicianStemFilter(source)); - } - }; + private Analyzer analyzer; + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new GalicianStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test against a vocabulary from the reference impl */ public void testVocabulary() throws IOException { @@ -58,5 +65,6 @@ public class TestGalicianStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiAnalyzer.java index be9eadaf9de..8b2793629c5 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiAnalyzer.java @@ -28,7 +28,7 @@ public class TestHindiAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new HindiAnalyzer(); + new HindiAnalyzer().close(); } public void testBasics() throws Exception { @@ -36,6 +36,7 @@ public class TestHindiAnalyzer extends BaseTokenStreamTestCase { // two ways to write 'hindi' itself. checkOneTerm(a, "हिन्दी", "हिंद"); checkOneTerm(a, "हिंदी", "हिंद"); + a.close(); } public void testExclusionSet() throws Exception { @@ -43,10 +44,13 @@ public class TestHindiAnalyzer extends BaseTokenStreamTestCase { Analyzer a = new HindiAnalyzer( HindiAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "हिंदी", "हिंदी"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new HindiAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new HindiAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java index f68563dfff7..d30dae716dc 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java @@ -76,5 +76,6 @@ public class TestHindiNormalizer extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java index 05f664b438b..a10fa8ff897 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java @@ -95,5 +95,6 @@ public class TestHindiStemmer extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianAnalyzer.java index 5caff3fb30d..b925f4224ca 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianAnalyzer.java @@ -27,7 +27,7 @@ public class TestHungarianAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new HungarianAnalyzer(); + new HungarianAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestHungarianAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "babakocsijáért", "babakocs"); // stopword assertAnalyzesTo(a, "által", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestHungarianAnalyzer extends BaseTokenStreamTestCase { HungarianAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "babakocsi", "babakocsi"); checkOneTerm(a, "babakocsijáért", "babakocs"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new HungarianAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new HungarianAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java index b8740f9c4ee..ac2c0b8dc13 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java @@ -34,13 +34,25 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Simple tests for {@link HungarianLightStemFilter} */ public class TestHungarianLightStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new HungarianLightStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new HungarianLightStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test against a vocabulary from the reference impl */ public void testVocabulary() throws IOException { @@ -58,6 +70,7 @@ public class TestHungarianLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "babakocsi", "babakocsi"); + a.close(); } public void testEmptyTerm() throws IOException { @@ -69,5 +82,6 @@ public class TestHungarianLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java index 9deaca10795..00d6f400328 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java @@ -87,6 +87,7 @@ public class TestHunspellStemFilter extends BaseTokenStreamTestCase { } }; checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } public void testEmptyTerm() throws IOException { @@ -98,6 +99,7 @@ public class TestHunspellStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } public void testIgnoreCaseNoSideEffects() throws Exception { @@ -118,5 +120,6 @@ public class TestHunspellStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "NoChAnGy", "NoChAnGy"); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hy/TestArmenianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hy/TestArmenianAnalyzer.java index 2e046189d01..73410adebba 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hy/TestArmenianAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hy/TestArmenianAnalyzer.java @@ -27,7 +27,7 @@ public class TestArmenianAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new ArmenianAnalyzer(); + new ArmenianAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestArmenianAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "արծիվներ", "արծ"); // stopword assertAnalyzesTo(a, "է", new String[] { }); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestArmenianAnalyzer extends BaseTokenStreamTestCase { ArmenianAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "արծիվներ", "արծիվներ"); checkOneTerm(a, "արծիվ", "արծ"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new ArmenianAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new ArmenianAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianAnalyzer.java index a134b31c015..42440f4a884 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianAnalyzer.java @@ -27,7 +27,7 @@ public class TestIndonesianAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new IndonesianAnalyzer(); + new IndonesianAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestIndonesianAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "pembunuhan", "bunuh"); // stopword assertAnalyzesTo(a, "bahwa", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestIndonesianAnalyzer extends BaseTokenStreamTestCase { IndonesianAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "peledakan", "peledakan"); checkOneTerm(a, "pembunuhan", "bunuh"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new IndonesianAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new IndonesianAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java index 24a2c7449ab..c72db639a76 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java @@ -18,26 +18,46 @@ package org.apache.lucene.analysis.id; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; +import org.apache.lucene.util.IOUtils; /** * Tests {@link IndonesianStemmer} */ public class TestIndonesianStemmer extends BaseTokenStreamTestCase { - /* full stemming, no stopwords */ - Analyzer a = new Analyzer() { - @Override - public TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false); - return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer)); - } - }; + private Analyzer a, b; + + @Override + public void setUp() throws Exception { + super.setUp(); + /* full stemming, no stopwords */ + a = new Analyzer() { + @Override + public TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false); + return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer)); + } + }; + /* inflectional-only stemming */ + b = new Analyzer() { + @Override + public TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false); + return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer, false)); + } + }; + } + + @Override + public void tearDown() throws Exception { + IOUtils.close(a, b); + super.tearDown(); + } /** Some examples from the paper */ public void testExamples() throws IOException { @@ -111,15 +131,6 @@ public class TestIndonesianStemmer extends BaseTokenStreamTestCase { checkOneTerm(a, "kecelakaan", "celaka"); } - /* inflectional-only stemming */ - Analyzer b = new Analyzer() { - @Override - public TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false); - return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer, false)); - } - }; - /** Test stemming only inflectional suffixes */ public void testInflectionalOnly() throws IOException { checkOneTerm(b, "bukunya", "buku"); @@ -143,5 +154,6 @@ public class TestIndonesianStemmer extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java index f0aad1a9c4d..e4bdd6772c0 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.in; */ import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; @@ -62,5 +61,6 @@ public class TestIndicNormalizer extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianAnalyzer.java index ec359fbd44b..fc97a8e705f 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianAnalyzer.java @@ -27,7 +27,7 @@ public class TestItalianAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new ItalianAnalyzer(); + new ItalianAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestItalianAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "abbandonati", "abbandonat"); // stopword assertAnalyzesTo(a, "dallo", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -47,11 +48,14 @@ public class TestItalianAnalyzer extends BaseTokenStreamTestCase { ItalianAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "abbandonata", "abbandonata"); checkOneTerm(a, "abbandonati", "abbandonat"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new ItalianAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new ItalianAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } /** test that the elisionfilter is working */ @@ -59,5 +63,6 @@ public class TestItalianAnalyzer extends BaseTokenStreamTestCase { Analyzer a = new ItalianAnalyzer(); assertAnalyzesTo(a, "dell'Italia", new String[] { "ital" }); assertAnalyzesTo(a, "l'Italiano", new String[] { "italian" }); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java index c56989d1f86..f2d62753208 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.it; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -32,13 +31,25 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Simple tests for {@link ItalianLightStemFilter} */ public class TestItalianLightStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new ItalianLightStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new ItalianLightStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test against a vocabulary from the reference impl */ public void testVocabulary() throws IOException { @@ -59,5 +70,6 @@ public class TestItalianLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianAnalyzer.java index 4bf69a503cf..30541242411 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianAnalyzer.java @@ -27,7 +27,7 @@ public class TestLatvianAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new LatvianAnalyzer(); + new LatvianAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestLatvianAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "tirgus", "tirg"); // stopword assertAnalyzesTo(a, "un", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestLatvianAnalyzer extends BaseTokenStreamTestCase { LatvianAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "tirgiem", "tirgiem"); checkOneTerm(a, "tirgus", "tirg"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new LatvianAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new LatvianAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java index 6c0cd479328..f532871bd29 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.lv; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -30,13 +29,25 @@ import org.apache.lucene.analysis.core.KeywordTokenizer; * Basic tests for {@link LatvianStemmer} */ public class TestLatvianStemmer extends BaseTokenStreamTestCase { - private Analyzer a = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(tokenizer, new LatvianStemFilter(tokenizer)); - } - }; + private Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(tokenizer, new LatvianStemFilter(tokenizer)); + } + }; + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } public void testNouns1() throws IOException { // decl. I @@ -279,5 +290,6 @@ public class TestLatvianStemmer extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java index 19ceeccf2e8..5214dd51603 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java @@ -1933,6 +1933,7 @@ public class TestASCIIFoldingFilter extends BaseTokenStreamTestCase { } }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } public void testEmptyTerm() throws IOException { @@ -1945,5 +1946,6 @@ public class TestASCIIFoldingFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java index 0d1141ef77a..d9a3563b938 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java @@ -137,6 +137,7 @@ public class TestCapitalizationFilter extends BaseTokenStreamTestCase { }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } public void testEmptyTerm() throws IOException { @@ -148,6 +149,7 @@ public class TestCapitalizationFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } /** diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java index 442cfe2b54f..8f8a40c4468 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java @@ -47,6 +47,7 @@ public class TestCodepointCountFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } public void testRandomStrings() throws IOException { diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java index 22af12b90dc..48748801319 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.miscellaneous; import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; @@ -78,6 +77,7 @@ public class TestHyphenatedWordsFilter extends BaseTokenStreamTestCase { }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } public void testEmptyTerm() throws IOException { @@ -89,5 +89,6 @@ public class TestHyphenatedWordsFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java index 50b5edcd717..1922ffd0728 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java @@ -66,5 +66,6 @@ public class TestKeepWordFilter extends BaseTokenStreamTestCase { }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java index 89e377f4fc7..1c5de9ccb71 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java @@ -18,12 +18,9 @@ package org.apache.lucene.analysis.miscellaneous; */ import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; @@ -49,6 +46,7 @@ public class TestLengthFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } /** diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java index 3d191d54d2e..55b07236fc0 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java @@ -52,6 +52,7 @@ public class TestLimitTokenCountAnalyzer extends BaseTokenStreamTestCase { // equal to limit assertTokenStreamContents(a.tokenStream("dummy", "1 2 "), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null); + a.close(); } } @@ -86,6 +87,7 @@ public class TestLimitTokenCountAnalyzer extends BaseTokenStreamTestCase { assertEquals(0, reader.docFreq(t)); reader.close(); dir.close(); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java index 56431fb401e..23b5307002a 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java @@ -56,6 +56,7 @@ public class TestLimitTokenPositionFilter extends BaseTokenStreamTestCase { // equal to limit assertTokenStreamContents(a.tokenStream("dummy", "1 2 "), new String[]{"1", "2"}, new int[]{0, 3}, new int[]{1, 4}, consumeAll ? 6 : null); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java index ff833866472..99e98c88dc3 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java @@ -15,6 +15,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.core.SimpleAnalyzer; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.Rethrow; /* @@ -40,9 +41,11 @@ public class TestPerFieldAnalyzerWrapper extends BaseTokenStreamTestCase { Map analyzerPerField = Collections.singletonMap("special", new SimpleAnalyzer()); + + Analyzer defaultAnalyzer = new WhitespaceAnalyzer(); PerFieldAnalyzerWrapper analyzer = - new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(), analyzerPerField); + new PerFieldAnalyzerWrapper(defaultAnalyzer, analyzerPerField); try (TokenStream tokenStream = analyzer.tokenStream("field", text)) { CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class); @@ -67,6 +70,10 @@ public class TestPerFieldAnalyzerWrapper extends BaseTokenStreamTestCase { assertFalse(tokenStream.incrementToken()); tokenStream.end(); } + // TODO: fix this about PFAW, this is crazy + analyzer.close(); + defaultAnalyzer.close(); + IOUtils.close(analyzerPerField.values()); } public void testReuseWrapped() throws Exception { @@ -124,6 +131,7 @@ public class TestPerFieldAnalyzerWrapper extends BaseTokenStreamTestCase { ts4 = wrapper3.tokenStream("moreSpecial", text); assertSame(ts3, ts4); assertSame(ts2, ts3); + IOUtils.close(wrapper3, wrapper2, wrapper1, specialAnalyzer, defaultAnalyzer); } public void testCharFilters() throws Exception { @@ -152,5 +160,7 @@ public class TestPerFieldAnalyzerWrapper extends BaseTokenStreamTestCase { new int[] { 0 }, new int[] { 2 } ); + p.close(); + a.close(); // TODO: fix this about PFAW, its a trap } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java index 667bedb6ec3..affe5675490 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java @@ -163,6 +163,7 @@ public class TestRemoveDuplicatesTokenFilter extends BaseTokenStreamTestCase { }; checkRandomData(random(), analyzer, 200); + analyzer.close(); } } @@ -175,6 +176,7 @@ public class TestRemoveDuplicatesTokenFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianFoldingFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianFoldingFilter.java index 91051463fe3..8ceb6eb0117 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianFoldingFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianFoldingFilter.java @@ -24,19 +24,27 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; -import java.io.Reader; - public class TestScandinavianFoldingFilter extends BaseTokenStreamTestCase { - - - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String field) { - final Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); - final TokenStream stream = new ScandinavianFoldingFilter(tokenizer); - return new TokenStreamComponents(tokenizer, stream); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String field) { + final Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); + final TokenStream stream = new ScandinavianFoldingFilter(tokenizer); + return new TokenStreamComponents(tokenizer, stream); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } public void test() throws Exception { @@ -117,6 +125,7 @@ public class TestScandinavianFoldingFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } /** blast some random strings through the analyzer */ diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianNormalizationFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianNormalizationFilter.java index 228dac38249..2db274b37b0 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianNormalizationFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianNormalizationFilter.java @@ -24,20 +24,27 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; -import java.io.Reader; - - public class TestScandinavianNormalizationFilter extends BaseTokenStreamTestCase { - - - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String field) { - final Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); - final TokenStream stream = new ScandinavianNormalizationFilter(tokenizer); - return new TokenStreamComponents(tokenizer, stream); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String field) { + final Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); + final TokenStream stream = new ScandinavianNormalizationFilter(tokenizer); + return new TokenStreamComponents(tokenizer, stream); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } public void test() throws Exception { @@ -116,6 +123,7 @@ public class TestScandinavianNormalizationFilter extends BaseTokenStreamTestCase } }; checkOneTerm(a, "", ""); + a.close(); } /** blast some random strings through the analyzer */ diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java index 0ea5140af2d..ecf1492c3e4 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.miscellaneous; import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -103,6 +102,7 @@ public class TestTrimFilter extends BaseTokenStreamTestCase { } }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } public void testEmptyTerm() throws IOException { @@ -114,5 +114,6 @@ public class TestTrimFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java index b2eca7bbe71..3d7c03ec82a 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java @@ -24,6 +24,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.util.CharArraySet; +import org.apache.lucene.util.IOUtils; import org.junit.Test; import java.io.IOException; @@ -292,6 +293,7 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase { new int[] { 4, 4, 11 }, new int[] { 10, 15, 15 }, new int[] { 2, 0, 1 }); + IOUtils.close(a, a2, a3); } /** concat numbers + words + all */ @@ -312,6 +314,7 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase { new int[] { 0, 0, 0, 4, 8, 8, 12 }, new int[] { 3, 7, 15, 7, 11, 15, 15 }, new int[] { 1, 0, 0, 1, 1, 0, 1 }); + a.close(); } /** concat numbers + words + all + preserve original */ @@ -332,6 +335,7 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase { new int[] { 0, 0, 0, 0, 4, 8, 8, 12 }, new int[] { 15, 3, 7, 15, 7, 11, 15, 15 }, new int[] { 1, 0, 0, 0, 1, 1, 0, 1 }); + a.close(); } /** blast some random strings through the analyzer */ @@ -356,6 +360,7 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase { }; // TODO: properly support positionLengthAttribute checkRandomData(random(), a, 200*RANDOM_MULTIPLIER, 20, false, false); + a.close(); } } @@ -381,6 +386,7 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase { }; // TODO: properly support positionLengthAttribute checkRandomData(random(), a, 20*RANDOM_MULTIPLIER, 8192, false, false); + a.close(); } } @@ -404,6 +410,7 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase { }; // depending upon options, this thing may or may not preserve the empty term checkAnalysisConsistency(random, a, random.nextBoolean(), ""); + a.close(); } } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java index 860d17e6c66..d2e1e168954 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java @@ -35,7 +35,6 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.util.TestUtil; -import org.apache.lucene.util.TestUtil; /** * Tests {@link EdgeNGramTokenFilter} for correctness. @@ -183,6 +182,7 @@ public class EdgeNGramTokenFilterTest extends BaseTokenStreamTestCase { } }; checkRandomData(random(), a, 100*RANDOM_MULTIPLIER); + a.close(); } } @@ -197,6 +197,7 @@ public class EdgeNGramTokenFilterTest extends BaseTokenStreamTestCase { } }; checkAnalysisConsistency(random, a, random.nextBoolean(), ""); + a.close(); } public void testGraphs() throws IOException { diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java index de92df84f4a..24847f8b110 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java @@ -113,6 +113,7 @@ public class EdgeNGramTokenizerTest extends BaseTokenStreamTestCase { }; checkRandomData(random(), a, 100*RANDOM_MULTIPLIER, 20); checkRandomData(random(), a, 10*RANDOM_MULTIPLIER, 8192); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java index 85a759826d2..cd760c1c7a3 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java @@ -29,7 +29,6 @@ import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.util.TestUtil; -import org.apache.lucene.util.Version; import java.io.IOException; import java.io.StringReader; @@ -140,6 +139,7 @@ public class NGramTokenFilterTest extends BaseTokenStreamTestCase { new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, new int[] { 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 }, new int[] { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }); + analyzer.close(); } /** blast some random strings through the analyzer */ @@ -156,6 +156,7 @@ public class NGramTokenFilterTest extends BaseTokenStreamTestCase { } }; checkRandomData(random(), a, 200*RANDOM_MULTIPLIER, 20); + a.close(); } } @@ -170,6 +171,7 @@ public class NGramTokenFilterTest extends BaseTokenStreamTestCase { } }; checkAnalysisConsistency(random, a, random.nextBoolean(), ""); + a.close(); } public void testSupplementaryCharacters() throws IOException { diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java index b0d8a553d38..a148ad04958 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java @@ -123,6 +123,7 @@ public class NGramTokenizerTest extends BaseTokenStreamTestCase { }; checkRandomData(random(), a, 200*RANDOM_MULTIPLIER, 20); checkRandomData(random(), a, 10*RANDOM_MULTIPLIER, 1027); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/nl/TestDutchAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/nl/TestDutchAnalyzer.java index d4165ee3c01..dba8db0ef76 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/nl/TestDutchAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/nl/TestDutchAnalyzer.java @@ -117,6 +117,7 @@ public class TestDutchAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "opheffen", "opheff"); checkOneTerm(a, "opheffende", "opheff"); checkOneTerm(a, "opheffing", "opheff"); + a.close(); } public void testReusableTokenStream() throws Exception { @@ -125,6 +126,7 @@ public class TestDutchAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "lichamelijk", "licham"); checkOneTerm(a, "lichamelijke", "licham"); checkOneTerm(a, "lichamelijkheden", "licham"); + a.close(); } public void testExclusionTableViaCtor() throws IOException { @@ -132,10 +134,11 @@ public class TestDutchAnalyzer extends BaseTokenStreamTestCase { set.add("lichamelijk"); DutchAnalyzer a = new DutchAnalyzer( CharArraySet.EMPTY_SET, set); assertAnalyzesTo(a, "lichamelijk lichamelijke", new String[] { "lichamelijk", "licham" }); - + a.close(); + a = new DutchAnalyzer( CharArraySet.EMPTY_SET, set); assertAnalyzesTo(a, "lichamelijk lichamelijke", new String[] { "lichamelijk", "licham" }); - + a.close(); } /** @@ -145,12 +148,14 @@ public class TestDutchAnalyzer extends BaseTokenStreamTestCase { public void testStemOverrides() throws IOException { DutchAnalyzer a = new DutchAnalyzer( CharArraySet.EMPTY_SET); checkOneTerm(a, "fiets", "fiets"); + a.close(); } public void testEmptyStemDictionary() throws IOException { DutchAnalyzer a = new DutchAnalyzer( CharArraySet.EMPTY_SET, CharArraySet.EMPTY_SET, CharArrayMap.emptyMap()); checkOneTerm(a, "fiets", "fiet"); + a.close(); } /** @@ -159,15 +164,20 @@ public class TestDutchAnalyzer extends BaseTokenStreamTestCase { public void testStopwordsCasing() throws IOException { DutchAnalyzer a = new DutchAnalyzer(); assertAnalyzesTo(a, "Zelf", new String[] { }); + a.close(); } private void check(final String input, final String expected) throws Exception { - checkOneTerm(new DutchAnalyzer(), input, expected); + Analyzer analyzer = new DutchAnalyzer(); + checkOneTerm(analyzer, input, expected); + analyzer.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new DutchAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new DutchAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianAnalyzer.java index f3900f87c90..3bb04e38d90 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianAnalyzer.java @@ -27,7 +27,7 @@ public class TestNorwegianAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new NorwegianAnalyzer(); + new NorwegianAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestNorwegianAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "havnedistrikter", "havnedistrikt"); // stopword assertAnalyzesTo(a, "det", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestNorwegianAnalyzer extends BaseTokenStreamTestCase { NorwegianAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "havnedistriktene", "havnedistriktene"); checkOneTerm(a, "havnedistrikter", "havnedistrikt"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new NorwegianAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new NorwegianAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java index 23d29d40496..0dfb39111b4 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java @@ -17,9 +17,7 @@ package org.apache.lucene.analysis.no; * limitations under the License. */ -import java.io.FileInputStream; import java.io.IOException; -import java.io.Reader; import java.nio.file.Files; import java.util.Random; @@ -36,18 +34,29 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; import static org.apache.lucene.analysis.no.NorwegianLightStemmer.BOKMAAL; import static org.apache.lucene.analysis.no.NorwegianLightStemmer.NYNORSK; - /** * Simple tests for {@link NorwegianLightStemFilter} */ public class TestNorwegianLightStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new NorwegianLightStemFilter(source, BOKMAAL)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new NorwegianLightStemFilter(source, BOKMAAL)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test against a vocabulary file */ public void testVocabulary() throws IOException { @@ -64,6 +73,7 @@ public class TestNorwegianLightStemFilter extends BaseTokenStreamTestCase { } }; assertVocabulary(analyzer, Files.newInputStream(getDataPath("nn_light.txt"))); + analyzer.close(); } public void testKeyword() throws IOException { @@ -77,6 +87,7 @@ public class TestNorwegianLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "sekretæren", "sekretæren"); + a.close(); } /** blast some random strings through the analyzer */ @@ -94,5 +105,6 @@ public class TestNorwegianLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java index d0f53ece4c4..1541075ad47 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java @@ -17,9 +17,7 @@ package org.apache.lucene.analysis.no; * limitations under the License. */ -import java.io.FileInputStream; import java.io.IOException; -import java.io.Reader; import java.nio.file.Files; import java.util.Random; @@ -40,13 +38,25 @@ import static org.apache.lucene.analysis.no.NorwegianLightStemmer.NYNORSK; * Simple tests for {@link NorwegianMinimalStemFilter} */ public class TestNorwegianMinimalStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(source, BOKMAAL)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(source, BOKMAAL)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test against a Bokmål vocabulary file */ public void testVocabulary() throws IOException { @@ -63,6 +73,7 @@ public class TestNorwegianMinimalStemFilter extends BaseTokenStreamTestCase { } }; assertVocabulary(analyzer, Files.newInputStream(getDataPath("nn_minimal.txt"))); + analyzer.close(); } public void testKeyword() throws IOException { @@ -76,6 +87,7 @@ public class TestNorwegianMinimalStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "sekretæren", "sekretæren"); + a.close(); } /** blast some random strings through the analyzer */ @@ -93,5 +105,6 @@ public class TestNorwegianMinimalStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java index 69a9e8996d3..7bacf864073 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java @@ -227,6 +227,7 @@ public class TestPathHierarchyTokenizer extends BaseTokenStreamTestCase { }; // TODO: properly support positionLengthAttribute checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false); + a.close(); } /** blast some random large strings through the analyzer */ @@ -241,5 +242,6 @@ public class TestPathHierarchyTokenizer extends BaseTokenStreamTestCase { }; // TODO: properly support positionLengthAttribute checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 1027, false, false); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java index 3a41fcc7094..4b9df02b88e 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java @@ -17,14 +17,12 @@ package org.apache.lucene.analysis.path; * limitations under the License. */ -import java.io.Reader; import java.io.StringReader; import java.util.Random; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.Analyzer.TokenStreamComponents; import static org.apache.lucene.analysis.path.ReversePathHierarchyTokenizer.DEFAULT_DELIMITER; import static org.apache.lucene.analysis.path.ReversePathHierarchyTokenizer.DEFAULT_SKIP; @@ -187,6 +185,7 @@ public class TestReversePathHierarchyTokenizer extends BaseTokenStreamTestCase { }; // TODO: properly support positionLengthAttribute checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false); + a.close(); } /** blast some random large strings through the analyzer */ @@ -201,5 +200,6 @@ public class TestReversePathHierarchyTokenizer extends BaseTokenStreamTestCase { }; // TODO: properly support positionLengthAttribute checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 1027, false, false); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternCaptureGroupTokenFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternCaptureGroupTokenFilter.java index 986bf834ebc..a911d395732 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternCaptureGroupTokenFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternCaptureGroupTokenFilter.java @@ -16,7 +16,7 @@ package org.apache.lucene.analysis.pattern; * See the License for the specific language governing permissions and * limitations under the License. */ -import java.io.Reader; + import java.io.StringReader; import java.util.regex.Pattern; @@ -606,6 +606,7 @@ public class TestPatternCaptureGroupTokenFilter extends BaseTokenStreamTestCase }; checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER); + a.close(); } private void testPatterns(String input, String[] regexes, String[] tokens, diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java index 1dd88d89c41..b5cec2ffbfd 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java @@ -324,6 +324,7 @@ public class TestPatternReplaceCharFilter extends BaseTokenStreamTestCase { /* ASCII only input?: */ final boolean asciiOnly = true; checkRandomData(random, a, 250 * RANDOM_MULTIPLIER, maxInputLength, asciiOnly); + a.close(); } } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java index 463f0684016..b4d98e15368 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java @@ -27,8 +27,6 @@ import org.apache.lucene.analysis.core.KeywordTokenizer; import java.io.IOException; import java.util.regex.Pattern; -/** - */ public class TestPatternReplaceFilter extends BaseTokenStreamTestCase { public void testReplaceAll() throws Exception { @@ -92,6 +90,7 @@ public class TestPatternReplaceFilter extends BaseTokenStreamTestCase { } }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); Analyzer b = new Analyzer() { @Override @@ -102,6 +101,7 @@ public class TestPatternReplaceFilter extends BaseTokenStreamTestCase { } }; checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER); + b.close(); } public void testEmptyTerm() throws IOException { @@ -113,6 +113,7 @@ public class TestPatternReplaceFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java index 892527b146e..316fbdb5b6f 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.pattern; import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; import java.util.List; @@ -29,10 +28,8 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.Analyzer.TokenStreamComponents; import org.apache.lucene.analysis.charfilter.MappingCharFilter; import org.apache.lucene.analysis.charfilter.NormalizeCharMap; -import org.apache.lucene.analysis.path.PathHierarchyTokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; public class TestPatternTokenizer extends BaseTokenStreamTestCase @@ -137,6 +134,7 @@ public class TestPatternTokenizer extends BaseTokenStreamTestCase } }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); Analyzer b = new Analyzer() { @Override @@ -146,5 +144,6 @@ public class TestPatternTokenizer extends BaseTokenStreamTestCase } }; checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER); + b.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseAnalyzer.java index 4c5cce54950..c4230260b22 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseAnalyzer.java @@ -27,7 +27,7 @@ public class TestPortugueseAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new PortugueseAnalyzer(); + new PortugueseAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestPortugueseAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "quilométricos", "quilometric"); // stopword assertAnalyzesTo(a, "não", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestPortugueseAnalyzer extends BaseTokenStreamTestCase { PortugueseAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "quilométricas", "quilométricas"); checkOneTerm(a, "quilométricos", "quilometric"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new PortugueseAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new PortugueseAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java index b53f8d3c3c9..8e84fb9103c 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java @@ -34,13 +34,25 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Simple tests for {@link PortugueseLightStemFilter} */ public class TestPortugueseLightStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.SIMPLE, true); - return new TokenStreamComponents(source, new PortugueseLightStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.SIMPLE, true); + return new TokenStreamComponents(source, new PortugueseLightStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** * Test the example from the paper "Assessing the impact of stemming accuracy @@ -102,6 +114,7 @@ public class TestPortugueseLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "quilométricas", "quilométricas"); + a.close(); } /** blast some random strings through the analyzer */ @@ -118,5 +131,6 @@ public class TestPortugueseLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java index 3b11516de34..832df17df81 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java @@ -34,13 +34,25 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Simple tests for {@link PortugueseMinimalStemFilter} */ public class TestPortugueseMinimalStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.SIMPLE, true); - return new TokenStreamComponents(source, new PortugueseMinimalStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.SIMPLE, true); + return new TokenStreamComponents(source, new PortugueseMinimalStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** * Test the example from the paper "Assessing the impact of stemming accuracy @@ -76,6 +88,7 @@ public class TestPortugueseMinimalStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "quilométricas", "quilométricas"); + a.close(); } /** blast some random strings through the analyzer */ @@ -92,5 +105,6 @@ public class TestPortugueseMinimalStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java index 11b330d11d1..eb45b11fadc 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java @@ -34,13 +34,25 @@ import org.apache.lucene.analysis.util.CharArraySet; * Simple tests for {@link PortugueseStemFilter} */ public class TestPortugueseStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.SIMPLE, true); - return new TokenStreamComponents(source, new PortugueseStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.SIMPLE, true); + return new TokenStreamComponents(source, new PortugueseStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** * Test the example from the paper "Assessing the impact of stemming accuracy @@ -76,6 +88,7 @@ public class TestPortugueseStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "quilométricas", "quilométricas"); + a.close(); } /** blast some random strings through the analyzer */ @@ -92,5 +105,6 @@ public class TestPortugueseStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java index 675a96ce226..9b29f60c827 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java @@ -58,6 +58,7 @@ public class QueryAutoStopWordAnalyzerTest extends BaseTokenStreamTestCase { @Override public void tearDown() throws Exception { + appAnalyzer.close(); reader.close(); super.tearDown(); } @@ -70,12 +71,14 @@ public class QueryAutoStopWordAnalyzerTest extends BaseTokenStreamTestCase { protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring"); assertTokenStreamContents(protectedTokenStream, new String[]{"boring"}); + protectedAnalyzer.close(); } public void testDefaultStopwordsAllFields() throws Exception { protectedAnalyzer = new QueryAutoStopWordAnalyzer( appAnalyzer, reader); TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring"); assertTokenStreamContents(protectedTokenStream, new String[0]); // Default stop word filtering will remove boring + protectedAnalyzer.close(); } public void testStopwordsAllFieldsMaxPercentDocs() throws Exception { @@ -88,11 +91,13 @@ public class QueryAutoStopWordAnalyzerTest extends BaseTokenStreamTestCase { protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "vaguelyboring"); // A filter on terms in > half of docs should not remove vaguelyBoring assertTokenStreamContents(protectedTokenStream, new String[]{"vaguelyboring"}); + protectedAnalyzer.close(); protectedAnalyzer = new QueryAutoStopWordAnalyzer( appAnalyzer, reader, 1f / 4f); protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "vaguelyboring"); // A filter on terms in > quarter of docs should remove vaguelyBoring assertTokenStreamContents(protectedTokenStream, new String[0]); + protectedAnalyzer.close(); } public void testStopwordsPerFieldMaxPercentDocs() throws Exception { @@ -100,21 +105,25 @@ public class QueryAutoStopWordAnalyzerTest extends BaseTokenStreamTestCase { TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring"); // A filter on one Field should not affect queries on another assertTokenStreamContents(protectedTokenStream, new String[]{"boring"}); + protectedAnalyzer.close(); protectedAnalyzer = new QueryAutoStopWordAnalyzer( appAnalyzer, reader, Arrays.asList("variedField", "repetitiveField"), 1f / 2f); protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring"); // A filter on the right Field should affect queries on it assertTokenStreamContents(protectedTokenStream, new String[0]); + protectedAnalyzer.close(); } public void testStopwordsPerFieldMaxDocFreq() throws Exception { protectedAnalyzer = new QueryAutoStopWordAnalyzer( appAnalyzer, reader, Arrays.asList("repetitiveField"), 10); int numStopWords = protectedAnalyzer.getStopWords("repetitiveField").length; assertTrue("Should have identified stop words", numStopWords > 0); + protectedAnalyzer.close(); protectedAnalyzer = new QueryAutoStopWordAnalyzer( appAnalyzer, reader, Arrays.asList("repetitiveField", "variedField"), 10); int numNewStopWords = protectedAnalyzer.getStopWords("repetitiveField").length + protectedAnalyzer.getStopWords("variedField").length; assertTrue("Should have identified more stop words", numNewStopWords > numStopWords); + protectedAnalyzer.close(); } public void testNoFieldNamePollution() throws Exception { @@ -127,6 +136,7 @@ public class QueryAutoStopWordAnalyzerTest extends BaseTokenStreamTestCase { protectedTokenStream = protectedAnalyzer.tokenStream("variedField", "boring"); // Filter should not prevent stopwords in one field being used in another assertTokenStreamContents(protectedTokenStream, new String[]{"boring"}); + protectedAnalyzer.close(); } public void testTokenStream() throws Exception { @@ -134,5 +144,6 @@ public class QueryAutoStopWordAnalyzerTest extends BaseTokenStreamTestCase { new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false), reader, 10); TokenStream ts = a.tokenStream("repetitiveField", "this boring"); assertTokenStreamContents(ts, new String[] { "this" }); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java index 28f3c7c3fd2..7f70eba4c8f 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java @@ -101,6 +101,7 @@ public class TestReverseStringFilter extends BaseTokenStreamTestCase { } }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } public void testEmptyTerm() throws IOException { @@ -112,5 +113,6 @@ public class TestReverseStringFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ro/TestRomanianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ro/TestRomanianAnalyzer.java index 7af63248732..18cdb94b2b8 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ro/TestRomanianAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ro/TestRomanianAnalyzer.java @@ -27,7 +27,7 @@ public class TestRomanianAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new RomanianAnalyzer(); + new RomanianAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestRomanianAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "absenţi", "absenţ"); // stopword assertAnalyzesTo(a, "îl", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestRomanianAnalyzer extends BaseTokenStreamTestCase { RomanianAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "absenţa", "absenţa"); checkOneTerm(a, "absenţi", "absenţ"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new RomanianAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new RomanianAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java index 35dd3efc514..3435d05ec36 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java @@ -28,34 +28,38 @@ import org.apache.lucene.analysis.util.CharArraySet; */ public class TestRussianAnalyzer extends BaseTokenStreamTestCase { - - /** Check that RussianAnalyzer doesnt discard any numbers */ - public void testDigitsInRussianCharset() throws IOException - { - RussianAnalyzer ra = new RussianAnalyzer(); - assertAnalyzesTo(ra, "text 1000", new String[] { "text", "1000" }); - } - - public void testReusableTokenStream() throws Exception { - Analyzer a = new RussianAnalyzer(); - assertAnalyzesTo(a, "Вместе с тем о силе электромагнитной энергии имели представление еще", - new String[] { "вмест", "сил", "электромагнитн", "энерг", "имел", "представлен" }); - assertAnalyzesTo(a, "Но знание это хранилось в тайне", - new String[] { "знан", "эт", "хран", "тайн" }); - } - - - public void testWithStemExclusionSet() throws Exception { - CharArraySet set = new CharArraySet( 1, true); - set.add("представление"); - Analyzer a = new RussianAnalyzer( RussianAnalyzer.getDefaultStopSet() , set); - assertAnalyzesTo(a, "Вместе с тем о силе электромагнитной энергии имели представление еще", - new String[] { "вмест", "сил", "электромагнитн", "энерг", "имел", "представление" }); - - } - - /** blast some random strings through the analyzer */ - public void testRandomStrings() throws Exception { - checkRandomData(random(), new RussianAnalyzer(), 1000*RANDOM_MULTIPLIER); - } + + /** Check that RussianAnalyzer doesnt discard any numbers */ + public void testDigitsInRussianCharset() throws IOException + { + RussianAnalyzer ra = new RussianAnalyzer(); + assertAnalyzesTo(ra, "text 1000", new String[] { "text", "1000" }); + ra.close(); + } + + public void testReusableTokenStream() throws Exception { + Analyzer a = new RussianAnalyzer(); + assertAnalyzesTo(a, "Вместе с тем о силе электромагнитной энергии имели представление еще", + new String[] { "вмест", "сил", "электромагнитн", "энерг", "имел", "представлен" }); + assertAnalyzesTo(a, "Но знание это хранилось в тайне", + new String[] { "знан", "эт", "хран", "тайн" }); + a.close(); + } + + + public void testWithStemExclusionSet() throws Exception { + CharArraySet set = new CharArraySet( 1, true); + set.add("представление"); + Analyzer a = new RussianAnalyzer( RussianAnalyzer.getDefaultStopSet() , set); + assertAnalyzesTo(a, "Вместе с тем о силе электромагнитной энергии имели представление еще", + new String[] { "вмест", "сил", "электромагнитн", "энерг", "имел", "представление" }); + a.close(); + } + + /** blast some random strings through the analyzer */ + public void testRandomStrings() throws Exception { + Analyzer analyzer = new RussianAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); + } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java index cbafbee446f..ef83eb7ef89 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.ru; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -35,13 +34,25 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Simple tests for {@link RussianLightStemFilter} */ public class TestRussianLightStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new RussianLightStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new RussianLightStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test against a vocabulary from the reference impl */ public void testVocabulary() throws IOException { @@ -59,6 +70,7 @@ public class TestRussianLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "энергии", "энергии"); + a.close(); } /** blast some random strings through the analyzer */ @@ -75,5 +87,6 @@ public class TestRussianLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java index 15d498a0c9f..27ddb141494 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java @@ -83,6 +83,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { public void tearDown() throws Exception { reader.close(); directory.close(); + analyzer.close(); super.tearDown(); } @@ -156,6 +157,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 0, 7, 7, 10, 10, 13 }, new int[] { 6, 9, 9, 12, 12, 18, 18 }, new int[] { 1, 0, 1, 0, 1, 0, 1 }); + a.close(); } public void testNonDefaultMinShingleSize() throws Exception { @@ -171,6 +173,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 0, 0, 7, 7, 7, 14, 14, 14, 19, 19, 28, 33 }, new int[] { 6, 18, 27, 13, 27, 32, 18, 32, 41, 27, 41, 32, 41 }, new int[] { 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1 }); + analyzer.close(); analyzer = new ShingleAnalyzerWrapper( new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false), 3, 4, @@ -183,6 +186,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 0, 7, 7, 14, 14, 19 }, new int[] { 18, 27, 27, 32, 32, 41, 41 }, new int[] { 1, 0, 1, 0, 1, 0, 1 }); + analyzer.close(); } public void testNonDefaultMinAndSameMaxShingleSize() throws Exception { @@ -198,6 +202,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 0, 7, 7, 14, 14, 19, 19, 28, 33 }, new int[] { 6, 18, 13, 27, 18, 32, 27, 41, 32, 41 }, new int[] { 1, 0, 1, 0, 1, 0, 1, 0, 1, 1 }); + analyzer.close(); analyzer = new ShingleAnalyzerWrapper( new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false), 3, 3, @@ -210,6 +215,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 7, 14, 19 }, new int[] { 18, 27, 32, 41 }, new int[] { 1, 1, 1, 1 }); + analyzer.close(); } public void testNoTokenSeparator() throws Exception { @@ -227,6 +233,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 0, 7, 7, 14, 14, 19 }, new int[] { 6, 13, 13, 18, 18, 27, 27 }, new int[] { 1, 0, 1, 0, 1, 0, 1 }); + analyzer.close(); analyzer = new ShingleAnalyzerWrapper( new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false), @@ -241,6 +248,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 7, 14 }, new int[] { 13, 18, 27 }, new int[] { 1, 1, 1 }); + analyzer.close(); } public void testNullTokenSeparator() throws Exception { @@ -258,6 +266,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 0, 7, 7, 14, 14, 19 }, new int[] { 6, 13, 13, 18, 18, 27, 27 }, new int[] { 1, 0, 1, 0, 1, 0, 1 }); + analyzer.close(); analyzer = new ShingleAnalyzerWrapper( new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false), @@ -272,6 +281,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 7, 14 }, new int[] { 13, 18, 27 }, new int[] { 1, 1, 1 }); + analyzer.close(); } public void testAltTokenSeparator() throws Exception { @@ -289,6 +299,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 0, 7, 7, 14, 14, 19 }, new int[] { 6, 13, 13, 18, 18, 27, 27 }, new int[] { 1, 0, 1, 0, 1, 0, 1 }); + analyzer.close(); analyzer = new ShingleAnalyzerWrapper( new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false), @@ -303,6 +314,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 7, 14 }, new int[] { 13, 18, 27 }, new int[] { 1, 1, 1 }); + analyzer.close(); } public void testAltFillerToken() throws Exception { @@ -329,7 +341,17 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 0, 7, 7, 19, 19 }, new int[] { 6, 13, 13, 19, 27, 27 }, new int[] { 1, 0, 1, 0, 1, 1 }); + analyzer.close(); + delegate = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + CharArraySet stopSet = StopFilter.makeStopSet("into"); + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); + TokenFilter filter = new StopFilter(tokenizer, stopSet); + return new TokenStreamComponents(tokenizer, filter); + } + }; analyzer = new ShingleAnalyzerWrapper( delegate, ShingleFilter.DEFAULT_MIN_SHINGLE_SIZE, @@ -341,7 +363,17 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 7, 19 }, new int[] { 13, 19, 27 }, new int[] { 1, 1, 1 }); + analyzer.close(); + delegate = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + CharArraySet stopSet = StopFilter.makeStopSet("into"); + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); + TokenFilter filter = new StopFilter(tokenizer, stopSet); + return new TokenStreamComponents(tokenizer, filter); + } + }; analyzer = new ShingleAnalyzerWrapper( delegate, ShingleFilter.DEFAULT_MIN_SHINGLE_SIZE, @@ -353,6 +385,7 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0, 7, 19 }, new int[] { 13, 19, 27 }, new int[] { 1, 1, 1 }); + analyzer.close(); } public void testOutputUnigramsIfNoShinglesSingleToken() throws Exception { @@ -367,5 +400,6 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase { new int[] { 0 }, new int[] { 6 }, new int[] { 1 }); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java index 83bca7dcc81..52866f61538 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java @@ -1113,6 +1113,7 @@ public class ShingleFilterTest extends BaseTokenStreamTestCase { } }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } /** blast some random large strings through the analyzer */ @@ -1126,6 +1127,7 @@ public class ShingleFilterTest extends BaseTokenStreamTestCase { } }; checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 8192); + a.close(); } public void testEmptyTerm() throws IOException { @@ -1137,6 +1139,7 @@ public class ShingleFilterTest extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } public void testTrailingHole1() throws IOException { diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java index 03ed84bd67c..7ba4fd297d7 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java @@ -123,6 +123,7 @@ public class TestTeeSinkTokenFilter extends BaseTokenStreamTestCase { assertEquals(DocIdSetIterator.NO_MORE_DOCS, positions.nextDoc()); r.close(); dir.close(); + analyzer.close(); } public void testGeneral() throws IOException { diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java index 0acedbafedf..4b16fe914ff 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java @@ -48,6 +48,7 @@ public class TestSnowball extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "he abhorred accents", new String[]{"he", "abhor", "accent"}); + a.close(); } public void testFilterTokens() throws Exception { @@ -113,6 +114,7 @@ public class TestSnowball extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } @@ -131,5 +133,6 @@ public class TestSnowball extends BaseTokenStreamTestCase { } }; checkRandomData(random(), a, 100*RANDOM_MULTIPLIER); + a.close(); } } \ No newline at end of file diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java index 912d8fba808..f467b8d16ef 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.snowball; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Tokenizer; @@ -80,5 +79,6 @@ public class TestSnowballVocab extends LuceneTestCase { assertVocabulary(a, getDataPath("TestSnowballVocabData.zip"), dataDirectory + "/voc.txt", dataDirectory + "/output.txt"); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/sr/TestSerbianNormalizationFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/sr/TestSerbianNormalizationFilter.java index 96cbe5d6e90..7217cf4eeba 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/sr/TestSerbianNormalizationFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/sr/TestSerbianNormalizationFilter.java @@ -30,14 +30,26 @@ import org.apache.lucene.analysis.core.KeywordTokenizer; * Tests {@link SerbianNormalizationFilter} */ public class TestSerbianNormalizationFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String field) { - final Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); - final TokenStream stream = new SerbianNormalizationFilter(tokenizer); - return new TokenStreamComponents(tokenizer, stream); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + final Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); + final TokenStream stream = new SerbianNormalizationFilter(tokenizer); + return new TokenStreamComponents(tokenizer, stream); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** * Tests Cyrillic text. @@ -67,5 +79,6 @@ public class TestSerbianNormalizationFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestClassicAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestClassicAnalyzer.java index 1180007424b..5f005754782 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestClassicAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestClassicAnalyzer.java @@ -35,17 +35,29 @@ import org.apache.lucene.util.BytesRef; import java.io.IOException; import java.util.Arrays; -import java.util.Random; /** tests for classicanalyzer */ public class TestClassicAnalyzer extends BaseTokenStreamTestCase { - private Analyzer a = new ClassicAnalyzer(); + private Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new ClassicAnalyzer(); + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } public void testMaxTermLength() throws Exception { ClassicAnalyzer sa = new ClassicAnalyzer(); sa.setMaxTokenLength(5); assertAnalyzesTo(sa, "ab cd toolong xy z", new String[]{"ab", "cd", "xy", "z"}); + sa.close(); } public void testMaxTermLength2() throws Exception { @@ -54,6 +66,7 @@ public class TestClassicAnalyzer extends BaseTokenStreamTestCase { sa.setMaxTokenLength(5); assertAnalyzesTo(sa, "ab cd toolong xy z", new String[]{"ab", "cd", "xy", "z"}, new int[]{1, 1, 2, 1}); + sa.close(); } public void testMaxTermLength3() throws Exception { @@ -115,6 +128,7 @@ public class TestClassicAnalyzer extends BaseTokenStreamTestCase { try { ClassicAnalyzer analyzer = new ClassicAnalyzer(); assertAnalyzesTo(analyzer, "www.nutch.org.", new String[]{ "www.nutch.org" }, new String[] { "" }); + analyzer.close(); } catch (NullPointerException e) { fail("Should not throw an NPE and it did"); } @@ -137,8 +151,10 @@ public class TestClassicAnalyzer extends BaseTokenStreamTestCase { // 2.4 should not show the bug. But, alas, it's also obsolete, // so we check latest released (Robert's gonna break this on 4.0 soon :) ) + a2.close(); a2 = new ClassicAnalyzer(); assertAnalyzesTo(a2, "www.nutch.org.", new String[]{ "www.nutch.org" }, new String[] { "" }); + a2.close(); } public void testEMailAddresses() throws Exception { @@ -246,6 +262,7 @@ public class TestClassicAnalyzer extends BaseTokenStreamTestCase { public void testJava14BWCompatibility() throws Exception { ClassicAnalyzer sa = new ClassicAnalyzer(); assertAnalyzesTo(sa, "test\u02C6test", new String[] { "test", "test" }); + sa.close(); } /** @@ -253,7 +270,8 @@ public class TestClassicAnalyzer extends BaseTokenStreamTestCase { */ public void testWickedLongTerm() throws IOException { RAMDirectory dir = new RAMDirectory(); - IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new ClassicAnalyzer())); + Analyzer analyzer = new ClassicAnalyzer(); + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzer)); char[] chars = new char[IndexWriter.MAX_TERM_LENGTH]; Arrays.fill(chars, 'x'); @@ -309,16 +327,21 @@ public class TestClassicAnalyzer extends BaseTokenStreamTestCase { reader.close(); dir.close(); + analyzer.close(); + sa.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new ClassicAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new ClassicAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } /** blast some random large strings through the analyzer */ public void testRandomHugeStrings() throws Exception { - Random random = random(); - checkRandomData(random, new ClassicAnalyzer(), 100*RANDOM_MULTIPLIER, 8192); + Analyzer analyzer = new ClassicAnalyzer(); + checkRandomData(random(), analyzer, 100*RANDOM_MULTIPLIER, 8192); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardAnalyzer.java index 2f28ba6f691..113d94de9b8 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardAnalyzer.java @@ -114,14 +114,25 @@ public class TestStandardAnalyzer extends BaseTokenStreamTestCase { BaseTokenStreamTestCase.assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" }); } - private Analyzer a = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - - Tokenizer tokenizer = new StandardTokenizer(newAttributeFactory()); - return new TokenStreamComponents(tokenizer); - } - }; + private Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new StandardTokenizer(newAttributeFactory()); + return new TokenStreamComponents(tokenizer); + } + }; + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } public void testArmenian() throws Exception { BaseTokenStreamTestCase.assertAnalyzesTo(a, "Վիքիպեդիայի 13 միլիոն հոդվածները (4,600` հայերեն վիքիպեդիայում) գրվել են կամավորների կողմից ու համարյա բոլոր հոդվածները կարող է խմբագրել ցանկաց մարդ ով կարող է բացել Վիքիպեդիայի կայքը։", @@ -350,27 +361,30 @@ public class TestStandardAnalyzer extends BaseTokenStreamTestCase { /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new StandardAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new StandardAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } /** blast some random large strings through the analyzer */ public void testRandomHugeStrings() throws Exception { - Random random = random(); - checkRandomData(random, new StandardAnalyzer(), 100*RANDOM_MULTIPLIER, 8192); + Analyzer analyzer = new StandardAnalyzer(); + checkRandomData(random(), analyzer, 100*RANDOM_MULTIPLIER, 8192); + analyzer.close(); } // Adds random graph after: public void testRandomHugeStringsGraphAfter() throws Exception { Random random = random(); - checkRandomData(random, - new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new StandardTokenizer(newAttributeFactory()); - TokenStream tokenStream = new MockGraphTokenFilter(random(), tokenizer); - return new TokenStreamComponents(tokenizer, tokenStream); - } - }, - 100*RANDOM_MULTIPLIER, 8192); + Analyzer analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new StandardTokenizer(newAttributeFactory()); + TokenStream tokenStream = new MockGraphTokenFilter(random(), tokenizer); + return new TokenStreamComponents(tokenizer, tokenStream); + } + }; + checkRandomData(random, analyzer, 100*RANDOM_MULTIPLIER, 8192); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailAnalyzer.java index 4b4da03c70e..ea798a367df 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailAnalyzer.java @@ -26,7 +26,19 @@ import java.util.Arrays; public class TestUAX29URLEmailAnalyzer extends BaseTokenStreamTestCase { - private Analyzer a = new UAX29URLEmailAnalyzer(); + private Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new UAX29URLEmailAnalyzer(); + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } public void testHugeDoc() throws IOException { StringBuilder sb = new StringBuilder(); @@ -343,6 +355,6 @@ public class TestUAX29URLEmailAnalyzer extends BaseTokenStreamTestCase { /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new UAX29URLEmailAnalyzer(), 1000*RANDOM_MULTIPLIER); + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizer.java index 839f24d6f07..24f2c1a9bb8 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizer.java @@ -8,6 +8,7 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer; import org.apache.lucene.analysis.standard.WordBreakTestUnicode_6_3_0; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.TestUtil; import java.io.BufferedReader; @@ -88,15 +89,42 @@ public class TestUAX29URLEmailTokenizer extends BaseTokenStreamTestCase { BaseTokenStreamTestCase.assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" }); } - private Analyzer a = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - - Tokenizer tokenizer = new UAX29URLEmailTokenizer(newAttributeFactory()); - return new TokenStreamComponents(tokenizer); - } - }; - + private Analyzer a, urlAnalyzer, emailAnalyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new UAX29URLEmailTokenizer(newAttributeFactory()); + return new TokenStreamComponents(tokenizer); + } + }; + urlAnalyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(newAttributeFactory()); + tokenizer.setMaxTokenLength(Integer.MAX_VALUE); // Tokenize arbitrary length URLs + TokenFilter filter = new URLFilter(tokenizer); + return new TokenStreamComponents(tokenizer, filter); + } + }; + emailAnalyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(newAttributeFactory()); + TokenFilter filter = new EmailFilter(tokenizer); + return new TokenStreamComponents(tokenizer, filter); + } + }; + } + + @Override + public void tearDown() throws Exception { + IOUtils.close(a, urlAnalyzer, emailAnalyzer); + super.tearDown(); + } /** Passes through tokens with type "" and blocks all other types. */ private class URLFilter extends TokenFilter { @@ -134,27 +162,7 @@ public class TestUAX29URLEmailTokenizer extends BaseTokenStreamTestCase { } return isTokenAvailable; } - } - - private Analyzer urlAnalyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(newAttributeFactory()); - tokenizer.setMaxTokenLength(Integer.MAX_VALUE); // Tokenize arbitrary length URLs - TokenFilter filter = new URLFilter(tokenizer); - return new TokenStreamComponents(tokenizer, filter); - } - }; - - private Analyzer emailAnalyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(newAttributeFactory()); - TokenFilter filter = new EmailFilter(tokenizer); - return new TokenStreamComponents(tokenizer, filter); - } - }; - + } public void testArmenian() throws Exception { BaseTokenStreamTestCase.assertAnalyzesTo(a, "Վիքիպեդիայի 13 միլիոն հոդվածները (4,600` հայերեն վիքիպեդիայում) գրվել են կամավորների կողմից ու համարյա բոլոր հոդվածները կարող է խմբագրել ցանկաց մարդ ով կարող է բացել Վիքիպեդիայի կայքը։", diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishAnalyzer.java index 6220f9ce5f9..0fd8a9aa250 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishAnalyzer.java @@ -27,7 +27,7 @@ public class TestSwedishAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new SwedishAnalyzer(); + new SwedishAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestSwedishAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "jaktkarlens", "jaktkarl"); // stopword assertAnalyzesTo(a, "och", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -47,10 +48,13 @@ public class TestSwedishAnalyzer extends BaseTokenStreamTestCase { SwedishAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "jaktkarlarne", "jaktkarlarne"); checkOneTerm(a, "jaktkarlens", "jaktkarl"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new SwedishAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new SwedishAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java index 09ab5258956..12db7623835 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.sv; */ import java.io.IOException; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -35,13 +34,25 @@ import static org.apache.lucene.analysis.VocabularyAssert.*; * Simple tests for {@link SwedishLightStemFilter} */ public class TestSwedishLightStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new SwedishLightStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new SwedishLightStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test against a vocabulary from the reference impl */ public void testVocabulary() throws IOException { @@ -59,6 +70,7 @@ public class TestSwedishLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "jaktkarlens", "jaktkarlens"); + a.close(); } /** blast some random strings through the analyzer */ @@ -75,5 +87,6 @@ public class TestSwedishLightStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java index 36897b7c5bd..66d0aad2124 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java @@ -17,7 +17,6 @@ package org.apache.lucene.analysis.synonym; * limitations under the License. */ -import java.io.Reader; import java.io.StringReader; import java.text.ParseException; @@ -27,7 +26,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.en.EnglishAnalyzer; -import org.junit.Test; /** * Tests parser for the Solr synonyms format @@ -43,11 +41,13 @@ public class TestSolrSynonymParser extends BaseTokenStreamTestCase { "foo => baz\n" + "this test, that testing"; - SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer); parser.parse(new StringReader(testFile)); final SynonymMap map = parser.build(); + analyzer.close(); - Analyzer analyzer = new Analyzer() { + analyzer = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true); @@ -70,46 +70,77 @@ public class TestSolrSynonymParser extends BaseTokenStreamTestCase { assertAnalyzesTo(analyzer, "this test", new String[] { "this", "that", "test", "testing" }, new int[] { 1, 0, 1, 0 }); + analyzer.close(); } /** parse a syn file with bad syntax */ - @Test(expected=ParseException.class) public void testInvalidDoubleMap() throws Exception { - String testFile = "a => b => c"; - SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random())); - parser.parse(new StringReader(testFile)); + String testFile = "a => b => c"; + Analyzer analyzer = new MockAnalyzer(random()); + SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer); + try { + parser.parse(new StringReader(testFile)); + fail("didn't get expected exception"); + } catch (ParseException expected) { + // expected exc + } + analyzer.close(); } /** parse a syn file with bad syntax */ - @Test(expected=ParseException.class) public void testInvalidAnalyzesToNothingOutput() throws Exception { String testFile = "a => 1"; - SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random(), MockTokenizer.SIMPLE, false)); - parser.parse(new StringReader(testFile)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, false); + SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer); + try { + parser.parse(new StringReader(testFile)); + fail("didn't get expected exception"); + } catch (ParseException expected) { + // expected exc + } + analyzer.close(); } /** parse a syn file with bad syntax */ - @Test(expected=ParseException.class) public void testInvalidAnalyzesToNothingInput() throws Exception { - String testFile = "1 => a"; - SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random(), MockTokenizer.SIMPLE, false)); - parser.parse(new StringReader(testFile)); + String testFile = "1 => a"; + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, false); + SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer); + try { + parser.parse(new StringReader(testFile)); + fail("didn't get expected exception"); + } catch (ParseException expected) { + // expected exc + } + analyzer.close(); } /** parse a syn file with bad syntax */ - @Test(expected=ParseException.class) public void testInvalidPositionsInput() throws Exception { String testFile = "testola => the test"; - SolrSynonymParser parser = new SolrSynonymParser(true, true, new EnglishAnalyzer()); - parser.parse(new StringReader(testFile)); + Analyzer analyzer = new EnglishAnalyzer(); + SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer); + try { + parser.parse(new StringReader(testFile)); + fail("didn't get expected exception"); + } catch (ParseException expected) { + // expected exc + } + analyzer.close(); } /** parse a syn file with bad syntax */ - @Test(expected=ParseException.class) public void testInvalidPositionsOutput() throws Exception { String testFile = "the test => testola"; - SolrSynonymParser parser = new SolrSynonymParser(true, true, new EnglishAnalyzer()); - parser.parse(new StringReader(testFile)); + Analyzer analyzer = new EnglishAnalyzer(); + SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer); + try { + parser.parse(new StringReader(testFile)); + fail("didn't get expected exception"); + } catch (ParseException expected) { + // expected exc + } + analyzer.close(); } /** parse a syn file with some escaped syntax chars */ @@ -117,10 +148,12 @@ public class TestSolrSynonymParser extends BaseTokenStreamTestCase { String testFile = "a\\=>a => b\\=>b\n" + "a\\,a => b\\,b"; - SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false); + SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer); parser.parse(new StringReader(testFile)); final SynonymMap map = parser.build(); - Analyzer analyzer = new Analyzer() { + analyzer.close(); + analyzer = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false); @@ -139,5 +172,6 @@ public class TestSolrSynonymParser extends BaseTokenStreamTestCase { assertAnalyzesTo(analyzer, "a,a", new String[] { "b,b" }, new int[] { 1 }); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java index b8b95725bcd..9b2b69dfa07 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java @@ -37,7 +37,6 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.MockGraphTokenFilter; import org.apache.lucene.analysis.core.KeywordTokenizer; import org.apache.lucene.analysis.tokenattributes.*; -import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.CharsRefBuilder; import org.apache.lucene.util.TestUtil; @@ -166,6 +165,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { new int[] {1, 1}, true); checkAnalysisConsistency(random(), analyzer, false, "a b c"); + analyzer.close(); } public void testDoKeepOrig() throws Exception { @@ -191,6 +191,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { new int[] {1, 2, 1, 1}, true); checkAnalysisConsistency(random(), analyzer, false, "a b c"); + analyzer.close(); } public void testBasic() throws Exception { @@ -502,6 +503,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { }; checkRandomData(random(), analyzer, 100); + analyzer.close(); } } @@ -560,6 +562,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { }; checkRandomData(random, analyzer, 100); + analyzer.close(); } } @@ -584,6 +587,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { }; checkAnalysisConsistency(random, analyzer, random.nextBoolean(), ""); + analyzer.close(); } } @@ -613,6 +617,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { }; checkRandomData(random, analyzer, 100, 1024); + analyzer.close(); } } @@ -621,10 +626,11 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { String testFile = "aaa => aaaa1 aaaa2 aaaa3\n" + "bbb => bbbb1 bbbb2\n"; - - SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random())); + Analyzer synAnalyzer = new MockAnalyzer(random()); + SolrSynonymParser parser = new SolrSynonymParser(true, true, synAnalyzer); parser.parse(new StringReader(testFile)); final SynonymMap map = parser.build(); + synAnalyzer.close(); Analyzer analyzer = new Analyzer() { @Override @@ -642,6 +648,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { // xyzzy aaa pot of gold -> xyzzy aaaa1 aaaa2 aaaa3 gold assertAnalyzesTo(analyzer, "xyzzy aaa pot of gold", new String[] { "xyzzy", "aaaa1", "pot", "aaaa2", "of", "aaaa3", "gold" }); + analyzer.close(); } public void testBasic2() throws Exception { @@ -716,6 +723,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "z x c $", new String[] { "z", "xc", "$" }, new int[] { 1, 1, 1 }); + a.close(); } public void testRepeatsOff() throws Exception { @@ -736,6 +744,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "a b", new String[] { "ab" }, new int[] { 1 }); + a.close(); } public void testRepeatsOn() throws Exception { @@ -756,6 +765,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "a b", new String[] { "ab", "ab", "ab" }, new int[] { 1, 0, 0 }); + a.close(); } public void testRecursion() throws Exception { @@ -774,6 +784,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "zoo zoo $ zoo", new String[] { "zoo", "zoo", "$", "zoo" }, new int[] { 1, 1, 1, 1 }); + a.close(); } public void testRecursion2() throws Exception { @@ -794,6 +805,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "zoo zoo $ zoo", new String[] { "zoo", "zoo", "zoo", "zoo", "zoo", "$", "zoo", "zoo", "zoo", "zoo" }, new int[] { 1, 0, 1, 0, 0, 1, 0, 1, 0, 1 }); + a.close(); } public void testOutputHangsOffEnd() throws Exception { @@ -869,6 +881,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "z x c $", new String[] { "z", "x", "xc", "c", "$" }, new int[] { 1, 1, 0, 1, 1 }); + a.close(); } public void testRecursion3() throws Exception { @@ -887,6 +900,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "zoo zoo $ zoo", new String[] { "zoo", "zoo", "zoo", "$", "zoo" }, new int[] { 1, 0, 1, 1, 1 }); + a.close(); } public void testRecursion4() throws Exception { @@ -906,6 +920,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "zoo zoo $ zoo", new String[] { "zoo", "zoo", "zoo", "$", "zoo", "zoo", "zoo" }, new int[] { 1, 0, 1, 1, 1, 0, 1 }); + a.close(); } public void testMultiwordOffsets() throws Exception { @@ -926,6 +941,7 @@ public class TestSynonymMapFilter extends BaseTokenStreamTestCase { new int[] { 0, 0, 9, 16 }, new int[] { 8, 22, 15, 22 }, new int[] { 1, 0, 1, 1 }); + a.close(); } public void testEmpty() throws Exception { diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java index 03417282750..d70e849cfaf 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java @@ -17,7 +17,6 @@ package org.apache.lucene.analysis.synonym; -import java.io.Reader; import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; @@ -40,11 +39,13 @@ public class TestWordnetSynonymParser extends BaseTokenStreamTestCase { "s(100000004,2,'king''s meany',n,1,1).\n"; public void testSynonyms() throws Exception { - WordnetSynonymParser parser = new WordnetSynonymParser(true, true, new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + WordnetSynonymParser parser = new WordnetSynonymParser(true, true, analyzer); parser.parse(new StringReader(synonymsFile)); final SynonymMap map = parser.build(); + analyzer.close(); - Analyzer analyzer = new Analyzer() { + analyzer = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); @@ -66,5 +67,6 @@ public class TestWordnetSynonymParser extends BaseTokenStreamTestCase { /* multi words */ assertAnalyzesTo(analyzer, "king's evil", new String[] { "king's", "king's", "evil", "meany" }); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java index ab502f00cba..6cd488ed57a 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java @@ -17,15 +17,9 @@ package org.apache.lucene.analysis.th; * limitations under the License. */ -import java.io.IOException; -import java.io.Reader; -import java.util.Random; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.core.KeywordTokenizer; import org.apache.lucene.analysis.core.StopAnalyzer; import org.apache.lucene.analysis.tokenattributes.FlagsAttribute; import org.apache.lucene.analysis.util.CharArraySet; @@ -46,18 +40,22 @@ public class TestThaiAnalyzer extends BaseTokenStreamTestCase { * testcase for offsets */ public void testOffsets() throws Exception { - assertAnalyzesTo(new ThaiAnalyzer(CharArraySet.EMPTY_SET), "การที่ได้ต้องแสดงว่างานดี", + Analyzer analyzer = new ThaiAnalyzer(CharArraySet.EMPTY_SET); + assertAnalyzesTo(analyzer, "การที่ได้ต้องแสดงว่างานดี", new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี" }, new int[] { 0, 3, 6, 9, 13, 17, 20, 23 }, new int[] { 3, 6, 9, 13, 17, 20, 23, 25 }); + analyzer.close(); } public void testStopWords() throws Exception { - assertAnalyzesTo(new ThaiAnalyzer(), "การที่ได้ต้องแสดงว่างานดี", + Analyzer analyzer = new ThaiAnalyzer(); + assertAnalyzesTo(analyzer, "การที่ได้ต้องแสดงว่างานดี", new String[] { "แสดง", "งาน", "ดี" }, new int[] { 13, 20, 23 }, new int[] { 17, 23, 25 }, new int[] { 5, 2, 1 }); + analyzer.close(); } /* @@ -78,32 +76,37 @@ public class TestThaiAnalyzer extends BaseTokenStreamTestCase { new int[] { 0, 3, 6, 9, 17, 21, 24, 27 }, new int[] { 3, 6, 9, 13, 21, 24, 27, 29 }, new int[] { 1, 1, 1, 1, 2, 1, 1, 1 }); + analyzer.close(); } public void testReusableTokenStream() throws Exception { ThaiAnalyzer analyzer = new ThaiAnalyzer(CharArraySet.EMPTY_SET); assertAnalyzesTo(analyzer, "", new String[] {}); - - assertAnalyzesTo( - analyzer, - "การที่ได้ต้องแสดงว่างานดี", - new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี"}); - - assertAnalyzesTo( - analyzer, - "บริษัทชื่อ XY&Z - คุยกับ xyz@demo.com", - new String[] { "บริษัท", "ชื่อ", "xy", "z", "คุย", "กับ", "xyz", "demo.com" }); + + assertAnalyzesTo( + analyzer, + "การที่ได้ต้องแสดงว่างานดี", + new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี"}); + + assertAnalyzesTo( + analyzer, + "บริษัทชื่อ XY&Z - คุยกับ xyz@demo.com", + new String[] { "บริษัท", "ชื่อ", "xy", "z", "คุย", "กับ", "xyz", "demo.com" }); + analyzer.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new ThaiAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new ThaiAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } /** blast some random large strings through the analyzer */ public void testRandomHugeStrings() throws Exception { - Random random = random(); - checkRandomData(random, new ThaiAnalyzer(), 100*RANDOM_MULTIPLIER, 8192); + Analyzer analyzer = new ThaiAnalyzer(); + checkRandomData(random(), analyzer, 100*RANDOM_MULTIPLIER, 8192); + analyzer.close(); } // LUCENE-3044 @@ -116,12 +119,15 @@ public class TestThaiAnalyzer extends BaseTokenStreamTestCase { ts = analyzer.tokenStream("dummy", "ภาษาไทย"); ts.addAttribute(FlagsAttribute.class); assertTokenStreamContents(ts, new String[] { "ภาษา", "ไทย" }); + analyzer.close(); } public void testTwoSentences() throws Exception { - assertAnalyzesTo(new ThaiAnalyzer(CharArraySet.EMPTY_SET), "This is a test. การที่ได้ต้องแสดงว่างานดี", + Analyzer analyzer = new ThaiAnalyzer(CharArraySet.EMPTY_SET); + assertAnalyzesTo(analyzer, "This is a test. การที่ได้ต้องแสดงว่างานดี", new String[] { "this", "is", "a", "test", "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี" }, new int[] { 0, 5, 8, 10, 16, 19, 22, 25, 29, 33, 36, 39 }, new int[] { 4, 7, 9, 14, 19, 22, 25, 29, 33, 36, 39, 41 }); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java index bc40ed5bad8..3577ea067c9 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java @@ -27,7 +27,7 @@ public class TestTurkishAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new TurkishAnalyzer(); + new TurkishAnalyzer().close(); } /** test stopwords and stemming */ @@ -41,6 +41,7 @@ public class TestTurkishAnalyzer extends BaseTokenStreamTestCase { // apostrophes checkOneTerm(a, "Kıbrıs'ta", "kıbrıs"); assertAnalyzesTo(a, "Van Gölü'ne", new String[]{"van", "göl"}); + a.close(); } /** test use of exclusion set */ @@ -49,10 +50,13 @@ public class TestTurkishAnalyzer extends BaseTokenStreamTestCase { Analyzer a = new TurkishAnalyzer(TurkishAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "ağacı", "ağacı"); checkOneTerm(a, "ağaç", "ağaç"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new TurkishAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new TurkishAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java index 9be5d79caf9..0ca34211d78 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java @@ -18,12 +18,9 @@ package org.apache.lucene.analysis.tr; */ import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; @@ -80,5 +77,6 @@ public class TestTurkishLowerCaseFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java index 0ed68dab40b..a9c7f49fecb 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java @@ -141,6 +141,7 @@ public class TestCharTokenizers extends BaseTokenStreamTestCase { } // just for fun checkRandomData(random(), analyzer, num); + analyzer.close(); } // LUCENE-3642: normalize BMP->SMP and check that offsets are correct @@ -179,5 +180,6 @@ public class TestCharTokenizers extends BaseTokenStreamTestCase { } // just for fun checkRandomData(random(), analyzer, num); + analyzer.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java index 61822839211..2ff55655d09 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.util; */ import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; import java.util.List; @@ -71,6 +70,7 @@ public class TestElision extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java index 699f4eb4205..3a4cc3292dd 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java @@ -27,22 +27,35 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.util.IOUtils; /** Basic tests for {@link SegmentingTokenizerBase} */ public class TestSegmentingTokenizerBase extends BaseTokenStreamTestCase { - private Analyzer sentence = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - return new TokenStreamComponents(new WholeSentenceTokenizer()); - } - }; + private Analyzer sentence, sentenceAndWord; - private Analyzer sentenceAndWord = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - return new TokenStreamComponents(new SentenceAndWordTokenizer()); - } - }; + @Override + public void setUp() throws Exception { + super.setUp(); + sentence = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + return new TokenStreamComponents(new WholeSentenceTokenizer()); + } + }; + sentenceAndWord = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + return new TokenStreamComponents(new SentenceAndWordTokenizer()); + } + }; + } + + @Override + public void tearDown() throws Exception { + IOUtils.close(sentence, sentenceAndWord); + super.tearDown(); + } + /** Some simple examples, just outputting the whole sentence boundaries as "terms" */ public void testBasics() throws IOException { diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java index 40ec291ab8a..039311d16ba 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java @@ -194,6 +194,7 @@ public class WikipediaTokenizerTest extends BaseTokenStreamTestCase { }; // TODO: properly support positionLengthAttribute checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false); + a.close(); } /** blast some random large strings through the analyzer */ @@ -209,5 +210,6 @@ public class WikipediaTokenizerTest extends BaseTokenStreamTestCase { }; // TODO: properly support positionLengthAttribute checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 8192, false, false); + a.close(); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java b/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java index c0167958314..10021fcaca5 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java @@ -27,7 +27,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; diff --git a/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java index 13ec32bfa67..6d626c0e280 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java @@ -17,7 +17,6 @@ package org.apache.lucene.collation; * limitations under the License. */ - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CollationTestBase; import org.apache.lucene.util.BytesRef; @@ -25,17 +24,24 @@ import org.apache.lucene.util.BytesRef; import java.text.Collator; import java.util.Locale; -public class TestCollationKeyAnalyzer extends CollationTestBase { - // the sort order of Ø versus U depends on the version of the rules being used - // for the inherited root locale: Ø's order isnt specified in Locale.US since - // it's not used in english. - private boolean oStrokeFirst = Collator.getInstance(new Locale("")).compare("Ø", "U") < 0; - +public class TestCollationKeyAnalyzer extends CollationTestBase { // Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in // RuleBasedCollator. However, the Arabic Locale seems to order the Farsi // characters properly. private Collator collator = Collator.getInstance(new Locale("ar")); - private Analyzer analyzer = new CollationKeyAnalyzer(collator); + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new CollationKeyAnalyzer(collator); + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } private BytesRef firstRangeBeginning = new BytesRef(collator.getCollationKey(firstRangeBeginningOriginal).toByteArray()); private BytesRef firstRangeEnd = new BytesRef(collator.getCollationKey(firstRangeEndOriginal).toByteArray()); @@ -65,7 +71,9 @@ public class TestCollationKeyAnalyzer extends CollationTestBase { for (int i = 0; i < iters; i++) { Collator collator = Collator.getInstance(Locale.GERMAN); collator.setStrength(Collator.PRIMARY); - assertThreadSafe(new CollationKeyAnalyzer(collator)); + Analyzer analyzer = new CollationKeyAnalyzer(collator); + assertThreadSafe(analyzer); + analyzer.close(); } } } diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java index b034817b79d..259895c69f0 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java @@ -18,22 +18,37 @@ package org.apache.lucene.analysis.icu; */ import java.io.IOException; -import java.io.Reader; -import org.apache.lucene.analysis.*; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; /** * Tests ICUFoldingFilter */ public class TestICUFoldingFilter extends BaseTokenStreamTestCase { - Analyzer a = new Analyzer() { - @Override - public TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(tokenizer, new ICUFoldingFilter(tokenizer)); - } - }; + Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new Analyzer() { + @Override + public TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(tokenizer, new ICUFoldingFilter(tokenizer)); + } + }; + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } + public void testDefaults() throws IOException { // case folding assertAnalyzesTo(a, "This is a test", new String[] { "this", "is", "a", "test" }); @@ -88,5 +103,6 @@ public class TestICUFoldingFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java index 88bc689ae79..408177a46d5 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java @@ -129,6 +129,7 @@ public class TestICUNormalizer2CharFilter extends BaseTokenStreamTestCase { } checkOneTerm(a, input, normalized); } + a.close(); } public void testNFC() throws Exception { @@ -187,6 +188,7 @@ public class TestICUNormalizer2CharFilter extends BaseTokenStreamTestCase { checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); // huge strings checkRandomData(random(), a, 25*RANDOM_MULTIPLIER, 8192); + a.close(); // nfkd a = new Analyzer() { @@ -203,6 +205,7 @@ public class TestICUNormalizer2CharFilter extends BaseTokenStreamTestCase { checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); // huge strings checkRandomData(random(), a, 25*RANDOM_MULTIPLIER, 8192); + a.close(); } public void testCuriousString() throws Exception { @@ -221,6 +224,7 @@ public class TestICUNormalizer2CharFilter extends BaseTokenStreamTestCase { for (int i = 0; i < 1000; i++) { checkAnalysisConsistency(random(), a, false, text); } + a.close(); } public void testCuriousMassiveString() throws Exception { @@ -411,5 +415,6 @@ public class TestICUNormalizer2CharFilter extends BaseTokenStreamTestCase { for (int i = 0; i < 25; i++) { checkAnalysisConsistency(random(), a, false, text); } + a.close(); } } diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java index d4d57fe4f21..ac0b59e352a 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java @@ -18,9 +18,11 @@ package org.apache.lucene.analysis.icu; */ import java.io.IOException; -import java.io.Reader; -import org.apache.lucene.analysis.*; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; import com.ibm.icu.text.Normalizer2; @@ -29,13 +31,25 @@ import com.ibm.icu.text.Normalizer2; * Tests the ICUNormalizer2Filter */ public class TestICUNormalizer2Filter extends BaseTokenStreamTestCase { - Analyzer a = new Analyzer() { - @Override - public TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(tokenizer)); - } - }; + Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new Analyzer() { + @Override + public TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(tokenizer)); + } + }; + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } public void testDefaults() throws IOException { // case folding @@ -72,6 +86,7 @@ public class TestICUNormalizer2Filter extends BaseTokenStreamTestCase { // decompose EAcute into E + combining Acute assertAnalyzesTo(a, "\u00E9", new String[] { "\u0065\u0301" }); + a.close(); } /** blast some random strings through the analyzer */ @@ -88,5 +103,6 @@ public class TestICUNormalizer2Filter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java index d9fa22ed244..6fe935388b5 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.icu; */ import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; @@ -105,6 +104,7 @@ public class TestICUTransformFilter extends BaseTokenStreamTestCase { } }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); } public void testEmptyTerm() throws IOException { @@ -116,5 +116,6 @@ public class TestICUTransformFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java index 13085a8fed9..ca742ae1f27 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java @@ -67,14 +67,26 @@ public class TestICUTokenizer extends BaseTokenStreamTestCase { assertTokenStreamContents(tokenizer, expected); } - private Analyzer a = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false)); - TokenFilter filter = new ICUNormalizer2Filter(tokenizer); - return new TokenStreamComponents(tokenizer, filter); - } - }; + private Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false)); + TokenFilter filter = new ICUNormalizer2Filter(tokenizer); + return new TokenStreamComponents(tokenizer, filter); + } + }; + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } public void testArmenian() throws Exception { assertAnalyzesTo(a, "Վիքիպեդիայի 13 միլիոն հոդվածները (4,600` հայերեն վիքիպեդիայում) գրվել են կամավորների կողմից ու համարյա բոլոր հոդվածները կարող է խմբագրել ցանկաց մարդ ով կարող է բացել Վիքիպեդիայի կայքը։", diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java index ee806334176..2791a47ec07 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java @@ -17,7 +17,6 @@ package org.apache.lucene.analysis.icu.segmentation; * limitations under the License. */ -import java.io.Reader; import java.util.Random; import org.apache.lucene.analysis.Analyzer; @@ -27,12 +26,24 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; * test ICUTokenizer with dictionary-based CJ segmentation */ public class TestICUTokenizerCJK extends BaseTokenStreamTestCase { - Analyzer a = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - return new TokenStreamComponents(new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(true))); - } - }; + Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + return new TokenStreamComponents(new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(true))); + } + }; + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } /** * test stolen from smartcn diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java index a4e5c3c3847..6d58a5fd7eb 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java @@ -27,6 +27,7 @@ import org.apache.lucene.analysis.cjk.CJKBigramFilter; import org.apache.lucene.analysis.core.StopFilter; import org.apache.lucene.analysis.icu.ICUNormalizer2Filter; import org.apache.lucene.analysis.util.CharArraySet; +import org.apache.lucene.util.IOUtils; /** * Tests ICUTokenizer's ability to work with CJKBigramFilter. @@ -34,35 +35,46 @@ import org.apache.lucene.analysis.util.CharArraySet; */ public class TestWithCJKBigramFilter extends BaseTokenStreamTestCase { - /** - * ICUTokenizer+CJKBigramFilter - */ - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false)); - TokenStream result = new CJKBigramFilter(source); - return new TokenStreamComponents(source, new StopFilter(result, CharArraySet.EMPTY_SET)); - } - }; + Analyzer analyzer, analyzer2; - /** - * ICUTokenizer+ICUNormalizer2Filter+CJKBigramFilter. - * - * ICUNormalizer2Filter uses nfkc_casefold by default, so this is a language-independent - * superset of CJKWidthFilter's foldings. - */ - private Analyzer analyzer2 = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false)); - // we put this before the CJKBigramFilter, because the normalization might combine - // some halfwidth katakana forms, which will affect the bigramming. - TokenStream result = new ICUNormalizer2Filter(source); - result = new CJKBigramFilter(source); - return new TokenStreamComponents(source, new StopFilter(result, CharArraySet.EMPTY_SET)); - } - }; + @Override + public void setUp() throws Exception { + super.setUp(); + /* + * ICUTokenizer+CJKBigramFilter + */ + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false)); + TokenStream result = new CJKBigramFilter(source); + return new TokenStreamComponents(source, new StopFilter(result, CharArraySet.EMPTY_SET)); + } + }; + /* + * ICUTokenizer+ICUNormalizer2Filter+CJKBigramFilter. + * + * ICUNormalizer2Filter uses nfkc_casefold by default, so this is a language-independent + * superset of CJKWidthFilter's foldings. + */ + analyzer2 = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false)); + // we put this before the CJKBigramFilter, because the normalization might combine + // some halfwidth katakana forms, which will affect the bigramming. + TokenStream result = new ICUNormalizer2Filter(source); + result = new CJKBigramFilter(result); + return new TokenStreamComponents(source, new StopFilter(result, CharArraySet.EMPTY_SET)); + } + }; + } + + @Override + public void tearDown() throws Exception { + IOUtils.close(analyzer, analyzer2); + super.tearDown(); + } public void testJa1() throws IOException { assertAnalyzesTo(analyzer, "一二三四五六七八九十", diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java b/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java index 1b64b022925..532f03ff93a 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java @@ -29,7 +29,19 @@ import java.util.Locale; public class TestICUCollationKeyAnalyzer extends CollationTestBase { private Collator collator = Collator.getInstance(new Locale("fa")); - private Analyzer analyzer = new ICUCollationKeyAnalyzer(collator); + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new ICUCollationKeyAnalyzer(collator); + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } private BytesRef firstRangeBeginning = new BytesRef (collator.getCollationKey(firstRangeBeginningOriginal).toByteArray()); @@ -62,7 +74,9 @@ public class TestICUCollationKeyAnalyzer extends CollationTestBase { Locale locale = Locale.GERMAN; Collator collator = Collator.getInstance(locale); collator.setStrength(Collator.IDENTICAL); - assertThreadSafe(new ICUCollationKeyAnalyzer(collator)); + Analyzer a = new ICUCollationKeyAnalyzer(collator); + assertThreadSafe(a); + a.close(); } } } diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java index 931e5ffd673..2bbbf52eb44 100644 --- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java +++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java @@ -30,14 +30,25 @@ import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.UnicodeUtil; public class TestExtendedMode extends BaseTokenStreamTestCase { - private final Analyzer analyzer = new Analyzer() { - - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, Mode.EXTENDED); - return new TokenStreamComponents(tokenizer, tokenizer); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, Mode.EXTENDED); + return new TokenStreamComponents(tokenizer, tokenizer); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** simple test for supplementary characters */ public void testSurrogates() throws IOException { diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java index 5859f3b4dc8..b31e4b00eef 100644 --- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java +++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java @@ -31,7 +31,7 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new JapaneseAnalyzer(); + new JapaneseAnalyzer().close(); } /** @@ -40,12 +40,14 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { * and offsets are correct. */ public void testBasics() throws IOException { - assertAnalyzesTo(new JapaneseAnalyzer(), "多くの学生が試験に落ちた。", + Analyzer a = new JapaneseAnalyzer(); + assertAnalyzesTo(a, "多くの学生が試験に落ちた。", new String[] { "多く", "学生", "試験", "落ちる" }, new int[] { 0, 3, 6, 9 }, new int[] { 2, 5, 8, 11 }, new int[] { 1, 2, 2, 2 } ); + a.close(); } /** @@ -53,7 +55,7 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { */ public void testDecomposition() throws IOException { - final Analyzer a = new JapaneseAnalyzer(null, Mode.SEARCH, + Analyzer a = new JapaneseAnalyzer(null, Mode.SEARCH, JapaneseAnalyzer.getDefaultStopSet(), JapaneseAnalyzer.getDefaultStopTags()); @@ -108,7 +110,9 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { ); // Kyoto University Baseball Club - assertAnalyzesToPositions(new JapaneseAnalyzer(), "京都大学硬式野球部", + a.close(); + a = new JapaneseAnalyzer(); + assertAnalyzesToPositions(a, "京都大学硬式野球部", new String[] { "京都大", "学", "硬式", @@ -117,6 +121,7 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { new int[] {1, 1, 1, 1, 1}, new int[] {1, 1, 1, 1, 1}); // toDotFile(a, "成田空港", "/mnt/scratch/out.dot"); + a.close(); } @@ -129,6 +134,7 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { JapaneseAnalyzer.getDefaultStopSet(), JapaneseAnalyzer.getDefaultStopTags()); checkRandomData(random, a, atLeast(1000)); + a.close(); } /** blast some random large strings through the analyzer */ @@ -138,6 +144,7 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { JapaneseAnalyzer.getDefaultStopSet(), JapaneseAnalyzer.getDefaultStopTags()); checkRandomData(random, a, 2*RANDOM_MULTIPLIER, 8192); + a.close(); } // Copied from TestJapaneseTokenizer, to make sure passing @@ -154,6 +161,7 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { new int[] { 1, 2, 4 }, new Integer(4) ); + a.close(); } // LUCENE-3897: this string (found by running all jawiki @@ -165,6 +173,7 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { JapaneseAnalyzer.getDefaultStopSet(), JapaneseAnalyzer.getDefaultStopTags()); checkAnalysisConsistency(random, a, random.nextBoolean(), s); + a.close(); } // LUCENE-3897: this string (found by @@ -176,6 +185,7 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { JapaneseAnalyzer.getDefaultStopSet(), JapaneseAnalyzer.getDefaultStopTags()); checkAnalysisConsistency(random, a, random.nextBoolean(), s); + a.close(); } // LUCENE-3897: this string (found by @@ -187,6 +197,7 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { JapaneseAnalyzer.getDefaultStopSet(), JapaneseAnalyzer.getDefaultStopTags()); checkAnalysisConsistency(random, a, random.nextBoolean(), s); + a.close(); } public void test4thCuriousString() throws Exception { @@ -194,8 +205,8 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { final Analyzer a = new JapaneseAnalyzer(null, Mode.SEARCH, JapaneseAnalyzer.getDefaultStopSet(), JapaneseAnalyzer.getDefaultStopTags()); - Random random = random(); - checkAnalysisConsistency(random, a, true, s); + checkAnalysisConsistency(random(), a, true, s); + a.close(); } public void test5thCuriousString() throws Exception { @@ -203,7 +214,7 @@ public class TestJapaneseAnalyzer extends BaseTokenStreamTestCase { final Analyzer a = new JapaneseAnalyzer(null, Mode.SEARCH, JapaneseAnalyzer.getDefaultStopSet(), JapaneseAnalyzer.getDefaultStopTags()); - Random random = random(); - checkAnalysisConsistency(random, a, false, s); + checkAnalysisConsistency(random(), a, false, s); + a.close(); } } diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java index e2397621f63..68488dc7e40 100644 --- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java +++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java @@ -28,13 +28,25 @@ import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter; import org.apache.lucene.analysis.util.CharArraySet; public class TestJapaneseBaseFormFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.DEFAULT_MODE); - return new TokenStreamComponents(tokenizer, new JapaneseBaseFormFilter(tokenizer)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.DEFAULT_MODE); + return new TokenStreamComponents(tokenizer, new JapaneseBaseFormFilter(tokenizer)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } public void testBasics() throws IOException { assertAnalyzesTo(analyzer, "それはまだ実験段階にあります", @@ -55,6 +67,7 @@ public class TestJapaneseBaseFormFilter extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "それはまだ実験段階にあります", new String[] { "それ", "は", "まだ", "実験", "段階", "に", "あり", "ます" } ); + a.close(); } public void testEnglish() throws IOException { @@ -75,5 +88,6 @@ public class TestJapaneseBaseFormFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java index 9cfc1820d1c..bdd1006dd08 100644 --- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java +++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java @@ -22,38 +22,49 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.util.IOUtils; import java.io.IOException; import java.io.Reader; import java.io.StringReader; public class TestJapaneseIterationMarkCharFilter extends BaseTokenStreamTestCase { + private Analyzer keywordAnalyzer, japaneseAnalyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + keywordAnalyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false); + return new TokenStreamComponents(tokenizer, tokenizer); + } - private Analyzer keywordAnalyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false); - return new TokenStreamComponents(tokenizer, tokenizer); - } + @Override + protected Reader initReader(String fieldName, Reader reader) { + return new JapaneseIterationMarkCharFilter(reader); + } + }; + japaneseAnalyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, false, JapaneseTokenizer.Mode.SEARCH); + return new TokenStreamComponents(tokenizer, tokenizer); + } - @Override - protected Reader initReader(String fieldName, Reader reader) { - return new JapaneseIterationMarkCharFilter(reader); - } - }; - - private Analyzer japaneseAnalyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, false, JapaneseTokenizer.Mode.SEARCH); - return new TokenStreamComponents(tokenizer, tokenizer); - } - - @Override - protected Reader initReader(String fieldName, Reader reader) { - return new JapaneseIterationMarkCharFilter(reader); - } - }; + @Override + protected Reader initReader(String fieldName, Reader reader) { + return new JapaneseIterationMarkCharFilter(reader); + } + }; + } + + @Override + public void tearDown() throws Exception { + IOUtils.close(keywordAnalyzer, japaneseAnalyzer); + super.tearDown(); + } public void testKanji() throws IOException { // Test single repetition diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java index c7014fd2883..9720f61580e 100644 --- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java +++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java @@ -32,14 +32,26 @@ import java.io.IOException; * Tests for {@link JapaneseKatakanaStemFilter} */ public class TestJapaneseKatakanaStemFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - // Use a MockTokenizer here since this filter doesn't really depend on Kuromoji - Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(source, new JapaneseKatakanaStemFilter(source)); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + // Use a MockTokenizer here since this filter doesn't really depend on Kuromoji + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new JapaneseKatakanaStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** * Test a few common katakana spelling variations. @@ -73,6 +85,7 @@ public class TestJapaneseKatakanaStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "コーヒー", "コーヒー"); + a.close(); } public void testUnsupportedHalfWidthVariants() throws IOException { @@ -93,5 +106,6 @@ public class TestJapaneseKatakanaStemFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseNumberFilter.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseNumberFilter.java index b0675beaf29..503fafdfcdc 100644 --- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseNumberFilter.java +++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseNumberFilter.java @@ -36,15 +36,26 @@ import org.junit.Ignore; import org.junit.Test; public class TestJapaneseNumberFilter extends BaseTokenStreamTestCase { - - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, false, JapaneseTokenizer.Mode.SEARCH); - return new TokenStreamComponents(tokenizer, new JapaneseNumberFilter(tokenizer)); - } - }; - + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, false, JapaneseTokenizer.Mode.SEARCH); + return new TokenStreamComponents(tokenizer, new JapaneseNumberFilter(tokenizer)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } + @Test public void testBasics() throws IOException { @@ -188,6 +199,7 @@ public class TestJapaneseNumberFilter extends BaseTokenStreamTestCase { new int[]{2, 4}, new int[]{1, 1} ); + keywordMarkingAnalyzer.close(); } @Test @@ -285,6 +297,7 @@ public class TestJapaneseNumberFilter extends BaseTokenStreamTestCase { Files.newBufferedReader(input, StandardCharsets.UTF_8), Files.newBufferedWriter(normalizedOutput, StandardCharsets.UTF_8) ); + plainAnalyzer.close(); } public void analyze(Analyzer analyzer, Reader reader, Writer writer) throws IOException { diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java index 5252b919279..dfc6d13d58f 100644 --- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java +++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java @@ -23,31 +23,41 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.cjk.CJKWidthFilter; import org.apache.lucene.analysis.core.KeywordTokenizer; +import org.apache.lucene.util.IOUtils; import java.io.IOException; -import java.io.Reader; import java.util.Random; /** * Tests for {@link TestJapaneseReadingFormFilter} */ public class TestJapaneseReadingFormFilter extends BaseTokenStreamTestCase { - private Analyzer katakanaAnalyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH); - return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, false)); - } - }; - - private Analyzer romajiAnalyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH); - return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, true)); - } - }; - + private Analyzer katakanaAnalyzer, romajiAnalyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + katakanaAnalyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH); + return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, false)); + } + }; + romajiAnalyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH); + return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, true)); + } + }; + } + + @Override + public void tearDown() throws Exception { + IOUtils.close(katakanaAnalyzer, romajiAnalyzer); + super.tearDown(); + } public void testKatakanaReadings() throws IOException { assertAnalyzesTo(katakanaAnalyzer, "今夜はロバート先生と話した", @@ -67,6 +77,7 @@ public class TestJapaneseReadingFormFilter extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した", new String[] { "コンヤ", "ハ", "ロバート", "センセイ", "ト", "ハナシ", "タ" } ); + a.close(); } public void testRomajiReadings() throws IOException { @@ -87,6 +98,7 @@ public class TestJapaneseReadingFormFilter extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した", new String[] { "kon'ya", "ha", "robato", "sensei", "to", "hanashi", "ta" } ); + a.close(); } public void testRandomData() throws IOException { @@ -104,5 +116,6 @@ public class TestJapaneseReadingFormFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java index 4c2c1f96004..384f0a8f87c 100644 --- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java +++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java @@ -35,9 +35,9 @@ import org.apache.lucene.analysis.ja.dict.ConnectionCosts; import org.apache.lucene.analysis.ja.dict.UserDictionary; import org.apache.lucene.analysis.ja.tokenattributes.*; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.UnicodeUtil; -import org.apache.lucene.util.LuceneTestCase.Slow; public class TestJapaneseTokenizer extends BaseTokenStreamTestCase { @@ -57,38 +57,47 @@ public class TestJapaneseTokenizer extends BaseTokenStreamTestCase { throw new RuntimeException(ioe); } } - - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH); - return new TokenStreamComponents(tokenizer, tokenizer); - } - }; - - private Analyzer analyzerNormal = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.NORMAL); - return new TokenStreamComponents(tokenizer, tokenizer); - } - }; - - private Analyzer analyzerNoPunct = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), true, Mode.SEARCH); - return new TokenStreamComponents(tokenizer, tokenizer); - } - }; - - private Analyzer extendedModeAnalyzerNoPunct = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), true, Mode.EXTENDED); - return new TokenStreamComponents(tokenizer, tokenizer); - } - }; + + private Analyzer analyzer, analyzerNormal, analyzerNoPunct, extendedModeAnalyzerNoPunct; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH); + return new TokenStreamComponents(tokenizer, tokenizer); + } + }; + analyzerNormal = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.NORMAL); + return new TokenStreamComponents(tokenizer, tokenizer); + } + }; + analyzerNoPunct = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), true, Mode.SEARCH); + return new TokenStreamComponents(tokenizer, tokenizer); + } + }; + extendedModeAnalyzerNoPunct = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), true, Mode.EXTENDED); + return new TokenStreamComponents(tokenizer, tokenizer); + } + }; + } + + @Override + public void tearDown() throws Exception { + IOUtils.close(analyzer, analyzerNormal, analyzerNoPunct, extendedModeAnalyzerNoPunct); + super.tearDown(); + } public void testNormalMode() throws Exception { assertAnalyzesTo(analyzerNormal, @@ -197,16 +206,16 @@ public class TestJapaneseTokenizer extends BaseTokenStreamTestCase { public void testRandomHugeStringsMockGraphAfter() throws Exception { // Randomly inject graph tokens after JapaneseTokenizer: Random random = random(); - checkRandomData(random, - new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH); - TokenStream graph = new MockGraphTokenFilter(random(), tokenizer); - return new TokenStreamComponents(tokenizer, graph); - } - }, - 20*RANDOM_MULTIPLIER, 8192); + Analyzer analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH); + TokenStream graph = new MockGraphTokenFilter(random(), tokenizer); + return new TokenStreamComponents(tokenizer, graph); + } + }; + checkRandomData(random, analyzer, 20*RANDOM_MULTIPLIER, 8192); + analyzer.close(); } public void testLargeDocReliability() throws Exception { @@ -367,6 +376,7 @@ public class TestJapaneseTokenizer extends BaseTokenStreamTestCase { surfaceForms); assertTrue(gv2.finish().indexOf("22.0") != -1); + analyzer.close(); } private void assertReadings(String input, String... readings) throws IOException { diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java index d100acd5a11..17f8269dcff 100644 --- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java +++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java @@ -31,13 +31,25 @@ import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode; public class TestSearchMode extends BaseTokenStreamTestCase { private final static String SEGMENTATION_FILENAME = "search-segmentation-tests.txt"; - private final Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, Mode.SEARCH); - return new TokenStreamComponents(tokenizer, tokenizer); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, Mode.SEARCH); + return new TokenStreamComponents(tokenizer, tokenizer); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** Test search mode segmentation */ public void testSearchSegmentation() throws IOException { diff --git a/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java b/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java index 08c983d68e4..1fa45918fb5 100644 --- a/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java +++ b/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java @@ -46,6 +46,7 @@ public class TestMorfologikAnalyzer extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "liście", new String[] { "liście", "liść", "list", "lista" }); assertAnalyzesTo(a, "danych", new String[] { "dany", "dana", "dane", "dać" }); assertAnalyzesTo(a, "ęóąśłżźćń", new String[] { "ęóąśłżźćń" }); + a.close(); } /** Test stemming of multiple tokens and proper term metrics. */ @@ -66,11 +67,13 @@ public class TestMorfologikAnalyzer extends BaseTokenStreamTestCase { new int[] { 0, 0, 3 }, new int[] { 1, 1, 13 }, new int[] { 1, 0, 1 }); + a.close(); } @SuppressWarnings("unused") private void dumpTokens(String input) throws IOException { - try (TokenStream ts = getTestAnalyzer().tokenStream("dummy", input)) { + try (Analyzer a = getTestAnalyzer(); + TokenStream ts = a.tokenStream("dummy", input)) { ts.reset(); MorphosyntacticTagsAttribute attribute = ts.getAttribute(MorphosyntacticTagsAttribute.class); @@ -100,6 +103,7 @@ public class TestMorfologikAnalyzer extends BaseTokenStreamTestCase { assertEquals("second stream", "dany", termAtt_2.toString()); ts_2.end(); } + a.close(); } /** Test stemming of mixed-case tokens. */ @@ -116,6 +120,7 @@ public class TestMorfologikAnalyzer extends BaseTokenStreamTestCase { assertAnalyzesTo(a, "aarona", new String[] { "aarona" }); assertAnalyzesTo(a, "Liście", new String[] { "liście", "liść", "list", "lista" }); + a.close(); } private void assertPOSToken(TokenStream ts, String term, String... tags) throws IOException { @@ -140,7 +145,8 @@ public class TestMorfologikAnalyzer extends BaseTokenStreamTestCase { /** Test morphosyntactic annotations. */ public final void testPOSAttribute() throws IOException { - try (TokenStream ts = getTestAnalyzer().tokenStream("dummy", "liście")) { + try (Analyzer a = getTestAnalyzer(); + TokenStream ts = a.tokenStream("dummy", "liście")) { ts.reset(); assertPOSToken(ts, "liście", "subst:sg:acc:n2", @@ -187,10 +193,13 @@ public class TestMorfologikAnalyzer extends BaseTokenStreamTestCase { new int[] { 0, 7, 7, 7, 7 }, new int[] { 6, 13, 13, 13, 13 }, new int[] { 1, 1, 0, 0, 0 }); + a.close(); } /** blast some random strings through the analyzer */ public void testRandom() throws Exception { - checkRandomData(random(), getTestAnalyzer(), 1000 * RANDOM_MULTIPLIER); + Analyzer a = getTestAnalyzer(); + checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER); + a.close(); } } diff --git a/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java b/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java index 169c3c69424..cfd58212c9f 100644 --- a/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java +++ b/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java @@ -81,6 +81,7 @@ public class DoubleMetaphoneFilterTest extends BaseTokenStreamTestCase { }; checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER); + a.close(); Analyzer b = new Analyzer() { @@ -92,6 +93,7 @@ public class DoubleMetaphoneFilterTest extends BaseTokenStreamTestCase { }; checkRandomData(random(), b, 1000 * RANDOM_MULTIPLIER); + b.close(); } public void testEmptyTerm() throws IOException { @@ -103,5 +105,6 @@ public class DoubleMetaphoneFilterTest extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java b/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java index bbcf0eda5cf..126f66928db 100644 --- a/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java +++ b/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java @@ -18,7 +18,6 @@ package org.apache.lucene.analysis.phonetic; */ import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import java.util.HashSet; import java.util.regex.Pattern; @@ -35,18 +34,29 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.core.KeywordTokenizer; import org.apache.lucene.analysis.miscellaneous.PatternKeywordMarkerFilter; import org.apache.lucene.analysis.tokenattributes.KeywordAttribute; -import org.junit.Ignore; /** Tests {@link BeiderMorseFilter} */ public class TestBeiderMorseFilter extends BaseTokenStreamTestCase { - private Analyzer analyzer = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(tokenizer, - new BeiderMorseFilter(tokenizer, new PhoneticEngine(NameType.GENERIC, RuleType.EXACT, true))); - } - }; + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(tokenizer, + new BeiderMorseFilter(tokenizer, new PhoneticEngine(NameType.GENERIC, RuleType.EXACT, true))); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } /** generic, "exact" configuration */ public void testBasicUsage() throws Exception { @@ -83,6 +93,7 @@ public class TestBeiderMorseFilter extends BaseTokenStreamTestCase { new int[] { 0, 0, 0, }, new int[] { 6, 6, 6, }, new int[] { 1, 0, 0, }); + analyzer.close(); } /** for convenience, if the input yields no output, we pass it thru as-is */ @@ -107,6 +118,7 @@ public class TestBeiderMorseFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } public void testCustomAttribute() throws IOException { diff --git a/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestDaitchMokotoffSoundexFilter.java b/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestDaitchMokotoffSoundexFilter.java index 4b0d436cd38..ed28022d773 100644 --- a/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestDaitchMokotoffSoundexFilter.java +++ b/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestDaitchMokotoffSoundexFilter.java @@ -58,6 +58,7 @@ public class TestDaitchMokotoffSoundexFilter extends BaseTokenStreamTestCase { }; checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER); + a.close(); Analyzer b = new Analyzer() { @Override @@ -68,6 +69,7 @@ public class TestDaitchMokotoffSoundexFilter extends BaseTokenStreamTestCase { }; checkRandomData(random(), b, 1000 * RANDOM_MULTIPLIER); + b.close(); } public void testEmptyTerm() throws IOException { @@ -79,6 +81,7 @@ public class TestDaitchMokotoffSoundexFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } diff --git a/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestPhoneticFilter.java b/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestPhoneticFilter.java index f5aff7a741e..ade1fd9f419 100644 --- a/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestPhoneticFilter.java +++ b/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestPhoneticFilter.java @@ -96,6 +96,7 @@ public class TestPhoneticFilter extends BaseTokenStreamTestCase { }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + a.close(); Analyzer b = new Analyzer() { @Override @@ -106,6 +107,7 @@ public class TestPhoneticFilter extends BaseTokenStreamTestCase { }; checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER); + b.close(); } } @@ -122,6 +124,7 @@ public class TestPhoneticFilter extends BaseTokenStreamTestCase { } }; checkOneTerm(a, "", ""); + a.close(); } } } diff --git a/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java b/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java index 1fd57ccc57a..940c6c5265c 100644 --- a/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java +++ b/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java @@ -17,19 +17,10 @@ package org.apache.lucene.analysis.cn.smart; -import java.io.IOException; -import java.io.Reader; -import java.util.Random; - import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.core.KeywordTokenizer; -import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; -import org.apache.lucene.util.Version; +import org.apache.lucene.util.IOUtils; public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { @@ -38,9 +29,11 @@ public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { String sentence = "我购买了道具和服装。"; String result[] = { "我", "购买", "了", "道具", "和", "服装" }; assertAnalyzesTo(ca, sentence, result); + ca.close(); // set stop-words from the outer world - must yield same behavior ca = new SmartChineseAnalyzer(SmartChineseAnalyzer.getDefaultStopSet()); assertAnalyzesTo(ca, sentence, result); + ca.close(); } /* @@ -52,6 +45,7 @@ public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { String sentence = "我购买了道具和服装。 我购买了道具和服装。"; String result[] = { "我", "购买", "了", "道具", "和", "服装", "我", "购买", "了", "道具", "和", "服装" }; assertAnalyzesTo(ca, sentence, result); + ca.close(); } /* @@ -63,6 +57,7 @@ public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { String sentence = "我购买了道具和服装　我购买了道具和服装。"; String result[] = { "我", "购买", "了", "道具", "和", "服装", "我", "购买", "了", "道具", "和", "服装" }; assertAnalyzesTo(ca, sentence, result); + ca.close(); } /* @@ -81,6 +76,7 @@ public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { assertAnalyzesTo(analyzer, sentence, result); assertAnalyzesTo(analyzer, sentence, result); } + IOUtils.close(analyzers); } /* @@ -95,6 +91,7 @@ public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { int endOffsets[] = { 5, 9 }; int posIncr[] = { 1, 2 }; assertAnalyzesTo(ca, sentence, result, startOffsets, endOffsets, posIncr); + ca.close(); } public void testChineseAnalyzer() throws Exception { @@ -102,38 +99,47 @@ public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { String sentence = "我购买了道具和服装。"; String[] result = { "我", "购买", "了", "道具", "和", "服装" }; assertAnalyzesTo(ca, sentence, result); + ca.close(); } /* * English words are lowercased and porter-stemmed. */ public void testMixedLatinChinese() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "我购买 Tests 了道具和服装", + Analyzer analyzer = new SmartChineseAnalyzer(true); + assertAnalyzesTo(analyzer, "我购买 Tests 了道具和服装", new String[] { "我", "购买", "test", "了", "道具", "和", "服装"}); + analyzer.close(); } /* * Numerics are parsed as their own tokens */ public void testNumerics() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "我购买 Tests 了道具和服装1234", + Analyzer analyzer = new SmartChineseAnalyzer(true); + assertAnalyzesTo(analyzer, "我购买 Tests 了道具和服装1234", new String[] { "我", "购买", "test", "了", "道具", "和", "服装", "1234"}); + analyzer.close(); } /* * Full width alphas and numerics are folded to half-width */ public void testFullWidth() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "我购买 Ｔｅｓｔｓ 了道具和服装１２３４", + Analyzer analyzer = new SmartChineseAnalyzer(true); + assertAnalyzesTo(analyzer, "我购买 Ｔｅｓｔｓ 了道具和服装１２３４", new String[] { "我", "购买", "test", "了", "道具", "和", "服装", "1234"}); + analyzer.close(); } /* * Presentation form delimiters are removed */ public void testDelimiters() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "我购买︱ Tests 了道具和服装", + Analyzer analyzer = new SmartChineseAnalyzer(true); + assertAnalyzesTo(analyzer, "我购买︱ Tests 了道具和服装", new String[] { "我", "购买", "test", "了", "道具", "和", "服装"}); + analyzer.close(); } /* @@ -141,8 +147,10 @@ public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { * (regardless of Unicode category) */ public void testNonChinese() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "我购买 روبرتTests 了道具和服装", + Analyzer analyzer = new SmartChineseAnalyzer(true); + assertAnalyzesTo(analyzer, "我购买 روبرتTests 了道具和服装", new String[] { "我", "购买", "ر", "و", "ب", "ر", "ت", "test", "了", "道具", "和", "服装"}); + analyzer.close(); } /* @@ -151,18 +159,22 @@ public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { * Currently it is being analyzed into single characters... */ public void testOOV() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "优素福·拉扎·吉拉尼", + Analyzer analyzer = new SmartChineseAnalyzer(true); + assertAnalyzesTo(analyzer, "优素福·拉扎·吉拉尼", new String[] { "优", "素", "福", "拉", "扎", "吉", "拉", "尼" }); - assertAnalyzesTo(new SmartChineseAnalyzer(true), "优素福拉扎吉拉尼", + assertAnalyzesTo(analyzer, "优素福拉扎吉拉尼", new String[] { "优", "素", "福", "拉", "扎", "吉", "拉", "尼" }); + analyzer.close(); } public void testOffsets() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "我购买了道具和服装", + Analyzer analyzer = new SmartChineseAnalyzer(true); + assertAnalyzesTo(analyzer, "我购买了道具和服装", new String[] { "我", "购买", "了", "道具", "和", "服装" }, new int[] { 0, 1, 3, 4, 6, 7 }, new int[] { 1, 3, 4, 6, 7, 9 }); + analyzer.close(); } public void testReusableTokenStream() throws Exception { @@ -175,6 +187,7 @@ public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { new String[] { "我", "购买", "了", "道具", "和", "服装" }, new int[] { 0, 1, 3, 4, 6, 7 }, new int[] { 1, 3, 4, 6, 7, 9 }); + a.close(); } // LUCENE-3026 @@ -183,8 +196,8 @@ public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { for (int i = 0; i < 5000; i++) { sb.append("我购买了道具和服装。"); } - Analyzer analyzer = new SmartChineseAnalyzer(); - try (TokenStream stream = analyzer.tokenStream("", sb.toString())) { + try (Analyzer analyzer = new SmartChineseAnalyzer(); + TokenStream stream = analyzer.tokenStream("", sb.toString())) { stream.reset(); while (stream.incrementToken()) { } @@ -198,8 +211,8 @@ public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { for (int i = 0; i < 5000; i++) { sb.append("我购买了道具和服装"); } - Analyzer analyzer = new SmartChineseAnalyzer(); - try (TokenStream stream = analyzer.tokenStream("", sb.toString())) { + try (Analyzer analyzer = new SmartChineseAnalyzer(); + TokenStream stream = analyzer.tokenStream("", sb.toString())) { stream.reset(); while (stream.incrementToken()) { } @@ -209,12 +222,15 @@ public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new SmartChineseAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new SmartChineseAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } /** blast some random large strings through the analyzer */ public void testRandomHugeStrings() throws Exception { - Random random = random(); - checkRandomData(random, new SmartChineseAnalyzer(), 100*RANDOM_MULTIPLIER, 8192); + Analyzer analyzer = new SmartChineseAnalyzer(); + checkRandomData(random(), analyzer, 100*RANDOM_MULTIPLIER, 8192); + analyzer.close(); } } diff --git a/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/pl/TestPolishAnalyzer.java b/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/pl/TestPolishAnalyzer.java index 9744eec3c09..483a9804117 100644 --- a/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/pl/TestPolishAnalyzer.java +++ b/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/pl/TestPolishAnalyzer.java @@ -27,7 +27,7 @@ public class TestPolishAnalyzer extends BaseTokenStreamTestCase { /** This test fails with NPE when the * stopwords file is missing in classpath */ public void testResourcesAvailable() { - new PolishAnalyzer(); + new PolishAnalyzer().close(); } /** test stopwords and stemming */ @@ -38,6 +38,7 @@ public class TestPolishAnalyzer extends BaseTokenStreamTestCase { checkOneTerm(a, "studenci", "student"); // stopword assertAnalyzesTo(a, "był", new String[] {}); + a.close(); } /** test use of exclusion set */ @@ -46,10 +47,13 @@ public class TestPolishAnalyzer extends BaseTokenStreamTestCase { Analyzer a = new PolishAnalyzer(PolishAnalyzer.getDefaultStopSet(), exclusionSet); checkOneTerm(a, "studenta", "studenta"); checkOneTerm(a, "studenci", "student"); + a.close(); } /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - checkRandomData(random(), new PolishAnalyzer(), 1000*RANDOM_MULTIPLIER); + Analyzer analyzer = new PolishAnalyzer(); + checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMABaseAnalyzerTest.java b/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMABaseAnalyzerTest.java index 892b35a20df..d3f53dc3585 100644 --- a/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMABaseAnalyzerTest.java +++ b/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMABaseAnalyzerTest.java @@ -17,6 +17,7 @@ package org.apache.lucene.analysis.uima; * limitations under the License. */ +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Document; @@ -120,16 +121,18 @@ public class UIMABaseAnalyzerTest extends BaseTokenStreamTestCase { @Test public void testRandomStrings() throws Exception { - checkRandomData(random(), new UIMABaseAnalyzer("/uima/TestAggregateSentenceAE.xml", "org.apache.lucene.uima.ts.TokenAnnotation", null), - 100 * RANDOM_MULTIPLIER); + Analyzer analyzer = new UIMABaseAnalyzer("/uima/TestAggregateSentenceAE.xml", "org.apache.lucene.uima.ts.TokenAnnotation", null); + checkRandomData(random(), analyzer, 100 * RANDOM_MULTIPLIER); + analyzer.close(); } @Test public void testRandomStringsWithConfigurationParameters() throws Exception { Map cp = new HashMap<>(); cp.put("line-end", "\r"); - checkRandomData(random(), new UIMABaseAnalyzer("/uima/TestWSTokenizerAE.xml", "org.apache.lucene.uima.ts.TokenAnnotation", cp), - 100 * RANDOM_MULTIPLIER); + Analyzer analyzer = new UIMABaseAnalyzer("/uima/TestWSTokenizerAE.xml", "org.apache.lucene.uima.ts.TokenAnnotation", cp); + checkRandomData(random(), analyzer, 100 * RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMATypeAwareAnalyzerTest.java b/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMATypeAwareAnalyzerTest.java index 454e45e9dd1..ab480e00d4a 100644 --- a/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMATypeAwareAnalyzerTest.java +++ b/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMATypeAwareAnalyzerTest.java @@ -17,9 +17,9 @@ package org.apache.lucene.analysis.uima; * limitations under the License. */ +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -61,8 +61,10 @@ public class UIMATypeAwareAnalyzerTest extends BaseTokenStreamTestCase { @Test public void testRandomStrings() throws Exception { - checkRandomData(random(), new UIMATypeAwareAnalyzer("/uima/TestAggregateSentenceAE.xml", - "org.apache.lucene.uima.ts.TokenAnnotation", "pos", null), 100 * RANDOM_MULTIPLIER); + Analyzer analyzer = new UIMATypeAwareAnalyzer("/uima/TestAggregateSentenceAE.xml", + "org.apache.lucene.uima.ts.TokenAnnotation", "pos", null); + checkRandomData(random(), analyzer, 100 * RANDOM_MULTIPLIER); + analyzer.close(); } } diff --git a/lucene/core/src/test/org/apache/lucene/TestDemo.java b/lucene/core/src/test/org/apache/lucene/TestDemo.java index 1812ed2106e..18f0875a714 100644 --- a/lucene/core/src/test/org/apache/lucene/TestDemo.java +++ b/lucene/core/src/test/org/apache/lucene/TestDemo.java @@ -77,5 +77,6 @@ public class TestDemo extends LuceneTestCase { ireader.close(); directory.close(); + analyzer.close(); } } diff --git a/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java b/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java index fa3d6398023..21bd68a212f 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java @@ -49,6 +49,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.PriorityQueue; @@ -126,9 +127,7 @@ public class CommonTermsQueryTest extends LuceneTestCase { assertEquals("3", r.document(search.scoreDocs[0].doc).get("id")); } - r.close(); - w.close(); - dir.close(); + IOUtils.close(r, w, dir, analyzer); } public void testEqualsHashCode() { @@ -321,9 +320,7 @@ public class CommonTermsQueryTest extends LuceneTestCase { r.document(search.scoreDocs[0].doc).get("id"), r.document(search.scoreDocs[1].doc).get("id")))); } - r.close(); - w.close(); - dir.close(); + IOUtils.close(r, w, dir, analyzer); } public void testIllegalOccur() { @@ -395,9 +392,7 @@ public class CommonTermsQueryTest extends LuceneTestCase { assertEquals("3", r.document(search.scoreDocs[1].doc).get("id")); assertEquals("0", r.document(search.scoreDocs[2].doc).get("id")); } - r.close(); - w.close(); - dir.close(); + IOUtils.close(r, w, dir, analyzer); } public void testRandomIndex() throws IOException { @@ -496,10 +491,7 @@ public class CommonTermsQueryTest extends LuceneTestCase { QueryUtils.check(random(), cq, newSearcher(reader2)); reader2.close(); } finally { - reader.close(); - wrapper.close(); - w.close(); - dir.close(); + IOUtils.close(reader, wrapper, w, dir, analyzer); } } diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java b/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java index 415331764e0..b71784c4d37 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java @@ -84,6 +84,7 @@ public abstract class FunctionTestSetup extends LuceneTestCase { public static void afterClassFunctionTestSetup() throws Exception { dir.close(); dir = null; + anlzr.close(); anlzr = null; } diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java index cb91ff274b7..23b8e0796fa 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java @@ -17,6 +17,7 @@ package org.apache.lucene.queries.function; * limitations under the License. */ +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -46,12 +47,15 @@ public class TestLongNormValueSource extends LuceneTestCase { static Directory dir; static IndexReader reader; static IndexSearcher searcher; + static Analyzer analyzer; + private static Similarity sim = new PreciseDefaultSimilarity(); @BeforeClass public static void beforeClass() throws Exception { dir = newDirectory(); - IndexWriterConfig iwConfig = newIndexWriterConfig(new MockAnalyzer(random())); + analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwConfig = newIndexWriterConfig(analyzer); iwConfig.setMergePolicy(newLogMergePolicy()); iwConfig.setSimilarity(sim); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig); @@ -76,6 +80,8 @@ public class TestLongNormValueSource extends LuceneTestCase { reader = null; dir.close(); dir = null; + analyzer.close(); + analyzer = null; } public void testNorm() throws Exception { diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java index 6fdd7281edd..d3cefa8f97b 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.io.IOException; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleField; @@ -92,6 +93,7 @@ import org.junit.BeforeClass; */ public class TestValueSources extends LuceneTestCase { static Directory dir; + static Analyzer analyzer; static IndexReader reader; static IndexSearcher searcher; @@ -109,7 +111,8 @@ public class TestValueSources extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { dir = newDirectory(); - IndexWriterConfig iwConfig = newIndexWriterConfig(new MockAnalyzer(random())); + analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwConfig = newIndexWriterConfig(analyzer); iwConfig.setMergePolicy(newLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig); Document document = new Document(); @@ -169,6 +172,8 @@ public class TestValueSources extends LuceneTestCase { reader = null; dir.close(); dir = null; + analyzer.close(); + analyzer = null; } public void testConst() throws Exception { diff --git a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java index ee03da55cec..34ff6017f3a 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; @@ -87,7 +88,8 @@ public class TestMoreLikeThis extends LuceneTestCase { Map originalValues = getOriginalValues(); MoreLikeThis mlt = new MoreLikeThis(reader); - mlt.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); + mlt.setAnalyzer(analyzer); mlt.setMinDocFreq(1); mlt.setMinTermFreq(1); mlt.setMinWordLen(1); @@ -116,12 +118,14 @@ public class TestMoreLikeThis extends LuceneTestCase { + tq.getTerm().text() + "' got " + tq.getBoost(), totalBoost, tq .getBoost(), 0.0001); } + analyzer.close(); } private Map getOriginalValues() throws IOException { Map originalValues = new HashMap<>(); MoreLikeThis mlt = new MoreLikeThis(reader); - mlt.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); + mlt.setAnalyzer(analyzer); mlt.setMinDocFreq(1); mlt.setMinTermFreq(1); mlt.setMinWordLen(1); @@ -135,24 +139,28 @@ public class TestMoreLikeThis extends LuceneTestCase { TermQuery tq = (TermQuery) clause.getQuery(); originalValues.put(tq.getTerm().text(), tq.getBoost()); } + analyzer.close(); return originalValues; } // LUCENE-3326 public void testMultiFields() throws Exception { MoreLikeThis mlt = new MoreLikeThis(reader); - mlt.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); + mlt.setAnalyzer(analyzer); mlt.setMinDocFreq(1); mlt.setMinTermFreq(1); mlt.setMinWordLen(1); mlt.setFieldNames(new String[] {"text", "foobar"}); mlt.like("foobar", new StringReader("this is a test")); + analyzer.close(); } // LUCENE-5725 public void testMultiValues() throws Exception { MoreLikeThis mlt = new MoreLikeThis(reader); - mlt.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false); + mlt.setAnalyzer(analyzer); mlt.setMinDocFreq(1); mlt.setMinTermFreq(1); mlt.setMinWordLen(1); @@ -167,12 +175,15 @@ public class TestMoreLikeThis extends LuceneTestCase { Term term = ((TermQuery) clause.getQuery()).getTerm(); assertTrue(Arrays.asList(new Term("text", "lucene"), new Term("text", "apache")).contains(term)); } + analyzer.close(); } // just basic equals/hashcode etc public void testMoreLikeThisQuery() throws Exception { - Query query = new MoreLikeThisQuery("this is a test", new String[] { "text" }, new MockAnalyzer(random()), "text"); + Analyzer analyzer = new MockAnalyzer(random()); + Query query = new MoreLikeThisQuery("this is a test", new String[] { "text" }, analyzer, "text"); QueryUtils.check(random(), query, searcher); + analyzer.close(); } public void testTopN() throws Exception { @@ -190,7 +201,8 @@ public class TestMoreLikeThis extends LuceneTestCase { // setup MLT query MoreLikeThis mlt = new MoreLikeThis(reader); - mlt.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); + mlt.setAnalyzer(analyzer); mlt.setMaxQueryTerms(topN); mlt.setMinDocFreq(1); mlt.setMinTermFreq(1); @@ -221,6 +233,7 @@ public class TestMoreLikeThis extends LuceneTestCase { // clean up reader.close(); dir.close(); + analyzer.close(); } private String[] generateStrSeq(int from, int size) { diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java index 921dd1fcbaa..2c232c6c9d7 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java @@ -20,6 +20,7 @@ package org.apache.lucene.sandbox.queries; import java.io.IOException; import java.util.HashSet; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -31,6 +32,7 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -40,12 +42,14 @@ public class DuplicateFilterTest extends LuceneTestCase { private IndexReader reader; TermQuery tq = new TermQuery(new Term("text", "lucene")); private IndexSearcher searcher; + Analyzer analyzer; @Override public void setUp() throws Exception { super.setUp(); directory = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); + analyzer = new MockAnalyzer(random()); + RandomIndexWriter writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig(analyzer).setMergePolicy(newLogMergePolicy())); //Add series of docs with filterable fields : url, text and dates flags addDoc(writer, "http://lucene.apache.org", "lucene 1.4.3 available", "20040101"); @@ -69,8 +73,7 @@ public class DuplicateFilterTest extends LuceneTestCase { @Override public void tearDown() throws Exception { - reader.close(); - directory.close(); + IOUtils.close(reader, directory, analyzer); super.tearDown(); } diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java index cd47a4789cc..4396ca31cd7 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java @@ -30,6 +30,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import java.io.IOException; @@ -47,7 +48,7 @@ public class FuzzyLikeThisQueryTest extends LuceneTestCase { analyzer = new MockAnalyzer(random()); directory = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); + RandomIndexWriter writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig(analyzer).setMergePolicy(newLogMergePolicy())); //Add series of docs with misspelt names addDoc(writer, "jonathon smythe", "1"); @@ -63,8 +64,7 @@ public class FuzzyLikeThisQueryTest extends LuceneTestCase { @Override public void tearDown() throws Exception { - reader.close(); - directory.close(); + IOUtils.close(reader, directory, analyzer); super.tearDown(); } diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery2.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery2.java index d46e626490f..01b9bfbf2bf 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery2.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery2.java @@ -22,6 +22,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; @@ -34,6 +35,7 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; /** @@ -90,7 +92,8 @@ public class TestSlowFuzzyQuery2 extends LuceneTestCase { int terms = (int) Math.pow(2, bits); Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)).setMergePolicy(newLogMergePolicy())); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(analyzer).setMergePolicy(newLogMergePolicy())); Document doc = new Document(); Field field = newTextField("field", "", Field.Store.NO); @@ -129,8 +132,7 @@ public class TestSlowFuzzyQuery2 extends LuceneTestCase { assertEquals(Float.parseFloat(scoreDoc[1]), docs.scoreDocs[i].score, epsilon); } } - r.close(); - dir.close(); + IOUtils.close(r, dir, analyzer); } /* map bits to unicode codepoints */ diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java index d91dc30686e..50f8638f1c0 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java @@ -18,6 +18,7 @@ package org.apache.lucene.sandbox.queries.regex; */ +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -30,6 +31,7 @@ import org.apache.lucene.search.spans.SpanFirstQuery; import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; public class TestSpanRegexQuery extends LuceneTestCase { @@ -53,7 +55,8 @@ public class TestSpanRegexQuery extends LuceneTestCase { public void testSpanRegex() throws Exception { Directory directory = newDirectory(); - IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(new MockAnalyzer(random()))); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(analyzer)); Document doc = new Document(); // doc.add(newField("field", "the quick brown fox jumps over the lazy dog", // Field.Store.NO, Field.Index.ANALYZED)); @@ -75,7 +78,6 @@ public class TestSpanRegexQuery extends LuceneTestCase { // true); int numHits = searcher.search(sfq, 1000).totalHits; assertEquals(1, numHits); - reader.close(); - directory.close(); + IOUtils.close(reader, directory, analyzer); } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java index 8e38750b428..da74e95b699 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java @@ -48,6 +48,7 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BitDocIdSet; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.automaton.Automata; @@ -610,9 +611,7 @@ public class TestTermAutomatonQuery extends LuceneTestCase { } } - w.close(); - r.close(); - dir.close(); + IOUtils.close(w, r, dir, analyzer); } private Set toDocIDs(IndexSearcher s, TopDocs hits) throws IOException { diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java index b8b386159ef..c7ee7931100 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java @@ -29,6 +29,8 @@ import com.spatial4j.core.context.SpatialContext; import com.spatial4j.core.distance.DistanceUtils; import com.spatial4j.core.shape.Point; import com.spatial4j.core.shape.Rectangle; + +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; @@ -46,8 +48,6 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks; import org.apache.lucene.util.TestUtil; -import org.junit.After; -import org.junit.Before; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomDouble; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomGaussian; @@ -63,6 +63,7 @@ public abstract class SpatialTestCase extends LuceneTestCase { private DirectoryReader indexReader; protected RandomIndexWriter indexWriter; private Directory directory; + private Analyzer analyzer; protected IndexSearcher indexSearcher; protected SpatialContext ctx;//subclass must initialize @@ -70,7 +71,6 @@ public abstract class SpatialTestCase extends LuceneTestCase { protected Map uninvertMap = new HashMap<>(); @Override - @Before public void setUp() throws Exception { super.setUp(); // TODO: change this module to index docvalues instead of uninverting @@ -80,13 +80,14 @@ public abstract class SpatialTestCase extends LuceneTestCase { directory = newDirectory(); final Random random = random(); - indexWriter = new RandomIndexWriter(random,directory, newIndexWriterConfig(random)); + analyzer = new MockAnalyzer(random); + indexWriter = new RandomIndexWriter(random,directory, newIWConfig(random, analyzer)); indexReader = UninvertingReader.wrap(indexWriter.getReader(), uninvertMap); indexSearcher = newSearcher(indexReader); } - protected IndexWriterConfig newIndexWriterConfig(Random random) { - final IndexWriterConfig indexWriterConfig = LuceneTestCase.newIndexWriterConfig(random, new MockAnalyzer(random)); + protected IndexWriterConfig newIWConfig(Random random, Analyzer analyzer) { + final IndexWriterConfig indexWriterConfig = LuceneTestCase.newIndexWriterConfig(random, analyzer); //TODO can we randomly choose a doc-values supported format? if (needsDocValues()) indexWriterConfig.setCodec( TestUtil.getDefaultCodec()); @@ -98,10 +99,8 @@ public abstract class SpatialTestCase extends LuceneTestCase { } @Override - @After public void tearDown() throws Exception { - indexWriter.close(); - IOUtils.close(indexReader,directory); + IOUtils.close(indexWriter, indexReader, analyzer, directory); super.tearDown(); } diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java index 647e29849b4..cdbf9eabad6 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java @@ -17,6 +17,7 @@ package org.apache.lucene.search.spell; * limitations under the License. */ +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; @@ -26,6 +27,7 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.util.English; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; public class TestDirectSpellChecker extends LuceneTestCase { @@ -33,8 +35,8 @@ public class TestDirectSpellChecker extends LuceneTestCase { public void testInternalLevenshteinDistance() throws Exception { DirectSpellChecker spellchecker = new DirectSpellChecker(); Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, - new MockAnalyzer(random(), MockTokenizer.KEYWORD, true)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, true); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer); String[] termsToAdd = { "metanoia", "metanoian", "metanoiai", "metanoias", "metanoi𐑍" }; for (int i = 0; i < termsToAdd.length; i++) { @@ -55,16 +57,15 @@ public class TestDirectSpellChecker extends LuceneTestCase { assertTrue(word.score==sd.getDistance(misspelled, word.string)); } - ir.close(); - writer.close(); - dir.close(); + IOUtils.close(ir, writer, dir, analyzer); } + public void testSimpleExamples() throws Exception { DirectSpellChecker spellChecker = new DirectSpellChecker(); spellChecker.setMinQueryLength(0); Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, - new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer); for (int i = 0; i < 20; i++) { Document doc = new Document(); @@ -121,15 +122,13 @@ public class TestDirectSpellChecker extends LuceneTestCase { assertTrue(similar.length > 0); assertEquals("thousand", similar[0].string); - ir.close(); - writer.close(); - dir.close(); + IOUtils.close(ir, writer, dir, analyzer); } public void testOptions() throws Exception { Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, - new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer); Document doc = new Document(); doc.add(newTextField("text", "foobar", Field.Store.NO)); @@ -187,16 +186,14 @@ public class TestDirectSpellChecker extends LuceneTestCase { SuggestMode.SUGGEST_ALWAYS); assertEquals(2, similar.length); - ir.close(); - writer.close(); - dir.close(); + IOUtils.close(ir, writer, dir, analyzer);; } public void testBogusField() throws Exception { DirectSpellChecker spellChecker = new DirectSpellChecker(); Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, - new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer); for (int i = 0; i < 20; i++) { Document doc = new Document(); @@ -210,17 +207,16 @@ public class TestDirectSpellChecker extends LuceneTestCase { "bogusFieldBogusField", "fvie"), 2, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX); assertEquals(0, similar.length); - ir.close(); - writer.close(); - dir.close(); + + IOUtils.close(ir, writer, dir, analyzer); } // simple test that transpositions work, we suggest five for fvie with ed=1 public void testTransposition() throws Exception { DirectSpellChecker spellChecker = new DirectSpellChecker(); Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, - new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer); for (int i = 0; i < 20; i++) { Document doc = new Document(); @@ -235,17 +231,16 @@ public class TestDirectSpellChecker extends LuceneTestCase { SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX); assertEquals(1, similar.length); assertEquals("five", similar[0].string); - ir.close(); - writer.close(); - dir.close(); + + IOUtils.close(ir, writer, dir, analyzer); } // simple test that transpositions work, we suggest seventeen for seevntene with ed=2 public void testTransposition2() throws Exception { DirectSpellChecker spellChecker = new DirectSpellChecker(); Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, - new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer); for (int i = 0; i < 20; i++) { Document doc = new Document(); @@ -260,8 +255,7 @@ public class TestDirectSpellChecker extends LuceneTestCase { SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX); assertEquals(1, similar.length); assertEquals("seventeen", similar[0].string); - ir.close(); - writer.close(); - dir.close(); + + IOUtils.close(ir, writer, dir, analyzer); } } diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java index 169246a6cdd..4dbbf4096b1 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java @@ -19,6 +19,7 @@ package org.apache.lucene.search.spell; import java.io.IOException; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; @@ -39,7 +40,7 @@ import org.apache.lucene.util.LuceneTestCase; public class TestLuceneDictionary extends LuceneTestCase { private Directory store; - + private Analyzer analyzer; private IndexReader indexReader = null; private LuceneDictionary ld; private BytesRefIterator it; @@ -49,7 +50,8 @@ public class TestLuceneDictionary extends LuceneTestCase { public void setUp() throws Exception { super.setUp(); store = newDirectory(); - IndexWriter writer = new IndexWriter(store, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))); + analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); + IndexWriter writer = new IndexWriter(store, newIndexWriterConfig(analyzer)); Document doc; @@ -82,6 +84,7 @@ public class TestLuceneDictionary extends LuceneTestCase { if (indexReader != null) indexReader.close(); store.close(); + analyzer.close(); super.tearDown(); } diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java index 70e83065454..823fade92e3 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java @@ -27,6 +27,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -47,6 +48,7 @@ import org.apache.lucene.util.NamedThreadFactory; public class TestSpellChecker extends LuceneTestCase { private SpellCheckerMock spellChecker; private Directory userindex, spellindex; + private Analyzer analyzer; private List searchers; @Override @@ -55,7 +57,8 @@ public class TestSpellChecker extends LuceneTestCase { //create a user index userindex = newDirectory(); - IndexWriter writer = new IndexWriter(userindex, new IndexWriterConfig(new MockAnalyzer(random()))); + analyzer = new MockAnalyzer(random()); + IndexWriter writer = new IndexWriter(userindex, new IndexWriterConfig(analyzer)); for (int i = 0; i < 1000; i++) { Document doc = new Document(); @@ -99,6 +102,7 @@ public class TestSpellChecker extends LuceneTestCase { if (!spellChecker.isClosed()) spellChecker.close(); spellindex.close(); + analyzer.close(); super.tearDown(); } diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java index de79db131ed..4cfb515b398 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java @@ -23,6 +23,7 @@ import java.util.regex.Pattern; import junit.framework.Assert; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; @@ -34,17 +35,20 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.spell.WordBreakSpellChecker.BreakSuggestionSortMethod; import org.apache.lucene.store.Directory; import org.apache.lucene.util.English; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; public class TestWordBreakSpellChecker extends LuceneTestCase { - private Directory dir = null; + private Directory dir; + private Analyzer analyzer; @Override public void setUp() throws Exception { super.setUp(); dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true)); + analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer); for (int i = 900; i < 1112; i++) { Document doc = new Document(); @@ -75,289 +79,262 @@ public class TestWordBreakSpellChecker extends LuceneTestCase { @Override public void tearDown() throws Exception { - if(dir!=null) { - dir.close(); - dir = null; - } + IOUtils.close(dir, analyzer); super.tearDown(); } + public void testCombiningWords() throws Exception { - IndexReader ir = null; - try { - ir = DirectoryReader.open(dir); - WordBreakSpellChecker wbsp = new WordBreakSpellChecker(); + IndexReader ir = DirectoryReader.open(dir); + WordBreakSpellChecker wbsp = new WordBreakSpellChecker(); + + { + Term[] terms = { + new Term("numbers", "one"), + new Term("numbers", "hun"), + new Term("numbers", "dred"), + new Term("numbers", "eight"), + new Term("numbers", "y"), + new Term("numbers", "eight"), + }; + wbsp.setMaxChanges(3); + wbsp.setMaxCombineWordLength(20); + wbsp.setMinSuggestionFrequency(1); + CombineSuggestion[] cs = wbsp.suggestWordCombinations(terms, 10, ir, SuggestMode.SUGGEST_ALWAYS); + Assert.assertTrue(cs.length==5); - { - Term[] terms = { - new Term("numbers", "one"), - new Term("numbers", "hun"), - new Term("numbers", "dred"), - new Term("numbers", "eight"), - new Term("numbers", "y"), - new Term("numbers", "eight"), - }; - wbsp.setMaxChanges(3); - wbsp.setMaxCombineWordLength(20); - wbsp.setMinSuggestionFrequency(1); - CombineSuggestion[] cs = wbsp.suggestWordCombinations(terms, 10, ir, SuggestMode.SUGGEST_ALWAYS); - Assert.assertTrue(cs.length==5); - - Assert.assertTrue(cs[0].originalTermIndexes.length==2); - Assert.assertTrue(cs[0].originalTermIndexes[0]==1); - Assert.assertTrue(cs[0].originalTermIndexes[1]==2); - Assert.assertTrue(cs[0].suggestion.string.equals("hundred")); - Assert.assertTrue(cs[0].suggestion.score==1); - - Assert.assertTrue(cs[1].originalTermIndexes.length==2); - Assert.assertTrue(cs[1].originalTermIndexes[0]==3); - Assert.assertTrue(cs[1].originalTermIndexes[1]==4); - Assert.assertTrue(cs[1].suggestion.string.equals("eighty")); - Assert.assertTrue(cs[1].suggestion.score==1); - - Assert.assertTrue(cs[2].originalTermIndexes.length==2); - Assert.assertTrue(cs[2].originalTermIndexes[0]==4); - Assert.assertTrue(cs[2].originalTermIndexes[1]==5); - Assert.assertTrue(cs[2].suggestion.string.equals("yeight")); - Assert.assertTrue(cs[2].suggestion.score==1); - - for(int i=3 ; i<5 ; i++) { - Assert.assertTrue(cs[i].originalTermIndexes.length==3); - Assert.assertTrue(cs[i].suggestion.score==2); - Assert.assertTrue( - (cs[i].originalTermIndexes[0]==1 && - cs[i].originalTermIndexes[1]==2 && - cs[i].originalTermIndexes[2]==3 && - cs[i].suggestion.string.equals("hundredeight")) || - (cs[i].originalTermIndexes[0]==3 && - cs[i].originalTermIndexes[1]==4 && - cs[i].originalTermIndexes[2]==5 && - cs[i].suggestion.string.equals("eightyeight")) - ); - } - - cs = wbsp.suggestWordCombinations(terms, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX); - Assert.assertTrue(cs.length==2); - Assert.assertTrue(cs[0].originalTermIndexes.length==2); - Assert.assertTrue(cs[0].suggestion.score==1); - Assert.assertTrue(cs[0].originalTermIndexes[0]==1); - Assert.assertTrue(cs[0].originalTermIndexes[1]==2); - Assert.assertTrue(cs[0].suggestion.string.equals("hundred")); - Assert.assertTrue(cs[0].suggestion.score==1); - - Assert.assertTrue(cs[1].originalTermIndexes.length==3); - Assert.assertTrue(cs[1].suggestion.score==2); - Assert.assertTrue(cs[1].originalTermIndexes[0] == 1); - Assert.assertTrue(cs[1].originalTermIndexes[1] == 2); - Assert.assertTrue(cs[1].originalTermIndexes[2] == 3); - Assert.assertTrue(cs[1].suggestion.string.equals("hundredeight")); - } - } catch(Exception e) { - throw e; - } finally { - try { ir.close(); } catch(Exception e1) { } - } + Assert.assertTrue(cs[0].originalTermIndexes.length==2); + Assert.assertTrue(cs[0].originalTermIndexes[0]==1); + Assert.assertTrue(cs[0].originalTermIndexes[1]==2); + Assert.assertTrue(cs[0].suggestion.string.equals("hundred")); + Assert.assertTrue(cs[0].suggestion.score==1); + + Assert.assertTrue(cs[1].originalTermIndexes.length==2); + Assert.assertTrue(cs[1].originalTermIndexes[0]==3); + Assert.assertTrue(cs[1].originalTermIndexes[1]==4); + Assert.assertTrue(cs[1].suggestion.string.equals("eighty")); + Assert.assertTrue(cs[1].suggestion.score==1); + + Assert.assertTrue(cs[2].originalTermIndexes.length==2); + Assert.assertTrue(cs[2].originalTermIndexes[0]==4); + Assert.assertTrue(cs[2].originalTermIndexes[1]==5); + Assert.assertTrue(cs[2].suggestion.string.equals("yeight")); + Assert.assertTrue(cs[2].suggestion.score==1); + + for(int i=3 ; i<5 ; i++) { + Assert.assertTrue(cs[i].originalTermIndexes.length==3); + Assert.assertTrue(cs[i].suggestion.score==2); + Assert.assertTrue( + (cs[i].originalTermIndexes[0]==1 && + cs[i].originalTermIndexes[1]==2 && + cs[i].originalTermIndexes[2]==3 && + cs[i].suggestion.string.equals("hundredeight")) || + (cs[i].originalTermIndexes[0]==3 && + cs[i].originalTermIndexes[1]==4 && + cs[i].originalTermIndexes[2]==5 && + cs[i].suggestion.string.equals("eightyeight")) + ); + } + + cs = wbsp.suggestWordCombinations(terms, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX); + Assert.assertTrue(cs.length==2); + Assert.assertTrue(cs[0].originalTermIndexes.length==2); + Assert.assertTrue(cs[0].suggestion.score==1); + Assert.assertTrue(cs[0].originalTermIndexes[0]==1); + Assert.assertTrue(cs[0].originalTermIndexes[1]==2); + Assert.assertTrue(cs[0].suggestion.string.equals("hundred")); + Assert.assertTrue(cs[0].suggestion.score==1); + + Assert.assertTrue(cs[1].originalTermIndexes.length==3); + Assert.assertTrue(cs[1].suggestion.score==2); + Assert.assertTrue(cs[1].originalTermIndexes[0] == 1); + Assert.assertTrue(cs[1].originalTermIndexes[1] == 2); + Assert.assertTrue(cs[1].originalTermIndexes[2] == 3); + Assert.assertTrue(cs[1].suggestion.string.equals("hundredeight")); + } + ir.close(); } + public void testBreakingWords() throws Exception { - IndexReader ir = null; - try { - ir = DirectoryReader.open(dir); - WordBreakSpellChecker wbsp = new WordBreakSpellChecker(); - - { - Term term = new Term("numbers", "ninetynine"); - wbsp.setMaxChanges(1); - wbsp.setMinBreakWordLength(1); - wbsp.setMinSuggestionFrequency(1); - SuggestWord[][] sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); - Assert.assertTrue(sw.length==1); - Assert.assertTrue(sw[0].length==2); - Assert.assertTrue(sw[0][0].string.equals("ninety")); - Assert.assertTrue(sw[0][1].string.equals("nine")); - Assert.assertTrue(sw[0][0].score == 1); - Assert.assertTrue(sw[0][1].score == 1); - } - { - Term term = new Term("numbers", "onethousand"); - wbsp.setMaxChanges(1); - wbsp.setMinBreakWordLength(1); - wbsp.setMinSuggestionFrequency(1); - SuggestWord[][] sw = wbsp.suggestWordBreaks(term, 2, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); - Assert.assertTrue(sw.length==1); - Assert.assertTrue(sw[0].length==2); - Assert.assertTrue(sw[0][0].string.equals("one")); - Assert.assertTrue(sw[0][1].string.equals("thousand")); - Assert.assertTrue(sw[0][0].score == 1); - Assert.assertTrue(sw[0][1].score == 1); - - wbsp.setMaxChanges(2); - wbsp.setMinSuggestionFrequency(1); - sw = wbsp.suggestWordBreaks(term, 1, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); - Assert.assertTrue(sw.length==1); - Assert.assertTrue(sw[0].length==2); - - wbsp.setMaxChanges(2); - wbsp.setMinSuggestionFrequency(2); - sw = wbsp.suggestWordBreaks(term, 2, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); - Assert.assertTrue(sw.length==1); - Assert.assertTrue(sw[0].length==2); - - wbsp.setMaxChanges(2); - wbsp.setMinSuggestionFrequency(1); - sw = wbsp.suggestWordBreaks(term, 2, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); - Assert.assertTrue(sw.length==2); - Assert.assertTrue(sw[0].length==2); - Assert.assertTrue(sw[0][0].string.equals("one")); - Assert.assertTrue(sw[0][1].string.equals("thousand")); - Assert.assertTrue(sw[0][0].score == 1); - Assert.assertTrue(sw[0][1].score == 1); - Assert.assertTrue(sw[0][1].freq>1); - Assert.assertTrue(sw[0][0].freq>sw[0][1].freq); - Assert.assertTrue(sw[1].length==3); - Assert.assertTrue(sw[1][0].string.equals("one")); - Assert.assertTrue(sw[1][1].string.equals("thou")); - Assert.assertTrue(sw[1][2].string.equals("sand")); - Assert.assertTrue(sw[1][0].score == 2); - Assert.assertTrue(sw[1][1].score == 2); - Assert.assertTrue(sw[1][2].score == 2); - Assert.assertTrue(sw[1][0].freq>1); - Assert.assertTrue(sw[1][1].freq==1); - Assert.assertTrue(sw[1][2].freq==1); - } - { - Term term = new Term("numbers", "onethousandonehundredeleven"); - wbsp.setMaxChanges(3); - wbsp.setMinBreakWordLength(1); - wbsp.setMinSuggestionFrequency(1); - SuggestWord[][] sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); - Assert.assertTrue(sw.length==0); - - wbsp.setMaxChanges(4); - sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); - Assert.assertTrue(sw.length==1); - Assert.assertTrue(sw[0].length==5); - - wbsp.setMaxChanges(5); - sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); - Assert.assertTrue(sw.length==2); - Assert.assertTrue(sw[0].length==5); - Assert.assertTrue(sw[0][1].string.equals("thousand")); - Assert.assertTrue(sw[1].length==6); - Assert.assertTrue(sw[1][1].string.equals("thou")); - Assert.assertTrue(sw[1][2].string.equals("sand")); - } - { - //make sure we can handle 2-char codepoints - Term term = new Term("numbers", "\uD864\uDC79"); - wbsp.setMaxChanges(1); - wbsp.setMinBreakWordLength(1); - wbsp.setMinSuggestionFrequency(1); - SuggestWord[][] sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); - Assert.assertTrue(sw.length==0); - } - - } catch(Exception e) { - throw e; - } finally { - try { ir.close(); } catch(Exception e1) { } - } - } - public void testRandom() throws Exception { - int numDocs = TestUtil.nextInt(random(), (10 * RANDOM_MULTIPLIER), - (100 * RANDOM_MULTIPLIER)); - Directory dir = null; - RandomIndexWriter writer = null; - IndexReader ir = null; - try { - dir = newDirectory(); - writer = new RandomIndexWriter(random(), dir, new MockAnalyzer(random(), - MockTokenizer.WHITESPACE, false)); - int maxLength = TestUtil.nextInt(random(), 5, 50); - List originals = new ArrayList<>(numDocs); - List breaks = new ArrayList<>(numDocs); - for (int i = 0; i < numDocs; i++) { - String orig = ""; - if (random().nextBoolean()) { - while (!goodTestString(orig)) { - orig = TestUtil.randomSimpleString(random(), maxLength); - } - } else { - while (!goodTestString(orig)) { - orig = TestUtil.randomUnicodeString(random(), maxLength); - } - } - originals.add(orig); - int totalLength = orig.codePointCount(0, orig.length()); - int breakAt = orig.offsetByCodePoints(0, - TestUtil.nextInt(random(), 1, totalLength - 1)); - String[] broken = new String[2]; - broken[0] = orig.substring(0, breakAt); - broken[1] = orig.substring(breakAt); - breaks.add(broken); - Document doc = new Document(); - doc.add(newTextField("random_break", broken[0] + " " + broken[1], - Field.Store.NO)); - doc.add(newTextField("random_combine", orig, Field.Store.NO)); - writer.addDocument(doc); - } - writer.commit(); - writer.close(); - - ir = DirectoryReader.open(dir); - WordBreakSpellChecker wbsp = new WordBreakSpellChecker(); + IndexReader ir = DirectoryReader.open(dir); + WordBreakSpellChecker wbsp = new WordBreakSpellChecker(); + + { + Term term = new Term("numbers", "ninetynine"); wbsp.setMaxChanges(1); wbsp.setMinBreakWordLength(1); wbsp.setMinSuggestionFrequency(1); - wbsp.setMaxCombineWordLength(maxLength); - for (int i = 0; i < originals.size(); i++) { - String orig = originals.get(i); - String left = breaks.get(i)[0]; - String right = breaks.get(i)[1]; - { - Term term = new Term("random_break", orig); - - SuggestWord[][] sw = wbsp.suggestWordBreaks(term, originals.size(), - ir, SuggestMode.SUGGEST_ALWAYS, - BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); - boolean failed = true; - for (SuggestWord[] sw1 : sw) { - Assert.assertTrue(sw1.length == 2); - if (sw1[0].string.equals(left) && sw1[1].string.equals(right)) { - failed = false; - } - } - Assert.assertFalse("Failed getting break suggestions\n >Original: " - + orig + "\n >Left: " + left + "\n >Right: " + right, failed); + SuggestWord[][] sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==1); + Assert.assertTrue(sw[0].length==2); + Assert.assertTrue(sw[0][0].string.equals("ninety")); + Assert.assertTrue(sw[0][1].string.equals("nine")); + Assert.assertTrue(sw[0][0].score == 1); + Assert.assertTrue(sw[0][1].score == 1); + } + { + Term term = new Term("numbers", "onethousand"); + wbsp.setMaxChanges(1); + wbsp.setMinBreakWordLength(1); + wbsp.setMinSuggestionFrequency(1); + SuggestWord[][] sw = wbsp.suggestWordBreaks(term, 2, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==1); + Assert.assertTrue(sw[0].length==2); + Assert.assertTrue(sw[0][0].string.equals("one")); + Assert.assertTrue(sw[0][1].string.equals("thousand")); + Assert.assertTrue(sw[0][0].score == 1); + Assert.assertTrue(sw[0][1].score == 1); + + wbsp.setMaxChanges(2); + wbsp.setMinSuggestionFrequency(1); + sw = wbsp.suggestWordBreaks(term, 1, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==1); + Assert.assertTrue(sw[0].length==2); + + wbsp.setMaxChanges(2); + wbsp.setMinSuggestionFrequency(2); + sw = wbsp.suggestWordBreaks(term, 2, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==1); + Assert.assertTrue(sw[0].length==2); + + wbsp.setMaxChanges(2); + wbsp.setMinSuggestionFrequency(1); + sw = wbsp.suggestWordBreaks(term, 2, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==2); + Assert.assertTrue(sw[0].length==2); + Assert.assertTrue(sw[0][0].string.equals("one")); + Assert.assertTrue(sw[0][1].string.equals("thousand")); + Assert.assertTrue(sw[0][0].score == 1); + Assert.assertTrue(sw[0][1].score == 1); + Assert.assertTrue(sw[0][1].freq>1); + Assert.assertTrue(sw[0][0].freq>sw[0][1].freq); + Assert.assertTrue(sw[1].length==3); + Assert.assertTrue(sw[1][0].string.equals("one")); + Assert.assertTrue(sw[1][1].string.equals("thou")); + Assert.assertTrue(sw[1][2].string.equals("sand")); + Assert.assertTrue(sw[1][0].score == 2); + Assert.assertTrue(sw[1][1].score == 2); + Assert.assertTrue(sw[1][2].score == 2); + Assert.assertTrue(sw[1][0].freq>1); + Assert.assertTrue(sw[1][1].freq==1); + Assert.assertTrue(sw[1][2].freq==1); + } + { + Term term = new Term("numbers", "onethousandonehundredeleven"); + wbsp.setMaxChanges(3); + wbsp.setMinBreakWordLength(1); + wbsp.setMinSuggestionFrequency(1); + SuggestWord[][] sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==0); + + wbsp.setMaxChanges(4); + sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==1); + Assert.assertTrue(sw[0].length==5); + + wbsp.setMaxChanges(5); + sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==2); + Assert.assertTrue(sw[0].length==5); + Assert.assertTrue(sw[0][1].string.equals("thousand")); + Assert.assertTrue(sw[1].length==6); + Assert.assertTrue(sw[1][1].string.equals("thou")); + Assert.assertTrue(sw[1][2].string.equals("sand")); + } + { + //make sure we can handle 2-char codepoints + Term term = new Term("numbers", "\uD864\uDC79"); + wbsp.setMaxChanges(1); + wbsp.setMinBreakWordLength(1); + wbsp.setMinSuggestionFrequency(1); + SuggestWord[][] sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==0); + } + + ir.close(); + } + + public void testRandom() throws Exception { + int numDocs = TestUtil.nextInt(random(), (10 * RANDOM_MULTIPLIER), + (100 * RANDOM_MULTIPLIER)); + IndexReader ir = null; + + Directory dir = newDirectory(); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer); + int maxLength = TestUtil.nextInt(random(), 5, 50); + List originals = new ArrayList<>(numDocs); + List breaks = new ArrayList<>(numDocs); + for (int i = 0; i < numDocs; i++) { + String orig = ""; + if (random().nextBoolean()) { + while (!goodTestString(orig)) { + orig = TestUtil.randomSimpleString(random(), maxLength); } - { - Term[] terms = {new Term("random_combine", left), - new Term("random_combine", right)}; - CombineSuggestion[] cs = wbsp.suggestWordCombinations(terms, - originals.size(), ir, SuggestMode.SUGGEST_ALWAYS); - boolean failed = true; - for (CombineSuggestion cs1 : cs) { - Assert.assertTrue(cs1.originalTermIndexes.length == 2); - if (cs1.suggestion.string.equals(left + right)) { - failed = false; - } - } - Assert.assertFalse("Failed getting combine suggestions\n >Original: " - + orig + "\n >Left: " + left + "\n >Right: " + right, failed); + } else { + while (!goodTestString(orig)) { + orig = TestUtil.randomUnicodeString(random(), maxLength); } } - - } catch (Exception e) { - throw e; - } finally { - try { - ir.close(); - } catch (Exception e1) {} - try { - writer.close(); - } catch (Exception e1) {} - try { - dir.close(); - } catch (Exception e1) {} + originals.add(orig); + int totalLength = orig.codePointCount(0, orig.length()); + int breakAt = orig.offsetByCodePoints(0, + TestUtil.nextInt(random(), 1, totalLength - 1)); + String[] broken = new String[2]; + broken[0] = orig.substring(0, breakAt); + broken[1] = orig.substring(breakAt); + breaks.add(broken); + Document doc = new Document(); + doc.add(newTextField("random_break", broken[0] + " " + broken[1], + Field.Store.NO)); + doc.add(newTextField("random_combine", orig, Field.Store.NO)); + writer.addDocument(doc); } + writer.commit(); + writer.close(); + + ir = DirectoryReader.open(dir); + WordBreakSpellChecker wbsp = new WordBreakSpellChecker(); + wbsp.setMaxChanges(1); + wbsp.setMinBreakWordLength(1); + wbsp.setMinSuggestionFrequency(1); + wbsp.setMaxCombineWordLength(maxLength); + for (int i = 0; i < originals.size(); i++) { + String orig = originals.get(i); + String left = breaks.get(i)[0]; + String right = breaks.get(i)[1]; + { + Term term = new Term("random_break", orig); + + SuggestWord[][] sw = wbsp.suggestWordBreaks(term, originals.size(), + ir, SuggestMode.SUGGEST_ALWAYS, + BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + boolean failed = true; + for (SuggestWord[] sw1 : sw) { + Assert.assertTrue(sw1.length == 2); + if (sw1[0].string.equals(left) && sw1[1].string.equals(right)) { + failed = false; + } + } + Assert.assertFalse("Failed getting break suggestions\n >Original: " + + orig + "\n >Left: " + left + "\n >Right: " + right, failed); + } + { + Term[] terms = {new Term("random_combine", left), + new Term("random_combine", right)}; + CombineSuggestion[] cs = wbsp.suggestWordCombinations(terms, + originals.size(), ir, SuggestMode.SUGGEST_ALWAYS); + boolean failed = true; + for (CombineSuggestion cs1 : cs) { + Assert.assertTrue(cs1.originalTermIndexes.length == 2); + if (cs1.suggestion.string.equals(left + right)) { + failed = false; + } + } + Assert.assertFalse("Failed getting combine suggestions\n >Original: " + + orig + "\n >Left: " + left + "\n >Right: " + right, failed); + } + } + IOUtils.close(ir, dir, analyzer); } private static final Pattern mockTokenizerWhitespacePattern = Pattern diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java index 2ebf11560db..74ff4bd080e 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Random; import java.util.Set; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -26,6 +27,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.spell.Dictionary; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.junit.Test; @@ -113,7 +115,8 @@ public class DocumentDictionaryTest extends LuceneTestCase { @Test public void testEmptyReader() throws IOException { Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwc = newIndexWriterConfig(analyzer); iwc.setMergePolicy(newLogMergePolicy()); // Make sure the index is created? RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); @@ -127,14 +130,14 @@ public class DocumentDictionaryTest extends LuceneTestCase { assertEquals(inputIterator.weight(), 0); assertNull(inputIterator.payload()); - ir.close(); - dir.close(); + IOUtils.close(ir, analyzer, dir); } @Test public void testBasic() throws IOException { Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwc = newIndexWriterConfig(analyzer); iwc.setMergePolicy(newLogMergePolicy()); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); Map.Entry, Map> res = generateIndexDocuments(atLeast(1000), true, false); @@ -162,14 +165,14 @@ public class DocumentDictionaryTest extends LuceneTestCase { } assertTrue(docs.isEmpty()); - ir.close(); - dir.close(); + IOUtils.close(ir, analyzer, dir); } @Test public void testWithoutPayload() throws IOException { Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwc = newIndexWriterConfig(analyzer); iwc.setMergePolicy(newLogMergePolicy()); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); Map.Entry, Map> res = generateIndexDocuments(atLeast(1000), false, false); @@ -198,14 +201,14 @@ public class DocumentDictionaryTest extends LuceneTestCase { assertTrue(docs.isEmpty()); - ir.close(); - dir.close(); + IOUtils.close(ir, analyzer, dir); } @Test public void testWithContexts() throws IOException { Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwc = newIndexWriterConfig(analyzer); iwc.setMergePolicy(newLogMergePolicy()); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); Map.Entry, Map> res = generateIndexDocuments(atLeast(1000), true, true); @@ -239,14 +242,14 @@ public class DocumentDictionaryTest extends LuceneTestCase { } assertTrue(docs.isEmpty()); - ir.close(); - dir.close(); + IOUtils.close(ir, analyzer, dir); } @Test public void testWithDeletions() throws IOException { Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwc = newIndexWriterConfig(analyzer); iwc.setMergePolicy(newLogMergePolicy()); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); Map.Entry, Map> res = generateIndexDocuments(atLeast(1000), false, false); @@ -296,14 +299,14 @@ public class DocumentDictionaryTest extends LuceneTestCase { } assertTrue(docs.isEmpty()); - ir.close(); - dir.close(); + IOUtils.close(ir, analyzer, dir); } @Test public void testMultiValuedField() throws IOException { Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(random(), new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwc = newIndexWriterConfig(random(), analyzer); iwc.setMergePolicy(newLogMergePolicy()); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); @@ -325,8 +328,7 @@ public class DocumentDictionaryTest extends LuceneTestCase { assertTrue(inputIterator.contexts().equals(nextSuggestion.contexts)); } assertFalse(suggestionsIter.hasNext()); - ir.close(); - dir.close(); + IOUtils.close(ir, analyzer, dir); } private List indexMultiValuedDocuments(int numDocs, RandomIndexWriter writer) throws IOException { diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java index 0bacfbb55a2..12d5c07df80 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Random; import java.util.Set; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -44,6 +45,7 @@ import org.apache.lucene.queries.function.valuesource.SumFloatFunction; import org.apache.lucene.search.spell.Dictionary; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.junit.Test; @@ -84,7 +86,8 @@ public class DocumentValueSourceDictionaryTest extends LuceneTestCase { @Test public void testEmptyReader() throws IOException { Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwc = newIndexWriterConfig(analyzer); iwc.setMergePolicy(newLogMergePolicy()); // Make sure the index is created? RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); @@ -98,14 +101,14 @@ public class DocumentValueSourceDictionaryTest extends LuceneTestCase { assertEquals(inputIterator.weight(), 0); assertNull(inputIterator.payload()); - ir.close(); - dir.close(); + IOUtils.close(ir, analyzer, dir); } @Test public void testBasic() throws IOException { Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwc = newIndexWriterConfig(analyzer); iwc.setMergePolicy(newLogMergePolicy()); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); Map docs = generateIndexDocuments(atLeast(100)); @@ -130,14 +133,14 @@ public class DocumentValueSourceDictionaryTest extends LuceneTestCase { assertTrue(inputIterator.payload().equals(doc.getField(PAYLOAD_FIELD_NAME).binaryValue())); } assertTrue(docs.isEmpty()); - ir.close(); - dir.close(); + IOUtils.close(ir, analyzer, dir); } @Test public void testWithContext() throws IOException { Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwc = newIndexWriterConfig(analyzer); iwc.setMergePolicy(newLogMergePolicy()); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); Map docs = generateIndexDocuments(atLeast(100)); @@ -167,14 +170,14 @@ public class DocumentValueSourceDictionaryTest extends LuceneTestCase { assertEquals(originalCtxs, inputIterator.contexts()); } assertTrue(docs.isEmpty()); - ir.close(); - dir.close(); + IOUtils.close(ir, analyzer, dir); } @Test public void testWithoutPayload() throws IOException { Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwc = newIndexWriterConfig(analyzer); iwc.setMergePolicy(newLogMergePolicy()); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); Map docs = generateIndexDocuments(atLeast(100)); @@ -199,14 +202,14 @@ public class DocumentValueSourceDictionaryTest extends LuceneTestCase { assertEquals(inputIterator.payload(), null); } assertTrue(docs.isEmpty()); - ir.close(); - dir.close(); + IOUtils.close(ir, analyzer, dir); } @Test public void testWithDeletions() throws IOException { Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwc = newIndexWriterConfig(analyzer); iwc.setMergePolicy(newLogMergePolicy()); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); Map docs = generateIndexDocuments(atLeast(100)); @@ -252,15 +255,14 @@ public class DocumentValueSourceDictionaryTest extends LuceneTestCase { assertTrue(inputIterator.payload().equals(doc.getField(PAYLOAD_FIELD_NAME).binaryValue())); } assertTrue(docs.isEmpty()); - ir.close(); - dir.close(); + IOUtils.close(ir, analyzer, dir); } @Test public void testWithValueSource() throws IOException { - Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriterConfig iwc = newIndexWriterConfig(analyzer); iwc.setMergePolicy(newLogMergePolicy()); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); Map docs = generateIndexDocuments(atLeast(100)); @@ -281,8 +283,7 @@ public class DocumentValueSourceDictionaryTest extends LuceneTestCase { assertTrue(inputIterator.payload().equals(doc.getField(PAYLOAD_FIELD_NAME).binaryValue())); } assertTrue(docs.isEmpty()); - ir.close(); - dir.close(); + IOUtils.close(ir, analyzer, dir); } } diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java index c8fc92d652c..b99f8cbeb0e 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java @@ -43,6 +43,7 @@ import org.apache.lucene.search.suggest.InputArrayIterator; import org.apache.lucene.search.suggest.Lookup.LookupResult; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.junit.Test; @@ -117,6 +118,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { testConstructorDefaults(suggester, keys, a, false, true); suggester.close(); + a.close(); } private void testConstructorDefaults(AnalyzingInfixSuggester suggester, Input[] keys, Analyzer a, @@ -158,6 +160,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { assertEquals(new BytesRef("foobaz"), results.get(0).payload); assertEquals(2, suggester.getCount()); suggester.close(); + a.close(); } /** Used to return highlighted result; see {@link @@ -239,6 +242,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { assertEquals(10, results.get(0).value); assertEquals(new BytesRef("foobaz"), results.get(0).payload); suggester.close(); + a.close(); } public String toString(List fragments) { @@ -320,6 +324,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { suggester = new AnalyzingInfixSuggester(newFSDirectory(tempDir), a, a, minPrefixLength, false); } suggester.close(); + a.close(); } public void testHighlight() throws Exception { @@ -335,6 +340,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { assertEquals("a penny saved is a penny earned", results.get(0).key); assertEquals("a penny saved is a penny earned", results.get(0).highlightKey); suggester.close(); + a.close(); } public void testHighlightCaseChange() throws Exception { @@ -367,6 +373,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { assertEquals("a Penny saved is a penny earned", results.get(0).key); assertEquals("a Penny saved is a penny earned", results.get(0).highlightKey); suggester.close(); + a.close(); } public void testDoubleClose() throws Exception { @@ -379,6 +386,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { suggester.build(new InputArrayIterator(keys)); suggester.close(); suggester.close(); + a.close(); } public void testSuggestStopFilter() throws Exception { @@ -413,6 +421,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { assertEquals("a bob for apples", results.get(0).key); assertEquals("a bob for apples", results.get(0).highlightKey); suggester.close(); + IOUtils.close(suggester, indexAnalyzer, queryAnalyzer); } public void testEmptyAtStart() throws Exception { @@ -456,6 +465,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { assertEquals(new BytesRef("foobaz"), results.get(0).payload); suggester.close(); + a.close(); } public void testBothExactAndPrefix() throws Exception { @@ -472,6 +482,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { assertEquals(10, results.get(0).value); assertEquals(new BytesRef("foobaz"), results.get(0).payload); suggester.close(); + a.close(); } private static String randomText() { @@ -741,6 +752,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { lookupThread.finish(); suggester.close(); + a.close(); } private static String hilite(boolean lastPrefix, String[] inputTerms, String[] queryTerms) { @@ -856,6 +868,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { assertEquals(10, results.get(1).value); assertEquals(new BytesRef("foobaz"), results.get(1).payload); suggester.close(); + a.close(); } public void testNRTWithParallelAdds() throws IOException, InterruptedException { @@ -896,6 +909,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { assertEquals("python", results.get(0).key); suggester.close(); + a.close(); } private class IndexDocument implements Runnable { @@ -1153,6 +1167,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { assertTrue(result.contexts.contains(new BytesRef("baz"))); suggester.close(); + a.close(); } } @@ -1170,6 +1185,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase { dir.close(); suggester.close(); + a.close(); } private String pfmToString(AnalyzingInfixSuggester suggester, String surface, String prefix) throws IOException { diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java index 8393759f4db..2b487bf820a 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java @@ -53,6 +53,7 @@ import org.apache.lucene.search.suggest.Input; import org.apache.lucene.search.suggest.InputArrayIterator; import org.apache.lucene.util.AttributeFactory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -71,7 +72,8 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { new Input("barbara", 1) ); - AnalyzingSuggester suggester = new AnalyzingSuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false); + AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer); suggester.build(new InputArrayIterator(keys)); // top N of 2, but only foo is available @@ -104,6 +106,8 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { assertEquals(10, results.get(1).value, 0.01F); assertEquals("barbara", results.get(2).key.toString()); assertEquals(6, results.get(2).value, 0.01F); + + analyzer.close(); } public void testKeywordWithPayloads() throws Exception { @@ -115,7 +119,8 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { new Input("bar", 8, new BytesRef("should also be deduplicated")), new Input("barbara", 6, new BytesRef("for all the fish"))); - AnalyzingSuggester suggester = new AnalyzingSuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false); + AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer); suggester.build(new InputArrayIterator(keys)); for (int i = 0; i < 2; i++) { // top N of 2, but only foo is available @@ -156,6 +161,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { assertEquals(6, results.get(2).value, 0.01F); assertEquals(new BytesRef("for all the fish"), results.get(2).payload); } + analyzer.close(); } public void testRandomRealisticKeys() throws IOException { @@ -173,7 +179,9 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { mapping.put(title, Long.valueOf(randomWeight)); } } - AnalyzingSuggester analyzingSuggester = new AnalyzingSuggester(new MockAnalyzer(random()), new MockAnalyzer(random()), + Analyzer indexAnalyzer = new MockAnalyzer(random()); + Analyzer queryAnalyzer = new MockAnalyzer(random()); + AnalyzingSuggester analyzingSuggester = new AnalyzingSuggester(indexAnalyzer, queryAnalyzer, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, random().nextBoolean()); boolean doPayloads = random().nextBoolean(); if (doPayloads) { @@ -198,7 +206,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { } } - lineFile.close(); + IOUtils.close(lineFile, indexAnalyzer, queryAnalyzer); } // TODO: more tests @@ -232,6 +240,8 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { assertEquals(1, results.size()); assertEquals("the ghost of christmas past", results.get(0).key.toString()); assertEquals(50, results.get(0).value, 0.01F); + + standard.close(); } public void testEmpty() throws Exception { @@ -241,6 +251,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { List result = suggester.lookup("a", false, 20); assertTrue(result.isEmpty()); + standard.close(); } public void testNoSeps() throws Exception { @@ -265,6 +276,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { // complete to "abcd", which has higher weight so should // appear first: assertEquals("abcd", r.get(0).key.toString()); + a.close(); } public void testGraphDups() throws Exception { @@ -330,6 +342,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { assertEquals(50, results.get(0).value); assertEquals("wi fi network is fast", results.get(1).key); assertEquals(10, results.get(1).value); + analyzer.close(); } public void testInputPathRequired() throws Exception { @@ -388,6 +401,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { suggester.build(new InputArrayIterator(keys)); List results = suggester.lookup("ab x", false, 1); assertTrue(results.size() == 1); + analyzer.close(); } private static Token token(String term, int posInc, int posLength) { @@ -492,6 +506,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { } } } + a.close(); } public void testNonExactFirst() throws Exception { @@ -529,6 +544,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { } } } + a.close(); } // Holds surface form separately: @@ -867,6 +883,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { } } } + a.close(); } public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception { @@ -881,6 +898,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { assertEquals(60, results.get(0).value); assertEquals("a ", results.get(1).key); assertEquals(50, results.get(1).value); + a.close(); } public void testQueueExhaustion() throws Exception { @@ -895,6 +913,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { })); suggester.lookup("a", false, 4); + a.close(); } public void testExactFirstMissingResult() throws Exception { @@ -941,6 +960,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { assertEquals(4, results.get(1).value); assertEquals("a b", results.get(2).key); assertEquals(3, results.get(2).value); + a.close(); } public void testDupSurfaceFormsMissingResults() throws Exception { @@ -999,6 +1019,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { assertEquals(6, results.get(0).value); assertEquals("nellie", results.get(1).key); assertEquals(5, results.get(1).value); + a.close(); } public void testDupSurfaceFormsMissingResults2() throws Exception { @@ -1068,6 +1089,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { assertEquals(6, results.get(0).value); assertEquals("b", results.get(1).key); assertEquals(5, results.get(1).value); + a.close(); } public void test0ByteKeys() throws Exception { @@ -1113,6 +1135,8 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { new Input("a a", 50), new Input("a b", 50), })); + + a.close(); } public void testDupSurfaceFormsMissingResults3() throws Exception { @@ -1126,6 +1150,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { new Input("a b", 5), })); assertEquals("[a a/7, a c/6, a b/5]", suggester.lookup("a", false, 3).toString()); + a.close(); } public void testEndingSpace() throws Exception { @@ -1137,6 +1162,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { })); assertEquals("[isla de muerta/8, i love lucy/7]", suggester.lookup("i", false, 3).toString()); assertEquals("[i love lucy/7]", suggester.lookup("i ", false, 3).toString()); + a.close(); } public void testTooManyExpansions() throws Exception { @@ -1166,6 +1192,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, 1, true); suggester.build(new InputArrayIterator(new Input[] {new Input("a", 1)})); assertEquals("[a/1]", suggester.lookup("a", false, 1).toString()); + a.close(); } public void testIllegalLookupArgument() throws Exception { @@ -1186,6 +1213,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { } catch (IllegalArgumentException e) { // expected } + a.close(); } static final Iterable shuffle(Input...values) { @@ -1209,5 +1237,6 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { } catch (IllegalArgumentException iae) { // expected } + a.close(); } } diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java index 5847adcf10b..dad44081837 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java @@ -72,6 +72,7 @@ public class FuzzySuggesterTest extends LuceneTestCase { assertEquals("foo bar boo far", results.get(0).key.toString()); assertEquals(12, results.get(0).value, 0.01F); } + analyzer.close(); } public void testNonLatinRandomEdits() throws IOException { @@ -93,6 +94,7 @@ public class FuzzySuggesterTest extends LuceneTestCase { assertEquals("фуу бар буу фар", results.get(0).key.toString()); assertEquals(12, results.get(0).value, 0.01F); } + analyzer.close(); } /** this is basically the WFST test ported to KeywordAnalyzer. so it acts the same */ @@ -104,7 +106,8 @@ public class FuzzySuggesterTest extends LuceneTestCase { new Input("barbara", 6) }; - FuzzySuggester suggester = new FuzzySuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false); + FuzzySuggester suggester = new FuzzySuggester(analyzer); suggester.build(new InputArrayIterator(keys)); List results = suggester.lookup(TestUtil.stringToCharSequence("bariar", random()), false, 2); @@ -166,6 +169,8 @@ public class FuzzySuggesterTest extends LuceneTestCase { assertEquals(10, results.get(1).value, 0.01F); assertEquals("barbara", results.get(2).key.toString()); assertEquals(6, results.get(2).value, 0.01F); + + analyzer.close(); } /** @@ -197,6 +202,8 @@ public class FuzzySuggesterTest extends LuceneTestCase { assertEquals(1, results.size()); assertEquals("the ghost of christmas past", results.get(0).key.toString()); assertEquals(50, results.get(0).value, 0.01F); + + standard.close(); } public void testNoSeps() throws Exception { @@ -221,6 +228,7 @@ public class FuzzySuggesterTest extends LuceneTestCase { // complete to "abcd", which has higher weight so should // appear first: assertEquals("abcd", r.get(0).key.toString()); + a.close(); } public void testGraphDups() throws Exception { @@ -286,14 +294,17 @@ public class FuzzySuggesterTest extends LuceneTestCase { assertEquals(50, results.get(0).value); assertEquals("wi fi network is fast", results.get(1).key); assertEquals(10, results.get(1).value); + analyzer.close(); } public void testEmpty() throws Exception { - FuzzySuggester suggester = new FuzzySuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false); + FuzzySuggester suggester = new FuzzySuggester(analyzer); suggester.build(new InputArrayIterator(new Input[0])); List result = suggester.lookup("a", false, 20); assertTrue(result.isEmpty()); + analyzer.close(); } public void testInputPathRequired() throws Exception { @@ -352,6 +363,7 @@ public class FuzzySuggesterTest extends LuceneTestCase { suggester.build(new InputArrayIterator(keys)); List results = suggester.lookup("ab x", false, 1); assertTrue(results.size() == 1); + analyzer.close(); } private static Token token(String term, int posInc, int posLength) { @@ -451,6 +463,7 @@ public class FuzzySuggesterTest extends LuceneTestCase { } } } + a.close(); } public void testNonExactFirst() throws Exception { @@ -488,6 +501,7 @@ public class FuzzySuggesterTest extends LuceneTestCase { } } } + a.close(); } // Holds surface form separately: @@ -820,6 +834,7 @@ public class FuzzySuggesterTest extends LuceneTestCase { assertEquals(matches.get(hit).value, r.get(hit).value, 0f); } } + a.close(); } public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception { @@ -841,6 +856,7 @@ public class FuzzySuggesterTest extends LuceneTestCase { assertEquals(60, results.get(0).value); assertEquals("a ", results.get(1).key); assertEquals(50, results.get(1).value); + a.close(); } public void testEditSeps() throws Exception { @@ -861,6 +877,7 @@ public class FuzzySuggesterTest extends LuceneTestCase { assertEquals("[foo bar baz/50]", suggester.lookup("foobarbaz", false, 5).toString()); assertEquals("[barbaz/60, barbazfoo/10]", suggester.lookup("bar baz", false, 5).toString()); assertEquals("[barbazfoo/10]", suggester.lookup("bar baz foo", false, 5).toString()); + a.close(); } @SuppressWarnings("fallthrough") @@ -1003,6 +1020,7 @@ public class FuzzySuggesterTest extends LuceneTestCase { } assertEquals(expected.size(), actual.size()); } + a.close(); } private List slowFuzzyMatch(int prefixLen, int maxEdits, boolean allowTransposition, List answers, String frag) { diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java index 84bdf2bfdd0..4d678c303d2 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java @@ -95,6 +95,7 @@ public class TestFreeTextSuggester extends LuceneTestCase { is.close(); assertEquals(2, sug.getCount()); } + a.close(); } public void testIllegalByteDuringBuild() throws Exception { @@ -103,13 +104,15 @@ public class TestFreeTextSuggester extends LuceneTestCase { Iterable keys = AnalyzingSuggesterTest.shuffle( new Input("foo\u001ebar baz", 50) ); - FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + FreeTextSuggester sug = new FreeTextSuggester(analyzer); try { sug.build(new InputArrayIterator(keys)); fail("did not hit expected exception"); } catch (IllegalArgumentException iae) { // expected } + analyzer.close(); } public void testIllegalByteDuringQuery() throws Exception { @@ -118,7 +121,8 @@ public class TestFreeTextSuggester extends LuceneTestCase { Iterable keys = AnalyzingSuggesterTest.shuffle( new Input("foo bar baz", 50) ); - FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + FreeTextSuggester sug = new FreeTextSuggester(analyzer); sug.build(new InputArrayIterator(keys)); try { @@ -127,6 +131,7 @@ public class TestFreeTextSuggester extends LuceneTestCase { } catch (IllegalArgumentException iae) { // expected } + analyzer.close(); } @Ignore @@ -134,7 +139,8 @@ public class TestFreeTextSuggester extends LuceneTestCase { final LineFileDocs lfd = new LineFileDocs(null, "/lucenedata/enwiki/enwiki-20120502-lines-1k.txt", false); // Skip header: lfd.nextDoc(); - FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random())); + Analyzer analyzer = new MockAnalyzer(random()); + FreeTextSuggester sug = new FreeTextSuggester(analyzer); sug.build(new InputIterator() { private int count; @@ -190,6 +196,7 @@ public class TestFreeTextSuggester extends LuceneTestCase { System.out.println(" " + result); } } + analyzer.close(); } // Make sure you can suggest based only on unigram model: @@ -204,6 +211,7 @@ public class TestFreeTextSuggester extends LuceneTestCase { // Sorts first by count, descending, second by term, ascending assertEquals("bar/0.22 baz/0.11 bee/0.11 blah/0.11 boo/0.11", toString(sug.lookup("b", 10))); + a.close(); } // Make sure the last token is not duplicated @@ -216,6 +224,7 @@ public class TestFreeTextSuggester extends LuceneTestCase { sug.build(new InputArrayIterator(keys)); assertEquals("foo bar/1.00", toString(sug.lookup("foo b", 10))); + a.close(); } // Lookup of just empty string produces unicode only matches: @@ -232,6 +241,7 @@ public class TestFreeTextSuggester extends LuceneTestCase { } catch (IllegalArgumentException iae) { // expected } + a.close(); } // With one ending hole, ShingleFilter produces "of _" and @@ -259,6 +269,7 @@ public class TestFreeTextSuggester extends LuceneTestCase { // prop 0.5: assertEquals("oz/0.20", toString(sug.lookup("wizard o", 10))); + a.close(); } // If the number of ending holes exceeds the ngrams window @@ -282,6 +293,7 @@ public class TestFreeTextSuggester extends LuceneTestCase { sug.build(new InputArrayIterator(keys)); assertEquals("", toString(sug.lookup("wizard of of", 10))); + a.close(); } private static Comparator byScoreThenKey = new Comparator() { @@ -578,6 +590,7 @@ public class TestFreeTextSuggester extends LuceneTestCase { assertEquals(expected.toString(), actual.toString()); } + a.close(); } private static String getZipfToken(String[] tokens) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java index 19c3fc4996a..cd5c18c3946 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java @@ -29,6 +29,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.NullInfoStream; @@ -49,6 +50,7 @@ public class RandomIndexWriter implements Closeable { private double flushAtFactor = 1.0; private boolean getReaderCalled; private final Codec codec; // sugar + private final Analyzer analyzer; // only if WE created it (then we close it) /** Returns an indexwriter that randomly mixes up thread scheduling (by yielding at test points) */ public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random r) throws IOException { @@ -73,7 +75,7 @@ public class RandomIndexWriter implements Closeable { /** create a RandomIndexWriter with a random config: Uses MockAnalyzer */ public RandomIndexWriter(Random r, Directory dir) throws IOException { - this(r, dir, LuceneTestCase.newIndexWriterConfig(r, new MockAnalyzer(r))); + this(r, dir, LuceneTestCase.newIndexWriterConfig(r, new MockAnalyzer(r)), true); } /** create a RandomIndexWriter with a random config */ @@ -83,10 +85,19 @@ public class RandomIndexWriter implements Closeable { /** create a RandomIndexWriter with the provided config */ public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c) throws IOException { + this(r, dir, c, false); + } + + private RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c, boolean closeAnalyzer) throws IOException { // TODO: this should be solved in a different way; Random should not be shared (!). this.r = new Random(r.nextLong()); w = mockIndexWriter(dir, c, r); flushAt = TestUtil.nextInt(r, 10, 1000); + if (closeAnalyzer) { + analyzer = w.getAnalyzer(); + } else { + analyzer = null; + } codec = w.getConfig().getCodec(); if (LuceneTestCase.VERBOSE) { System.out.println("RIW dir=" + dir + " config=" + w.getConfig()); @@ -361,7 +372,7 @@ public class RandomIndexWriter implements Closeable { w.commit(); } } - w.close(); + IOUtils.close(w, analyzer); } /** diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java index 379733522d3..ee6c3900d74 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java @@ -17,6 +17,7 @@ package org.apache.lucene.search; * limitations under the License. */ +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -51,6 +52,7 @@ public abstract class BaseExplanationTestCase extends LuceneTestCase { protected static IndexSearcher searcher; protected static IndexReader reader; protected static Directory directory; + protected static Analyzer analyzer; public static final String KEY = "KEY"; // boost on this field is the same as the iterator for the doc @@ -65,12 +67,15 @@ public abstract class BaseExplanationTestCase extends LuceneTestCase { reader = null; directory.close(); directory = null; + analyzer.close(); + analyzer = null; } @BeforeClass public static void beforeClassTestExplanations() throws Exception { directory = newDirectory(); - RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); + analyzer = new MockAnalyzer(random()); + RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(analyzer).setMergePolicy(newLogMergePolicy())); for (int i = 0; i < docFields.length; i++) { Document doc = new Document(); doc.add(newStringField(KEY, ""+i, Field.Store.NO)); diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java b/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java index e0d06dcd9dd..911fa5afae9 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java @@ -73,10 +73,8 @@ public class LineFileDocs implements Closeable { @Override public synchronized void close() throws IOException { - if (reader != null) { - reader.close(); - reader = null; - } + IOUtils.close(reader, threadDocs); + reader = null; } private long randomSeekPos(Random random, long size) { @@ -205,7 +203,7 @@ public class LineFileDocs implements Closeable { } } - private final ThreadLocal threadDocs = new ThreadLocal<>(); + private final CloseableThreadLocal threadDocs = new CloseableThreadLocal<>(); /** Note: Document instance is re-used per-thread */ public Document nextDoc() throws IOException {