From 6ec68ddbe07804663fef3a4b8a849e8bacf20d64 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 16 Nov 2016 04:53:50 -0500 Subject: [PATCH] LUCENE-7562: don't throw NPE when encountering a level 2 ghost field --- lucene/CHANGES.txt | 3 ++ .../document/CompletionFieldsConsumer.java | 4 ++ .../document/TestPrefixCompletionQuery.java | 49 ++++++++++++++++--- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index bdc118b95f2..4ef348c0f2c 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -65,6 +65,9 @@ Bug Fixes * LUCENE-7547: JapaneseTokenizerFactory was failing to close the dictionary file it opened (Markus via Mike McCandless) +* LUCENE-7562: CompletionFieldsConsumer sometimes throws + NullPointerException on ghost fields (Oliver Eilhard via Mike McCandless) + Improvements * LUCENE-6824: TermAutomatonQuery now rewrites to TermQuery, diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java index 672c5824463..9df9d605197 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java @@ -86,6 +86,10 @@ final class CompletionFieldsConsumer extends FieldsConsumer { for (String field : fields) { CompletionTermWriter termWriter = new CompletionTermWriter(); Terms terms = fields.terms(field); + if (terms == null) { + // this can happen from ghost fields, where the incoming Fields iterator claims a field exists but it does not + continue; + } TermsEnum termsEnum = terms.iterator(); // write terms diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java index 20561b3d37f..f5bacefc638 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java @@ -24,9 +24,12 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenFilter; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.SortedNumericDocValues; @@ -38,7 +41,6 @@ import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.LuceneTestCase; import org.junit.After; import org.junit.Before; -import org.junit.Test; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.search.suggest.document.TestSuggestField.Entry; @@ -112,7 +114,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase { dir.close(); } - @Test public void testSimple() throws Exception { Analyzer analyzer = new MockAnalyzer(random()); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field")); @@ -141,7 +142,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase { iw.close(); } - @Test public void testMostlyFilteredOutDocuments() throws Exception { Analyzer analyzer = new MockAnalyzer(random()); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field")); @@ -188,7 +188,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase { iw.close(); } - @Test public void testDocFiltering() throws Exception { Analyzer analyzer = new MockAnalyzer(random()); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field")); @@ -230,7 +229,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase { iw.close(); } - @Test public void testAnalyzerWithoutPreservePosAndSep() throws Exception { Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET); CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer, false, false); @@ -254,7 +252,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase { iw.close(); } - @Test public void testAnalyzerWithSepAndNoPreservePos() throws Exception { Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET); CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer, true, false); @@ -278,7 +275,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase { iw.close(); } - @Test public void testAnalyzerWithPreservePosAndNoSep() throws Exception { Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET); CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer, false, true); @@ -302,4 +298,43 @@ public class TestPrefixCompletionQuery extends LuceneTestCase { iw.close(); } + public void testGhostField() throws Exception { + Analyzer analyzer = new MockAnalyzer(random()); + IndexWriter iw = new IndexWriter(dir, iwcWithSuggestField(analyzer, "suggest_field", "suggest_field2", "suggest_field3")); + + Document document = new Document(); + document.add(new StringField("id", "0", Field.Store.NO)); + document.add(new SuggestField("suggest_field", "apples", 3)); + iw.addDocument(document); + // need another document so whole segment isn't deleted + iw.addDocument(new Document()); + iw.commit(); + + document = new Document(); + document.add(new StringField("id", "1", Field.Store.NO)); + document.add(new SuggestField("suggest_field2", "apples", 3)); + iw.addDocument(document); + iw.commit(); + + iw.deleteDocuments(new Term("id", "0")); + // first force merge is OK + iw.forceMerge(1); + + // second force merge causes MultiFields to include "suggest_field" in its iteration, yet a null Terms is returned (no documents have + // this field anymore) + iw.addDocument(new Document()); + iw.forceMerge(1); + + DirectoryReader reader = DirectoryReader.open(iw); + SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader); + + PrefixCompletionQuery query = new PrefixCompletionQuery(analyzer, new Term("suggest_field", "app")); + assertEquals(0, indexSearcher.suggest(query, 3).totalHits); + + query = new PrefixCompletionQuery(analyzer, new Term("suggest_field2", "app")); + assertSuggestions(indexSearcher.suggest(query, 3), new Entry("apples", 3)); + + reader.close(); + iw.close(); + } }