From a411d8af6cd1198d6d73bb7a7bb9d4747faabc08 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 25 Feb 2015 01:17:13 +0000 Subject: [PATCH] LUCENE-6271: initial patch, postings() funcs switched over to not return null, callers marked with nocommits that check against null git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene6271@1662157 13f79535-47bb-0310-9956-ffa450edef68 --- .../codecs/blockterms/BlockTermsReader.java | 8 ------- .../blocktreeords/OrdsIntersectTermsEnum.java | 8 ------- .../blocktreeords/OrdsSegmentTermsEnum.java | 8 ------- .../codecs/memory/DirectPostingsFormat.java | 24 +++++++++++++------ .../codecs/memory/MemoryPostingsFormat.java | 7 +++--- .../simpletext/SimpleTextFieldsReader.java | 7 ++---- .../SimpleTextTermVectorsReader.java | 13 +++++----- .../CompressingTermVectorsReader.java | 6 ----- .../lucene50/Lucene50PostingsReader.java | 19 ++++++--------- .../org/apache/lucene/index/CheckIndex.java | 3 +++ .../org/apache/lucene/index/MultiFields.java | 3 +-- .../apache/lucene/index/MultiTermsEnum.java | 22 ++++++----------- .../lucene/search/MultiPhraseQuery.java | 2 ++ .../org/apache/lucene/search/PhraseQuery.java | 1 + .../lucene/search/spans/SpanTermQuery.java | 1 + .../org/apache/lucene/index/TestCodecs.java | 1 + .../apache/lucene/index/TestPostingsEnum.java | 4 ++++ .../lucene/index/TestStressIndexing2.java | 2 ++ .../lucene/index/TestTermVectorsReader.java | 1 + .../PostingsHighlighter.java | 1 + .../vectorhighlight/FieldTermStack.java | 1 + .../lucene/index/SortingLeafReader.java | 14 +++++------ .../lucene/search/TermAutomatonQuery.java | 1 + .../lucene/index/AssertingLeafReader.java | 5 ++-- .../index/BasePostingsFormatTestCase.java | 1 + .../index/BaseTermVectorsFormatTestCase.java | 1 + .../apache/lucene/util/LuceneTestCase.java | 1 + .../java/org/apache/lucene/util/TestUtil.java | 1 + .../component/QueryElevationComponent.java | 1 + .../apache/solr/search/SolrIndexSearcher.java | 2 ++ 30 files changed, 76 insertions(+), 93 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java index d50c2d24155..83914c51585 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java @@ -652,14 +652,6 @@ public class BlockTermsReader extends FieldsProducer { @Override public PostingsEnum postings(Bits liveDocs, PostingsEnum reuse, int flags) throws IOException { - - if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { - if (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { - // Positions were not indexed: - return null; - } - } - //System.out.println("BTR.docs this=" + this); decodeMetaData(); //System.out.println("BTR.docs: state.docFreq=" + state.docFreq); diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java index c267f9dbb75..3cf1c26b9bf 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java @@ -203,14 +203,6 @@ final class OrdsIntersectTermsEnum extends TermsEnum { @Override public PostingsEnum postings(Bits skipDocs, PostingsEnum reuse, int flags) throws IOException { - - if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { - if (fr.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { - // Positions were not indexed: - return null; - } - } - currentFrame.decodeMetaData(); return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.termState, skipDocs, reuse, flags); } diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java index 32455f9ff48..9073a86d36a 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java @@ -924,14 +924,6 @@ public final class OrdsSegmentTermsEnum extends TermsEnum { @Override public PostingsEnum postings(Bits skipDocs, PostingsEnum reuse, int flags) throws IOException { - - if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { - if (fr.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { - // Positions were not indexed: - return null; - } - } - assert !eof; //if (DEBUG) { //System.out.println("BTTR.docs seg=" + segment); diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java index 4d73eaa3e1b..84fda889848 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java @@ -860,14 +860,25 @@ public final class DirectPostingsFormat extends PostingsFormat { // TODO: implement reuse // it's hairy! + // TODO: the logic of which enum impl to choose should be refactored to be simpler... if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { - if (!hasPos) { - return null; - } if (terms[termOrd] instanceof LowFreqTerm) { final LowFreqTerm term = ((LowFreqTerm) terms[termOrd]); final int[] postings = term.postings; + if (hasPos == false) { + LowFreqDocsEnumNoPos docsEnum; + if (reuse instanceof LowFreqDocsEnumNoPos) { + docsEnum = (LowFreqDocsEnumNoPos) reuse; + if (!docsEnum.canReuse(liveDocs)) { + docsEnum = new LowFreqDocsEnumNoPos(liveDocs); + } + } else { + docsEnum = new LowFreqDocsEnumNoPos(liveDocs); + } + + return docsEnum.reset(postings); + } final byte[] payloads = term.payloads; return new LowFreqPostingsEnum(liveDocs, hasOffsets, hasPayloads).reset(postings, payloads); } else { @@ -1454,10 +1465,9 @@ public final class DirectPostingsFormat extends PostingsFormat { public PostingsEnum postings(Bits liveDocs, PostingsEnum reuse, int flags) { // TODO: implement reuse // it's hairy! - if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { - if (!hasPos) { - return null; - } + + // TODO: the logic of which enum impl to choose should be refactored to be simpler... + if (hasPos && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { if (terms[termOrd] instanceof LowFreqTerm) { final LowFreqTerm term = ((LowFreqTerm) terms[termOrd]); final int[] postings = term.postings; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java index ecc70cdfbf7..d34e6bb3b16 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java @@ -829,10 +829,9 @@ public final class MemoryPostingsFormat extends PostingsFormat { @Override public PostingsEnum postings(Bits liveDocs, PostingsEnum reuse, int flags) { - if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { - if (field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { - return null; - } + // TODO: the logic of which enum impl to choose should be refactored to be simpler... + boolean hasPositions = field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; + if (hasPositions && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { boolean hasOffsets = field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; decodeMetaData(); FSTPostingsEnum docsAndPositionsEnum; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java index cf23fc00e97..470169066b2 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java @@ -209,11 +209,8 @@ class SimpleTextFieldsReader extends FieldsProducer { @Override public PostingsEnum postings(Bits liveDocs, PostingsEnum reuse, int flags) throws IOException { - if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { - if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { - // Positions were not indexed - return null; - } + boolean hasPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; + if (hasPositions && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { SimpleTextPostingsEnum docsAndPositionsEnum; if (reuse != null && reuse instanceof SimpleTextPostingsEnum && ((SimpleTextPostingsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) { diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java index cc63d47a6e1..b8601c94ba6 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java @@ -388,16 +388,15 @@ public class SimpleTextTermVectorsReader extends TermVectorsReader { @Override public PostingsEnum postings(Bits liveDocs, PostingsEnum reuse, int flags) throws IOException { - + if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { SimpleTVPostings postings = current.getValue(); - if (postings.positions == null && postings.startOffsets == null) { - return null; + if (postings.positions != null || postings.startOffsets != null) { + // TODO: reuse + SimpleTVPostingsEnum e = new SimpleTVPostingsEnum(); + e.reset(liveDocs, postings.positions, postings.startOffsets, postings.endOffsets, postings.payloads); + return e; } - // TODO: reuse - SimpleTVPostingsEnum e = new SimpleTVPostingsEnum(); - e.reset(liveDocs, postings.positions, postings.startOffsets, postings.endOffsets, postings.payloads); - return e; } // TODO: reuse diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java index 7f2e0f7c2d0..1b048390a92 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java @@ -936,12 +936,6 @@ public final class CompressingTermVectorsReader extends TermVectorsReader implem @Override public final PostingsEnum postings(Bits liveDocs, PostingsEnum reuse, int flags) throws IOException { - - if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { - if (positions == null && startOffsets == null) - return null; - } - final TVPostingsEnum docsEnum; if (reuse != null && reuse instanceof TVPostingsEnum) { docsEnum = (TVPostingsEnum) reuse; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java index 4fe53a998bd..ab07b9e51dc 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java @@ -193,8 +193,12 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { @Override public PostingsEnum postings(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, PostingsEnum reuse, int flags) throws IOException { + + boolean indexHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; + boolean indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; + boolean indexHasPayloads = fieldInfo.hasPayloads(); - if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) { + if (indexHasPositions == false || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) { BlockDocsEnum docsEnum; if (reuse instanceof BlockDocsEnum) { docsEnum = (BlockDocsEnum) reuse; @@ -205,17 +209,8 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { docsEnum = new BlockDocsEnum(fieldInfo); } return docsEnum.reset(liveDocs, (IntBlockTermState) termState, flags); - } - - boolean indexHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; - boolean indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; - boolean indexHasPayloads = fieldInfo.hasPayloads(); - - if (!indexHasPositions) - return null; - - if ((!indexHasOffsets || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false) && - (!indexHasPayloads || PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) == false)) { + } else if ((indexHasOffsets == false || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false) && + (indexHasPayloads == false || PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) == false)) { BlockPostingsEnum docsAndPositionsEnum; if (reuse instanceof BlockPostingsEnum) { docsAndPositionsEnum = (BlockPostingsEnum) reuse; diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java index da6ae3e1f35..9749f89dc45 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java @@ -1050,6 +1050,7 @@ public class CheckIndex implements Closeable { sumDocFreq += docFreq; docs = termsEnum.postings(liveDocs, docs); + // nocommit: check null postings = termsEnum.postings(liveDocs, postings, PostingsEnum.ALL); if (hasFreqs == false) { @@ -1389,6 +1390,7 @@ public class CheckIndex implements Closeable { } docs = termsEnum.postings(liveDocs, docs, PostingsEnum.NONE); + // nocommit: null check still needed? how to replace? if (docs == null) { throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]); } @@ -1407,6 +1409,7 @@ public class CheckIndex implements Closeable { totDocFreq += termsEnum.docFreq(); docs = termsEnum.postings(null, docs, PostingsEnum.NONE); + // nocommit: null check still needed? how to replace? if (docs == null) { throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]); } diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java index 3acc1532891..80d83241fef 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java @@ -159,8 +159,7 @@ public final class MultiFields extends Fields { * required. Some codecs may be able to optimize * their implementation when offsets and/or payloads are not * required. This will return null if the field or term does not - * exist or positions were not indexed. See {@link - * TermsEnum#postings(Bits, PostingsEnum,int)}. */ + * exist. See {@link TermsEnum#postings(Bits, PostingsEnum,int)}. */ public static PostingsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, int flags) throws IOException { assert field != null; assert term != null; diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java index 35bca0c5eff..cb6518fd1c4 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java @@ -379,22 +379,14 @@ public final class MultiTermsEnum extends TermsEnum { assert entry.index < docsEnum.subPostingsEnums.length: entry.index + " vs " + docsEnum.subPostingsEnums.length + "; " + subs.length; final PostingsEnum subPostingsEnum = entry.terms.postings(b, docsEnum.subPostingsEnums[entry.index], flags); - if (subPostingsEnum != null) { - docsEnum.subPostingsEnums[entry.index] = subPostingsEnum; - subDocs[upto].postingsEnum = subPostingsEnum; - subDocs[upto].slice = entry.subSlice; - upto++; - } else { - // should this be an error? - return null; // We can't support what is being asked for - } - } - - if (upto == 0) { - return null; - } else { - return docsEnum.reset(subDocs, upto); + assert subPostingsEnum != null; + docsEnum.subPostingsEnums[entry.index] = subPostingsEnum; + subDocs[upto].postingsEnum = subPostingsEnum; + subDocs[upto].slice = entry.subSlice; + upto++; } + + return docsEnum.reset(subDocs, upto); } final static class TermsEnumWithSlice { diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java index 363d7dacb9d..d0fab2bd759 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java @@ -230,6 +230,7 @@ public class MultiPhraseQuery extends Query { termsEnum.seekExact(term.bytes(), termState); postingsEnum = termsEnum.postings(liveDocs, null, PostingsEnum.POSITIONS); + // nocommit: check if (postingsEnum == null) { // term does exist, but has no positions assert termsEnum.postings(liveDocs, null, PostingsEnum.NONE) != null: "termstate found but no term exists in reader"; @@ -483,6 +484,7 @@ class UnionPostingsEnum extends PostingsEnum { } termsEnum.seekExact(term.bytes(), termState); PostingsEnum postings = termsEnum.postings(liveDocs, null, PostingsEnum.POSITIONS); + // nocommit: check if (postings == null) { // term does exist, but has no positions throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")"); diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java index 5b2e661352e..d0247dcda83 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java @@ -312,6 +312,7 @@ public class PhraseQuery extends Query { // PhraseQuery on a field that did not index // positions. + // nocommit: check if (postingsEnum == null) { assert te.seekExact(t.bytes()) : "termstate found but no term exists in reader"; // term does exist, but has no positions diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java index 820363a99eb..5522590f7b9 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java @@ -119,6 +119,7 @@ public class SpanTermQuery extends SpanQuery { if (postings != null) { return new TermSpans(postings, term); } else { + // nocommit: check // term does exist, but has no positions throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run SpanTermQuery (term=" + term.text() + ")"); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java index 73e3555dc59..b31fe1b34a6 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java @@ -458,6 +458,7 @@ public class TestCodecs extends LuceneTestCase { if (postings != null) { docs = postings; } else { + // nocommit: check docs = TestUtil.docs(random(), termsEnum, null, null, PostingsEnum.FREQS); } } else { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPostingsEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestPostingsEnum.java index 9e94ace75a3..eb5bddaca44 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPostingsEnum.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPostingsEnum.java @@ -84,18 +84,22 @@ public class TestPostingsEnum extends LuceneTestCase { // we did not index positions PostingsEnum docsAndPositionsEnum = getOnlySegmentReader(reader).postings(new Term("foo", "bar"), PostingsEnum.POSITIONS); + // nocommit: check assertNull(docsAndPositionsEnum); // we did not index positions docsAndPositionsEnum = getOnlySegmentReader(reader).postings(new Term("foo", "bar"), PostingsEnum.PAYLOADS); + // nocommit: check assertNull(docsAndPositionsEnum); // we did not index positions docsAndPositionsEnum = getOnlySegmentReader(reader).postings(new Term("foo", "bar"), PostingsEnum.OFFSETS); + // nocommit: check assertNull(docsAndPositionsEnum); // we did not index positions docsAndPositionsEnum = getOnlySegmentReader(reader).postings(new Term("foo", "bar"), PostingsEnum.ALL); + // nocommit: check assertNull(docsAndPositionsEnum); iw.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java index fa6bf9cf163..d3ea6e57f5d 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java @@ -397,6 +397,7 @@ public class TestStressIndexing2 extends LuceneTestCase { while((term2 = termsEnum3.next()) != null) { System.out.println(" " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq()); dpEnum = termsEnum3.postings(null, dpEnum, PostingsEnum.ALL); + // nocommit: check if (dpEnum != null) { assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); final int freq = dpEnum.freq(); @@ -620,6 +621,7 @@ public class TestStressIndexing2 extends LuceneTestCase { dpEnum1 = termsEnum1.postings(null, dpEnum1, PostingsEnum.ALL); dpEnum2 = termsEnum2.postings(null, dpEnum2, PostingsEnum.ALL); + // nocommit: check if (dpEnum1 != null) { assertNotNull(dpEnum2); int docID1 = dpEnum1.nextDoc(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java b/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java index c00eb2b5097..53067ca92af 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java @@ -292,6 +292,7 @@ public class TestTermVectorsReader extends LuceneTestCase { //System.out.println("Term: " + term); assertEquals(testTerms[i], term); assertNotNull(termsEnum.postings(null, null)); + // nocommit: check assertNull(termsEnum.postings(null, null, PostingsEnum.ALL)); // no pos } reader.close(); diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java index a9a87620be5..99898604da4 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java @@ -552,6 +552,7 @@ public class PostingsHighlighter { if (!termsEnum.seekExact(terms[i])) { continue; // term not found } + // nocommit: check de = postings[i] = termsEnum.postings(null, null, PostingsEnum.OFFSETS); if (de == null) { // no positions available diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java index 964889c4f03..e1c4450b763 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java @@ -104,6 +104,7 @@ public class FieldTermStack { if (!termSet.contains(term)) { continue; } + // nocommit: check dpEnum = termsEnum.postings(null, dpEnum, PostingsEnum.POSITIONS); if (dpEnum == null) { // null snippet diff --git a/lucene/misc/src/java/org/apache/lucene/index/SortingLeafReader.java b/lucene/misc/src/java/org/apache/lucene/index/SortingLeafReader.java index 5740f62cf35..55092d5c3e1 100644 --- a/lucene/misc/src/java/org/apache/lucene/index/SortingLeafReader.java +++ b/lucene/misc/src/java/org/apache/lucene/index/SortingLeafReader.java @@ -88,13 +88,13 @@ public class SortingLeafReader extends FilterLeafReader { @Override public TermsEnum iterator(final TermsEnum reuse) throws IOException { - return new SortingTermsEnum(in.iterator(reuse), docMap, indexOptions); + return new SortingTermsEnum(in.iterator(reuse), docMap, indexOptions, hasPositions()); } @Override public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException { - return new SortingTermsEnum(in.intersect(compiled, startTerm), docMap, indexOptions); + return new SortingTermsEnum(in.intersect(compiled, startTerm), docMap, indexOptions, hasPositions()); } } @@ -103,11 +103,13 @@ public class SortingLeafReader extends FilterLeafReader { final Sorter.DocMap docMap; // pkg-protected to avoid synthetic accessor methods private final IndexOptions indexOptions; + private final boolean hasPositions; - public SortingTermsEnum(final TermsEnum in, Sorter.DocMap docMap, IndexOptions indexOptions) { + public SortingTermsEnum(final TermsEnum in, Sorter.DocMap docMap, IndexOptions indexOptions, boolean hasPositions) { super(in); this.docMap = docMap; this.indexOptions = indexOptions; + this.hasPositions = hasPositions; } Bits newToOld(final Bits liveDocs) { @@ -132,7 +134,7 @@ public class SortingLeafReader extends FilterLeafReader { @Override public PostingsEnum postings(Bits liveDocs, PostingsEnum reuse, final int flags) throws IOException { - if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { + if (hasPositions && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { final PostingsEnum inReuse; final SortingPostingsEnum wrapReuse; if (reuse != null && reuse instanceof SortingPostingsEnum) { @@ -146,10 +148,6 @@ public class SortingLeafReader extends FilterLeafReader { } final PostingsEnum inDocsAndPositions = in.postings(newToOld(liveDocs), inReuse, flags); - if (inDocsAndPositions == null) { - return null; - } - // we ignore the fact that offsets may be stored but not asked for, // since this code is expected to be used during addIndexes which will // ask for everything. if that assumption changes in the future, we can diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java index 85a33f14135..68b5f52fba8 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java @@ -335,6 +335,7 @@ public class TermAutomatonQuery extends Query { public EnumAndScorer(int termID, PostingsEnum posEnum) { this.termID = termID; + // nocommit: check uses this.posEnum = posEnum; } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java index b4638132095..44f636005f0 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java @@ -159,9 +159,8 @@ public class AssertingLeafReader extends FilterLeafReader { actualReuse = null; } PostingsEnum docs = super.postings(liveDocs, actualReuse, flags); - if (docs == null) { - return null; - } else if (docs == actualReuse) { + assert docs != null; + if (docs == actualReuse) { // codec reused, reset asserting state ((AssertingPostingsEnum)reuse).reset(); return reuse; diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java index 099949ade32..08f9004966d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java @@ -1626,6 +1626,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest } else { docs = termsEnum.postings(null, null, PostingsEnum.POSITIONS); } + // nocommit: can we remove the noPositions and always ask for positions here? int docFreq = 0; long totalTermFreq = 0; while (docs.nextDoc() != PostingsEnum.NO_MORE_DOCS) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java index 9bf2f4f9d2f..ccbc1577660 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java @@ -454,6 +454,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT bits.clear(0); PostingsEnum docsAndPositionsEnum = termsEnum.postings(bits, random().nextBoolean() ? null : this.docsEnum.get(), PostingsEnum.POSITIONS); assertEquals(ft.storeTermVectorOffsets() || ft.storeTermVectorPositions(), docsAndPositionsEnum != null); + // nocommit: check if (docsAndPositionsEnum != null) { assertEquals(PostingsEnum.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java index 586e34a43e0..78750c0e867 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java @@ -1968,6 +1968,7 @@ public abstract class LuceneTestCase extends Assert { */ public void assertDocsAndPositionsEnumEquals(String info, PostingsEnum leftDocs, PostingsEnum rightDocs) throws IOException { if (leftDocs == null || rightDocs == null) { + // nocommit: this should now only be for term or field not existing, is this assert used in that way? assertNull(leftDocs); assertNull(rightDocs); return; diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java index e152348e6e8..94d4e7d66d0 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java @@ -1020,6 +1020,7 @@ public final class TestUtil { default: posFlags = PostingsEnum.ALL; break; } PostingsEnum docsAndPositions = termsEnum.postings(liveDocs, null, posFlags); + // nocommit: check if (docsAndPositions != null) { return docsAndPositions; } diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java index a874d05ce2b..77bc0d636af 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java @@ -693,6 +693,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore term.copyChars(id); if (seen.contains(id) == false && termsEnum.seekExact(term.get())) { postingsEnum = termsEnum.postings(liveDocs, postingsEnum, PostingsEnum.NONE); + // nocommit: check, maybe just remove null check? if (postingsEnum != null) { int docId = postingsEnum.nextDoc(); if (docId == DocIdSetIterator.NO_MORE_DOCS ) continue; // must have been deleted diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 53af8efc0a2..70bafdca214 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -798,6 +798,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn return -1; } PostingsEnum docs = termsEnum.postings(leafReader.getLiveDocs(), null, PostingsEnum.NONE); + // nocommit: check if (docs == null) return -1; int id = docs.nextDoc(); return id == DocIdSetIterator.NO_MORE_DOCS ? -1 : id; @@ -1169,6 +1170,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn FixedBitSet fbs = null; PostingsEnum postingsEnum = deState.termsEnum.postings(deState.liveDocs, deState.postingsEnum, PostingsEnum.NONE); + // nocommit: check if (deState.postingsEnum == null) { deState.postingsEnum = postingsEnum; }