From b55eeb510d79384264eeae4522a4da1cedb99bcc Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Wed, 6 Jul 2011 13:54:38 +0000 Subject: [PATCH] LUCENE-3246: invert getDelDocs to getLiveDocs as pre-cursor for LUCENE-1536 git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1143415 13f79535-47bb-0310-9956-ffa450edef68 --- .../InstantiatedDocsAndPositionsEnum.java | 10 +- .../instantiated/InstantiatedDocsEnum.java | 10 +- .../store/instantiated/InstantiatedIndex.java | 6 +- .../instantiated/InstantiatedIndexReader.java | 6 +- .../instantiated/InstantiatedTermsEnum.java | 8 +- .../store/instantiated/TestIndicesEquals.java | 44 +++--- .../TestUnoptimizedReaderOnConstructor.java | 3 +- .../lucene/index/memory/MemoryIndex.java | 26 ++-- .../lucene/index/FieldNormModifier.java | 6 +- .../lucene/index/MultiPassIndexSplitter.java | 47 +++---- .../apache/lucene/index/PKIndexSplitter.java | 36 ++--- .../lucene/index/TermVectorAccessor.java | 6 +- .../org/apache/lucene/misc/HighFreqTerms.java | 6 +- .../apache/lucene/index/TestNRTManager.java | 4 +- .../lucene/index/TestPKIndexSplitter.java | 4 +- .../apache/lucene/search/DuplicateFilter.java | 8 +- .../org/apache/lucene/search/TermsFilter.java | 4 +- .../lucene/search/DuplicateFilterTest.java | 4 +- .../spatial/tier/CartesianShapeFilter.java | 6 +- .../lucene/index/BufferedDeletesStream.java | 4 +- .../org/apache/lucene/index/CheckIndex.java | 62 ++++---- .../apache/lucene/index/DirectoryReader.java | 4 +- .../org/apache/lucene/index/DocTermOrds.java | 12 +- .../index/DocumentsWriterPerThread.java | 15 +- .../lucene/index/FilterIndexReader.java | 20 +-- .../index/FreqProxTermsWriterPerField.java | 7 +- .../org/apache/lucene/index/IndexReader.java | 38 ++--- .../org/apache/lucene/index/IndexWriter.java | 18 +-- .../index/MultiDocsAndPositionsEnum.java | 2 - .../apache/lucene/index/MultiDocsEnum.java | 2 - .../org/apache/lucene/index/MultiFields.java | 24 ++-- .../org/apache/lucene/index/MultiReader.java | 4 +- .../apache/lucene/index/MultiTermsEnum.java | 44 +++--- .../apache/lucene/index/ParallelReader.java | 6 +- .../apache/lucene/index/SegmentMerger.java | 36 ++--- .../apache/lucene/index/SegmentReader.java | 132 +++++++++--------- .../lucene/index/SegmentWriteState.java | 2 +- .../lucene/index/SlowMultiReaderWrapper.java | 5 +- .../java/org/apache/lucene/index/Terms.java | 16 +-- .../org/apache/lucene/index/TermsEnum.java | 10 +- .../lucene/index/codecs/BlockTermsReader.java | 8 +- .../index/codecs/DocValuesConsumer.java | 12 +- .../lucene/index/codecs/MergeState.java | 2 +- .../index/codecs/PostingsReaderBase.java | 4 +- .../lucene/index/codecs/TermsConsumer.java | 4 +- .../index/codecs/memory/MemoryCodec.java | 24 ++-- .../index/codecs/preflex/PreFlexFields.java | 16 +-- .../index/codecs/preflex/SegmentTermDocs.java | 24 +--- .../pulsing/PulsingPostingsReaderImpl.java | 32 ++--- .../codecs/sep/SepPostingsReaderImpl.java | 26 ++-- .../simpletext/SimpleTextFieldsReader.java | 28 ++-- .../standard/StandardPostingsReader.java | 36 ++--- .../index/values/FixedStraightBytesImpl.java | 2 +- .../apache/lucene/index/values/Floats.java | 2 +- .../apache/lucene/index/values/IntsImpl.java | 2 +- .../index/values/VarStraightBytesImpl.java | 2 +- .../apache/lucene/index/values/Writer.java | 4 +- .../lucene/search/CachingSpanFilter.java | 4 +- .../lucene/search/CachingWrapperFilter.java | 20 +-- .../lucene/search/ConstantScoreQuery.java | 10 +- .../lucene/search/FieldCacheRangeFilter.java | 11 +- .../java/org/apache/lucene/search/Filter.java | 2 +- .../apache/lucene/search/FuzzyTermsEnum.java | 8 +- .../lucene/search/MatchAllDocsQuery.java | 6 +- .../lucene/search/MultiPhraseQuery.java | 12 +- .../search/MultiTermQueryWrapperFilter.java | 12 +- .../org/apache/lucene/search/PhraseQuery.java | 6 +- .../org/apache/lucene/search/TermQuery.java | 7 +- .../lucene/search/cache/DocTermsCreator.java | 4 +- .../search/cache/DocTermsIndexCreator.java | 4 +- .../lucene/search/spans/SpanTermQuery.java | 4 +- .../org/apache/lucene/util/BitVector.java | 127 ++++++++++++++--- .../org/apache/lucene/util/MultiBits.java | 7 +- .../org/apache/lucene/util/OpenBitSet.java | 3 +- .../org/apache/lucene/TestExternalCodecs.java | 27 ++-- .../analysis/TestCachingTokenFilter.java | 6 +- .../index/TestBackwardsCompatibility.java | 4 +- .../lucene/index/TestDeletionPolicy.java | 13 +- .../lucene/index/TestDirectoryReader.java | 4 +- .../test/org/apache/lucene/index/TestDoc.java | 2 +- .../lucene/index/TestDocsAndPositions.java | 8 +- .../lucene/index/TestDocumentWriter.java | 10 +- .../lucene/index/TestFilterIndexReader.java | 6 +- .../apache/lucene/index/TestIndexReader.java | 18 +-- .../lucene/index/TestIndexReaderClone.java | 34 ++--- .../lucene/index/TestIndexReaderDelete.java | 10 +- .../lucene/index/TestIndexReaderReopen.java | 14 +- .../apache/lucene/index/TestIndexWriter.java | 4 +- .../index/TestIndexWriterExceptions.java | 18 +-- .../lucene/index/TestIndexWriterReader.java | 2 +- .../index/TestIndexWriterWithThreads.java | 4 +- .../lucene/index/TestLazyProxSkipping.java | 4 +- .../apache/lucene/index/TestMultiFields.java | 8 +- .../lucene/index/TestMultiLevelSkipList.java | 2 +- .../apache/lucene/index/TestNRTThreads.java | 4 +- .../lucene/index/TestParallelTermEnum.java | 34 ++--- .../org/apache/lucene/index/TestPayloads.java | 12 +- .../lucene/index/TestPerSegmentDeletes.java | 2 +- .../lucene/index/TestSegmentMerger.java | 2 +- .../lucene/index/TestSegmentReader.java | 8 +- .../lucene/index/TestSegmentTermDocs.java | 18 +-- .../lucene/index/TestStressIndexing2.java | 18 +-- .../apache/lucene/index/TestTermdocPerf.java | 2 +- .../lucene/index/TestTransactionRollback.java | 4 +- .../lucene/search/TestPositionIncrement.java | 10 +- .../apache/lucene/search/TestTermVectors.java | 2 +- .../org/apache/lucene/util/TestBitVector.java | 13 +- .../analysis/core/TestClassicAnalyzer.java | 2 +- .../analysis/core/TestKeywordAnalyzer.java | 4 +- .../byTask/tasks/DeleteByPercentTask.java | 10 +- .../benchmark/byTask/tasks/ReadTask.java | 4 +- .../benchmark/byTask/TestPerfTasksLogic.java | 2 +- .../lucene/facet/search/PayloadIterator.java | 4 +- .../search/sampling/TakmiSampleFixer.java | 4 +- .../taxonomy/lucene/LuceneTaxonomyReader.java | 4 +- .../taxonomy/lucene/LuceneTaxonomyWriter.java | 18 +-- .../facet/taxonomy/lucene/ParentArray.java | 4 +- .../lucene/facet/util/ScoredDocIdsUtils.java | 8 +- .../apache/lucene/facet/FacetTestBase.java | 4 +- .../search/TestMultipleCategoryLists.java | 2 +- .../facet/util/TestScoredDocIDsUtils.java | 4 +- .../queries/function/FunctionQuery.java | 8 +- .../lucene/queries/function/ValueSource.java | 2 - .../queries/function/ValueSourceScorer.java | 6 +- .../org/apache/solr/request/SimpleFacets.java | 2 +- .../org/apache/solr/schema/LatLonType.java | 6 +- .../apache/solr/search/JoinQParserPlugin.java | 14 +- .../apache/solr/search/SolrIndexSearcher.java | 14 +- .../solr/search/function/FileFloatSource.java | 2 +- 129 files changed, 866 insertions(+), 771 deletions(-) diff --git a/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsAndPositionsEnum.java b/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsAndPositionsEnum.java index e4eea034bd1..bcdeca32302 100644 --- a/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsAndPositionsEnum.java +++ b/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsAndPositionsEnum.java @@ -23,13 +23,13 @@ import org.apache.lucene.util.BytesRef; public class InstantiatedDocsAndPositionsEnum extends DocsAndPositionsEnum { private int upto; private int posUpto; - private Bits skipDocs; + private Bits liveDocs; private InstantiatedTerm term; protected InstantiatedTermDocumentInformation currentDoc; private final BytesRef payload = new BytesRef(); - public InstantiatedDocsAndPositionsEnum reset(Bits skipDocs, InstantiatedTerm term) { - this.skipDocs = skipDocs; + public InstantiatedDocsAndPositionsEnum reset(Bits liveDocs, InstantiatedTerm term) { + this.liveDocs = liveDocs; this.term = term; upto = -1; return this; @@ -47,7 +47,7 @@ public class InstantiatedDocsAndPositionsEnum extends DocsAndPositionsEnum { return NO_MORE_DOCS; } else { currentDoc = term.getAssociatedDocuments()[upto]; - if (skipDocs == null || !skipDocs.get(currentDoc.getDocument().getDocumentNumber())) { + if (liveDocs == null || liveDocs.get(currentDoc.getDocument().getDocumentNumber())) { posUpto = -1; return docID(); } else { @@ -69,7 +69,7 @@ public class InstantiatedDocsAndPositionsEnum extends DocsAndPositionsEnum { } currentDoc = term.getAssociatedDocuments()[upto]; - if (skipDocs != null && skipDocs.get(currentDoc.getDocument().getDocumentNumber())) { + if (liveDocs != null && !liveDocs.get(currentDoc.getDocument().getDocumentNumber())) { return nextDoc(); } else { posUpto = -1; diff --git a/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsEnum.java b/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsEnum.java index 4bd43cb4027..2657c5129e7 100644 --- a/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsEnum.java +++ b/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsEnum.java @@ -21,12 +21,12 @@ import org.apache.lucene.util.Bits; public class InstantiatedDocsEnum extends DocsEnum { private int upto; - private Bits skipDocs; + private Bits liveDocs; private InstantiatedTerm term; protected InstantiatedTermDocumentInformation currentDoc; - public InstantiatedDocsEnum reset(Bits skipDocs, InstantiatedTerm term) { - this.skipDocs = skipDocs; + public InstantiatedDocsEnum reset(Bits liveDocs, InstantiatedTerm term) { + this.liveDocs = liveDocs; this.term = term; upto = -1; return this; @@ -44,7 +44,7 @@ public class InstantiatedDocsEnum extends DocsEnum { return NO_MORE_DOCS; } else { currentDoc = term.getAssociatedDocuments()[upto]; - if (skipDocs == null || !skipDocs.get(currentDoc.getDocument().getDocumentNumber())) { + if (liveDocs == null || liveDocs.get(currentDoc.getDocument().getDocumentNumber())) { return docID(); } else { return nextDoc(); @@ -65,7 +65,7 @@ public class InstantiatedDocsEnum extends DocsEnum { } currentDoc = term.getAssociatedDocuments()[upto]; - if (skipDocs != null && skipDocs.get(currentDoc.getDocument().getDocumentNumber())) { + if (liveDocs != null && !liveDocs.get(currentDoc.getDocument().getDocumentNumber())) { return nextDoc(); } else { return docID(); diff --git a/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java b/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java index 302480eec07..f486142d032 100644 --- a/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java +++ b/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java @@ -182,9 +182,9 @@ public class InstantiatedIndex } // create documents - final Bits delDocs = MultiFields.getDeletedDocs(sourceIndexReader); + final Bits liveDocs = MultiFields.getLiveDocs(sourceIndexReader); for (int i = 0; i < sourceIndexReader.maxDoc(); i++) { - if (delDocs != null && delDocs.get(i)) { + if (liveDocs != null && !liveDocs.get(i)) { deletedDocuments.set(i); } else { InstantiatedDocument document = new InstantiatedDocument(); @@ -254,7 +254,7 @@ public class InstantiatedIndex // create term-document informations for (InstantiatedTerm term : orderedTerms) { DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(sourceIndexReader, - MultiFields.getDeletedDocs(sourceIndexReader), + MultiFields.getLiveDocs(sourceIndexReader), term.getTerm().field(), new BytesRef(term.getTerm().text())); int position = 0; diff --git a/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java b/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java index 4582a8ae5bc..23d83073339 100644 --- a/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java +++ b/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java @@ -107,11 +107,11 @@ public class InstantiatedIndexReader extends IndexReader { } @Override - public Bits getDeletedDocs() { + public Bits getLiveDocs() { return new Bits() { public boolean get(int n) { - return (index.getDeletedDocuments() != null && index.getDeletedDocuments().get(n)) - || (uncommittedDeletedDocuments != null && uncommittedDeletedDocuments.get(n)); + return !(index.getDeletedDocuments() != null && index.getDeletedDocuments().get(n)) + && !(uncommittedDeletedDocuments != null && uncommittedDeletedDocuments.get(n)); } public int length() { diff --git a/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java b/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java index 645e7e80d2e..c83355c8ec3 100644 --- a/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java +++ b/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java @@ -118,19 +118,19 @@ public class InstantiatedTermsEnum extends TermsEnum { } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) { if (reuse == null || !(reuse instanceof InstantiatedDocsEnum)) { reuse = new InstantiatedDocsEnum(); } - return ((InstantiatedDocsEnum) reuse).reset(skipDocs, terms[upto]); + return ((InstantiatedDocsEnum) reuse).reset(liveDocs, terms[upto]); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) { if (reuse == null || !(reuse instanceof InstantiatedDocsAndPositionsEnum)) { reuse = new InstantiatedDocsAndPositionsEnum(); } - return ((InstantiatedDocsAndPositionsEnum) reuse).reset(skipDocs, terms[upto]); + return ((InstantiatedDocsAndPositionsEnum) reuse).reset(liveDocs, terms[upto]); } @Override diff --git a/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java b/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java index c0545a5dbba..692d1a6f80b 100644 --- a/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java +++ b/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java @@ -138,8 +138,8 @@ public class TestIndicesEquals extends LuceneTestCase { testTermEnum.seekCeil(new BytesRef(t.text())); assertEquals(aprioriTermEnum.term(), testTermEnum.term()); - DocsEnum aprioriTermDocs = aprioriTermEnum.docs(MultiFields.getDeletedDocs(aprioriReader), null); - DocsEnum testTermDocs = testTermEnum.docs(MultiFields.getDeletedDocs(testReader), null); + DocsEnum aprioriTermDocs = aprioriTermEnum.docs(MultiFields.getLiveDocs(aprioriReader), null); + DocsEnum testTermDocs = testTermEnum.docs(MultiFields.getLiveDocs(testReader), null); assertEquals(aprioriTermDocs.nextDoc(), testTermDocs.nextDoc()); assertEquals(aprioriTermDocs.freq(), testTermDocs.freq()); @@ -186,8 +186,8 @@ public class TestIndicesEquals extends LuceneTestCase { assertEquals(aprioriTermEnum.next(), testTermEnum.next()); - aprioriTermDocs = aprioriTermEnum.docs(MultiFields.getDeletedDocs(aprioriReader), aprioriTermDocs); - testTermDocs = testTermEnum.docs(MultiFields.getDeletedDocs(testReader), testTermDocs); + aprioriTermDocs = aprioriTermEnum.docs(MultiFields.getLiveDocs(aprioriReader), aprioriTermDocs); + testTermDocs = testTermEnum.docs(MultiFields.getLiveDocs(testReader), testTermDocs); while (aprioriTermDocs.nextDoc() != DocsEnum.NO_MORE_DOCS) { assertTrue(testTermDocs.nextDoc() != DocsEnum.NO_MORE_DOCS); @@ -309,13 +309,13 @@ public class TestIndicesEquals extends LuceneTestCase { assertEquals(air.numDocs(), tir.numDocs()); assertEquals(air.numDeletedDocs(), tir.numDeletedDocs()); - final Bits aDelDocs = MultiFields.getDeletedDocs(air); - final Bits tDelDocs = MultiFields.getDeletedDocs(tir); - assertTrue((aDelDocs != null && tDelDocs != null) || - (aDelDocs == null && tDelDocs == null)); - if (aDelDocs != null) { + final Bits aLiveDocs = MultiFields.getLiveDocs(air); + final Bits tLiveDocs = MultiFields.getLiveDocs(tir); + assertTrue((aLiveDocs != null && tLiveDocs != null) || + (aLiveDocs == null && tLiveDocs == null)); + if (aLiveDocs != null) { for (int d =0; d 1) { // unset potential duplicates - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(liveDocs, docs); int doc = docs.nextDoc(); if (doc != DocsEnum.NO_MORE_DOCS) { if (keepMode == KM_USE_FIRST_OCCURRENCE) { diff --git a/lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java b/lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java index 82617caa75f..68d75308efb 100644 --- a/lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java +++ b/lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java @@ -63,7 +63,7 @@ public class TermsFilter extends Filter OpenBitSet result=new OpenBitSet(reader.maxDoc()); Fields fields = reader.fields(); BytesRef br = new BytesRef(); - Bits delDocs = reader.getDeletedDocs(); + Bits liveDocs = reader.getLiveDocs(); if (fields != null) { String lastField = null; Terms termsC = null; @@ -80,7 +80,7 @@ public class TermsFilter extends Filter if (terms != null) { br.copy(term.bytes()); if (termsEnum.seekCeil(br) == TermsEnum.SeekStatus.FOUND) { - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(liveDocs, docs); while(docs.nextDoc() != DocsEnum.NO_MORE_DOCS) { result.set(docs.docID()); } diff --git a/lucene/contrib/queries/src/test/org/apache/lucene/search/DuplicateFilterTest.java b/lucene/contrib/queries/src/test/org/apache/lucene/search/DuplicateFilterTest.java index b4a6c8885bb..975239ef081 100644 --- a/lucene/contrib/queries/src/test/org/apache/lucene/search/DuplicateFilterTest.java +++ b/lucene/contrib/queries/src/test/org/apache/lucene/search/DuplicateFilterTest.java @@ -139,7 +139,7 @@ public class DuplicateFilterTest extends LuceneTestCase { Document d=searcher.doc(hits[i].doc); String url=d.get(KEY_FIELD); DocsEnum td = MultiFields.getTermDocsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getLiveDocs(reader), KEY_FIELD, new BytesRef(url)); int lastDoc=0; @@ -163,7 +163,7 @@ public class DuplicateFilterTest extends LuceneTestCase { Document d=searcher.doc(hits[i].doc); String url=d.get(KEY_FIELD); DocsEnum td = MultiFields.getTermDocsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getLiveDocs(reader), KEY_FIELD, new BytesRef(url)); int lastDoc=0; diff --git a/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java b/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java index 6ee8fbeb771..671d7e2f9c6 100644 --- a/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java +++ b/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java @@ -46,7 +46,7 @@ public class CartesianShapeFilter extends Filter { @Override public DocIdSet getDocIdSet(final AtomicReaderContext context) throws IOException { - final Bits delDocs = context.reader.getDeletedDocs(); + final Bits liveDocs = context.reader.getLiveDocs(); final List area = shape.getArea(); final int sz = area.size(); @@ -58,7 +58,7 @@ public class CartesianShapeFilter extends Filter { return new DocIdSet() { @Override public DocIdSetIterator iterator() throws IOException { - return context.reader.termDocsEnum(delDocs, fieldName, bytesRef); + return context.reader.termDocsEnum(liveDocs, fieldName, bytesRef); } @Override @@ -71,7 +71,7 @@ public class CartesianShapeFilter extends Filter { for (int i =0; i< sz; i++) { double boxId = area.get(i).doubleValue(); NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(boxId), 0, bytesRef); - final DocsEnum docsEnum = context.reader.termDocsEnum(delDocs, fieldName, bytesRef); + final DocsEnum docsEnum = context.reader.termDocsEnum(liveDocs, fieldName, bytesRef); if (docsEnum == null) continue; // iterate through all documents // which have this boxId diff --git a/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java b/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java index 2657a1edbcf..01b4a0c8520 100644 --- a/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java +++ b/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java @@ -162,7 +162,7 @@ class BufferedDeletesStream { }; /** Resolves the buffered deleted Term/Query/docIDs, into - * actual deleted docIDs in the deletedDocs BitVector for + * actual deleted docIDs in the liveDocs BitVector for * each SegmentReader. */ public synchronized ApplyDeletesResult applyDeletes(IndexWriter.ReaderPool readerPool, List infos) throws IOException { final long t0 = System.currentTimeMillis(); @@ -399,7 +399,7 @@ class BufferedDeletesStream { // System.out.println(" term=" + term); if (termsEnum.seekExact(term.bytes(), false)) { - DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs); + DocsEnum docsEnum = termsEnum.docs(reader.getLiveDocs(), docs); if (docsEnum != null) { while (true) { diff --git a/lucene/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/src/java/org/apache/lucene/index/CheckIndex.java index ff92206b3bf..07545129eed 100644 --- a/lucene/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/src/java/org/apache/lucene/index/CheckIndex.java @@ -17,12 +17,16 @@ package org.apache.lucene.index; * limitations under the License. */ -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IndexInput; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + import org.apache.lucene.document.AbstractField; // for javadocs import org.apache.lucene.document.Document; import org.apache.lucene.index.codecs.CodecProvider; @@ -30,20 +34,16 @@ import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.values.IndexDocValues; import org.apache.lucene.index.values.ValuesEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; -import java.text.NumberFormat; -import java.io.PrintStream; -import java.io.IOException; -import java.io.File; -import java.util.Collection; -import java.util.Comparator; -import java.util.List; -import java.util.ArrayList; -import java.util.Map; - /** * Basic tool and API to check the health of an index and * write a new segments file that removes reference to @@ -520,13 +520,13 @@ public class CheckIndex { final int numDocs = reader.numDocs(); toLoseDocCount = numDocs; if (reader.hasDeletions()) { - if (reader.deletedDocs.count() != info.getDelCount()) { - throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs deletedDocs.count()=" + reader.deletedDocs.count()); + if (reader.liveDocs.count() != info.docCount - info.getDelCount()) { + throw new RuntimeException("delete count mismatch: info=" + (info.docCount - info.getDelCount()) + " vs reader=" + reader.liveDocs.count()); } - if (reader.deletedDocs.count() > reader.maxDoc()) { - throw new RuntimeException("too many deleted docs: maxDoc()=" + reader.maxDoc() + " vs deletedDocs.count()=" + reader.deletedDocs.count()); + if ((info.docCount-reader.liveDocs.count()) > reader.maxDoc()) { + throw new RuntimeException("too many deleted docs: maxDoc()=" + reader.maxDoc() + " vs del count=" + (info.docCount-reader.liveDocs.count())); } - if (info.docCount - numDocs != info.getDelCount()){ + if (info.docCount - numDocs != info.getDelCount()) { throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.docCount - numDocs)); } segInfoStat.numDeleted = info.docCount - numDocs; @@ -654,7 +654,7 @@ public class CheckIndex { final Status.TermIndexStatus status = new Status.TermIndexStatus(); final int maxDoc = reader.maxDoc(); - final Bits delDocs = reader.getDeletedDocs(); + final Bits liveDocs = reader.getLiveDocs(); final IndexSearcher is = new IndexSearcher(reader); @@ -712,8 +712,8 @@ public class CheckIndex { final int docFreq = terms.docFreq(); status.totFreq += docFreq; - docs = terms.docs(delDocs, docs); - postings = terms.docsAndPositions(delDocs, postings); + docs = terms.docs(liveDocs, docs); + postings = terms.docsAndPositions(liveDocs, postings); if (hasOrd) { long ord = -1; @@ -815,7 +815,7 @@ public class CheckIndex { if (hasPositions) { for(int idx=0;idx<7;idx++) { final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8); - postings = terms.docsAndPositions(delDocs, postings); + postings = terms.docsAndPositions(liveDocs, postings); final int docID = postings.advance(skipDocID); if (docID == DocsEnum.NO_MORE_DOCS) { break; @@ -851,7 +851,7 @@ public class CheckIndex { } else { for(int idx=0;idx<7;idx++) { final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8); - docs = terms.docs(delDocs, docs); + docs = terms.docs(liveDocs, docs); final int docID = docs.advance(skipDocID); if (docID == DocsEnum.NO_MORE_DOCS) { break; @@ -919,7 +919,7 @@ public class CheckIndex { throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed"); } - docs = terms.docs(delDocs, docs); + docs = terms.docs(liveDocs, docs); if (docs == null) { throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]); } @@ -967,9 +967,9 @@ public class CheckIndex { } // Scan stored fields for all documents - final Bits delDocs = reader.getDeletedDocs(); + final Bits liveDocs = reader.getLiveDocs(); for (int j = 0; j < info.docCount; ++j) { - if (delDocs == null || !delDocs.get(j)) { + if (liveDocs == null || liveDocs.get(j)) { status.docCount++; Document doc = reader.document(j); status.totFields += doc.getFields().size(); @@ -1063,9 +1063,9 @@ public class CheckIndex { infoStream.print(" test: term vectors........"); } - final Bits delDocs = reader.getDeletedDocs(); + final Bits liveDocs = reader.getLiveDocs(); for (int j = 0; j < info.docCount; ++j) { - if (delDocs == null || !delDocs.get(j)) { + if (liveDocs == null || liveDocs.get(j)) { status.docCount++; TermFreqVector[] tfv = reader.getTermFreqVectors(j); if (tfv != null) { diff --git a/lucene/src/java/org/apache/lucene/index/DirectoryReader.java b/lucene/src/java/org/apache/lucene/index/DirectoryReader.java index 2b05adf7030..426935b70ab 100644 --- a/lucene/src/java/org/apache/lucene/index/DirectoryReader.java +++ b/lucene/src/java/org/apache/lucene/index/DirectoryReader.java @@ -350,8 +350,8 @@ class DirectoryReader extends IndexReader implements Cloneable { } @Override - public Bits getDeletedDocs() { - throw new UnsupportedOperationException("please use MultiFields.getDeletedDocs, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Bits deletedDocs"); + public Bits getLiveDocs() { + throw new UnsupportedOperationException("please use MultiFields.getLiveDocs, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Bits liveDocs"); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/DocTermOrds.java b/lucene/src/java/org/apache/lucene/index/DocTermOrds.java index 0e93dcbe0c3..0b834ad4127 100644 --- a/lucene/src/java/org/apache/lucene/index/DocTermOrds.java +++ b/lucene/src/java/org/apache/lucene/index/DocTermOrds.java @@ -249,7 +249,7 @@ public class DocTermOrds { boolean testedOrd = false; - final Bits delDocs = MultiFields.getDeletedDocs(reader); + final Bits liveDocs = MultiFields.getLiveDocs(reader); // we need a minimum of 9 bytes, but round up to 12 since the space would // be wasted with most allocators anyway. @@ -312,7 +312,7 @@ public class DocTermOrds { final int df = te.docFreq(); if (df <= maxTermDocFreq) { - docsEnum = te.docs(delDocs, docsEnum); + docsEnum = te.docs(liveDocs, docsEnum); final DocsEnum.BulkReadResult bulkResult = docsEnum.getBulkResult(); @@ -653,13 +653,13 @@ public class DocTermOrds { } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { - return termsEnum.docs(skipDocs, reuse); + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { + return termsEnum.docs(liveDocs, reuse); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { - return termsEnum.docsAndPositions(skipDocs, reuse); + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { + return termsEnum.docsAndPositions(liveDocs, reuse); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index baf5d3bf43b..d2a3e7a0ff4 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -111,13 +111,13 @@ public class DocumentsWriterPerThread { static class FlushedSegment { final SegmentInfo segmentInfo; final BufferedDeletes segmentDeletes; - final BitVector deletedDocuments; + final BitVector liveDocs; private FlushedSegment(SegmentInfo segmentInfo, - BufferedDeletes segmentDeletes, BitVector deletedDocuments) { + BufferedDeletes segmentDeletes, BitVector liveDocs) { this.segmentInfo = segmentInfo; this.segmentDeletes = segmentDeletes; - this.deletedDocuments = deletedDocuments; + this.liveDocs = liveDocs; } } @@ -434,9 +434,10 @@ public class DocumentsWriterPerThread { // happens when an exception is hit processing that // doc, eg if analyzer has some problem w/ the text): if (pendingDeletes.docIDs.size() > 0) { - flushState.deletedDocs = new BitVector(numDocsInRAM); + flushState.liveDocs = new BitVector(numDocsInRAM); + flushState.liveDocs.invertAll(); for(int delDocID : pendingDeletes.docIDs) { - flushState.deletedDocs.set(delDocID); + flushState.liveDocs.clear(delDocID); } pendingDeletes.bytesUsed.addAndGet(-pendingDeletes.docIDs.size() * BufferedDeletes.BYTES_PER_DEL_DOCID); pendingDeletes.docIDs.clear(); @@ -460,7 +461,7 @@ public class DocumentsWriterPerThread { pendingDeletes.terms.clear(); final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.segmentCodecs, fieldInfos.asReadOnly()); if (infoStream != null) { - message("new segment has " + (flushState.deletedDocs == null ? 0 : flushState.deletedDocs.count()) + " deleted docs"); + message("new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.liveDocs.count())) + " deleted docs"); message("new segment has " + (newSegment.getHasVectors() ? "vectors" : "no vectors")); message("flushedFiles=" + newSegment.files()); message("flushed codecs=" + newSegment.getSegmentCodecs()); @@ -489,7 +490,7 @@ public class DocumentsWriterPerThread { doAfterFlush(); success = true; - return new FlushedSegment(newSegment, segmentDeletes, flushState.deletedDocs); + return new FlushedSegment(newSegment, segmentDeletes, flushState.liveDocs); } finally { if (!success) { if (segment != null) { diff --git a/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java b/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java index 355a6932b0e..e7d2190774a 100644 --- a/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java @@ -87,13 +87,13 @@ public class FilterIndexReader extends IndexReader { } @Override - public DocsEnum docs(Bits skipDocs, BytesRef text, DocsEnum reuse) throws IOException { - return in.docs(skipDocs, text, reuse); + public DocsEnum docs(Bits liveDocs, BytesRef text, DocsEnum reuse) throws IOException { + return in.docs(liveDocs, text, reuse); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, BytesRef text, DocsAndPositionsEnum reuse) throws IOException { - return in.docsAndPositions(skipDocs, text, reuse); + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, BytesRef text, DocsAndPositionsEnum reuse) throws IOException { + return in.docsAndPositions(liveDocs, text, reuse); } @Override @@ -172,13 +172,13 @@ public class FilterIndexReader extends IndexReader { } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { - return in.docs(skipDocs, reuse); + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { + return in.docs(liveDocs, reuse); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { - return in.docsAndPositions(skipDocs, reuse); + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { + return in.docsAndPositions(liveDocs, reuse); } @Override @@ -301,8 +301,8 @@ public class FilterIndexReader extends IndexReader { } @Override - public Bits getDeletedDocs() { - return in.getDeletedDocs(); + public Bits getLiveDocs() { + return in.getLiveDocs(); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java b/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java index 59c611634d2..a770c9011bf 100644 --- a/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java +++ b/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java @@ -339,10 +339,11 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem // Mark it deleted. TODO: we could also skip // writing its postings; this would be // deterministic (just for this Term's docs). - if (state.deletedDocs == null) { - state.deletedDocs = new BitVector(state.numDocs); + if (state.liveDocs == null) { + state.liveDocs = new BitVector(state.numDocs); + state.liveDocs.invertAll(); } - state.deletedDocs.set(docID); + state.liveDocs.clear(docID); } // Carefully copy over the prox + payload info, diff --git a/lucene/src/java/org/apache/lucene/index/IndexReader.java b/lucene/src/java/org/apache/lucene/index/IndexReader.java index 7851fca88c5..c3df887ec4e 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/IndexReader.java @@ -962,7 +962,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * requested document is deleted, and therefore asking for a deleted document * may yield unspecified results. Usually this is not required, however you * can test if the doc is deleted by checking the {@link - * Bits} returned from {@link MultiFields#getDeletedDocs}. + * Bits} returned from {@link MultiFields#getLiveDocs}. * * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error @@ -987,7 +987,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * requested document is deleted, and therefore asking for a deleted document * may yield unspecified results. Usually this is not required, however you * can test if the doc is deleted by checking the {@link - * Bits} returned from {@link MultiFields#getDeletedDocs}. + * Bits} returned from {@link MultiFields#getLiveDocs}. * * @param n Get the document at the nth position * @param fieldSelector The {@link FieldSelector} to use to determine what @@ -1136,7 +1136,7 @@ public abstract class IndexReader implements Cloneable,Closeable { /** Returns {@link DocsEnum} for the specified field & * term. This may return null, if either the field or * term does not exist. */ - public DocsEnum termDocsEnum(Bits skipDocs, String field, BytesRef term) throws IOException { + public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term) throws IOException { assert field != null; assert term != null; final Fields fields = fields(); @@ -1145,7 +1145,7 @@ public abstract class IndexReader implements Cloneable,Closeable { } final Terms terms = fields.terms(field); if (terms != null) { - return terms.docs(skipDocs, term, null); + return terms.docs(liveDocs, term, null); } else { return null; } @@ -1155,7 +1155,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * field & term. This may return null, if either the * field or term does not exist, or, positions were not * stored for this term. */ - public DocsAndPositionsEnum termPositionsEnum(Bits skipDocs, String field, BytesRef term) throws IOException { + public DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term) throws IOException { assert field != null; assert term != null; final Fields fields = fields(); @@ -1164,7 +1164,7 @@ public abstract class IndexReader implements Cloneable,Closeable { } final Terms terms = fields.terms(field); if (terms != null) { - return terms.docsAndPositions(skipDocs, term, null); + return terms.docsAndPositions(liveDocs, term, null); } else { return null; } @@ -1175,7 +1175,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * {@link TermState}. This may return null, if either the field or the term * does not exists or the {@link TermState} is invalid for the underlying * implementation.*/ - public DocsEnum termDocsEnum(Bits skipDocs, String field, BytesRef term, TermState state) throws IOException { + public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException { assert state != null; assert field != null; final Fields fields = fields(); @@ -1184,7 +1184,7 @@ public abstract class IndexReader implements Cloneable,Closeable { } final Terms terms = fields.terms(field); if (terms != null) { - return terms.docs(skipDocs, term, state, null); + return terms.docs(liveDocs, term, state, null); } else { return null; } @@ -1195,7 +1195,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * {@link TermState}. This may return null, if either the field or the term * does not exists, the {@link TermState} is invalid for the underlying * implementation, or positions were not stored for this term.*/ - public DocsAndPositionsEnum termPositionsEnum(Bits skipDocs, String field, BytesRef term, TermState state) throws IOException { + public DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException { assert state != null; assert field != null; final Fields fields = fields(); @@ -1204,7 +1204,7 @@ public abstract class IndexReader implements Cloneable,Closeable { } final Terms terms = fields.terms(field); if (terms != null) { - return terms.docsAndPositions(skipDocs, term, state, null); + return terms.docsAndPositions(liveDocs, term, state, null); } else { return null; } @@ -1260,7 +1260,7 @@ public abstract class IndexReader implements Cloneable,Closeable { public int deleteDocuments(Term term) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException { ensureOpen(); DocsEnum docs = MultiFields.getTermDocsEnum(this, - MultiFields.getDeletedDocs(this), + MultiFields.getLiveDocs(this), term.field(), term.bytes()); if (docs == null) return 0; @@ -1385,15 +1385,17 @@ public abstract class IndexReader implements Cloneable,Closeable { */ public abstract Collection getFieldNames(FieldOption fldOption); - /** Returns the {@link Bits} representing deleted docs. A - * set bit indicates the doc ID has been deleted. This - * method should return null when there are no deleted - * docs. + /** Returns the {@link Bits} representing live (not + * deleted) docs. A set bit indicates the doc ID has not + * been deleted. If this method returns null it means + * there are no deleted documents (all documents are + * live). * - * The returned instance has been safely published for use by - * multiple threads without additional synchronization. + * The returned instance has been safely published for + * use by multiple threads without additional + * synchronization. * @lucene.experimental */ - public abstract Bits getDeletedDocs(); + public abstract Bits getLiveDocs(); /** * Expert: return the IndexCommit that this reader has diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index bcf5218c106..8efa46cdfca 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -2214,8 +2214,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Must write deleted docs after the CFS so we don't // slurp the del file into CFS: - if (flushedSegment.deletedDocuments != null) { - final int delCount = flushedSegment.deletedDocuments.count(); + if (flushedSegment.liveDocs != null) { + final int delCount = flushedSegment.segmentInfo.docCount - flushedSegment.liveDocs.count(); assert delCount > 0; newSegment.setDelCount(delCount); newSegment.advanceDelGen(); @@ -2230,7 +2230,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // shortly-to-be-opened SegmentReader and let it // carry the changes; there's no reason to use // filesystem as intermediary here. - flushedSegment.deletedDocuments.write(directory, delFileName); + flushedSegment.liveDocs.write(directory, delFileName); success2 = true; } finally { if (!success2) { @@ -2931,9 +2931,9 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Reader was skipped because it was 100% deletions continue; } - final Bits prevDelDocs = previousReader.getDeletedDocs(); + final Bits prevLiveDocs = previousReader.getLiveDocs(); final SegmentReader currentReader = merge.readers.get(i); - final Bits currentDelDocs = currentReader.getDeletedDocs(); + final Bits currentLiveDocs = currentReader.getLiveDocs(); if (previousReader.hasDeletions()) { // There were deletes on this segment when the merge @@ -2948,10 +2948,10 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // committed since we started the merge, so we // must merge them: for(int j=0;j delDocs = new ArrayList(); + final List liveDocs = new ArrayList(); final List starts = new ArrayList(); try { final int maxDoc = new ReaderUtil.Gather(r) { @Override protected void add(int base, IndexReader r) throws IOException { - // record all delDocs, even if they are null - delDocs.add(r.getDeletedDocs()); + // record all liveDocs, even if they are null + liveDocs.add(r.getLiveDocs()); starts.add(base); } }.run(); @@ -122,12 +122,12 @@ public final class MultiFields extends Fields { throw new RuntimeException(ioe); } - assert delDocs.size() > 0; - if (delDocs.size() == 1) { + assert liveDocs.size() > 0; + if (liveDocs.size() == 1) { // Only one actual sub reader -- optimize this case - result = delDocs.get(0); + result = liveDocs.get(0); } else { - result = new MultiBits(delDocs, starts); + result = new MultiBits(liveDocs, starts, true); } } else { @@ -150,12 +150,12 @@ public final class MultiFields extends Fields { /** Returns {@link DocsEnum} for the specified field & * term. This may return null if the term does not * exist. */ - public static DocsEnum getTermDocsEnum(IndexReader r, Bits skipDocs, String field, BytesRef term) throws IOException { + public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException { assert field != null; assert term != null; final Terms terms = getTerms(r, field); if (terms != null) { - return terms.docs(skipDocs, term, null); + return terms.docs(liveDocs, term, null); } else { return null; } @@ -164,12 +164,12 @@ public final class MultiFields extends Fields { /** Returns {@link DocsAndPositionsEnum} for the specified * field & term. This may return null if the term does * not exist or positions were not indexed. */ - public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits skipDocs, String field, BytesRef term) throws IOException { + public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException { assert field != null; assert term != null; final Terms terms = getTerms(r, field); if (terms != null) { - return terms.docsAndPositions(skipDocs, term, null); + return terms.docsAndPositions(liveDocs, term, null); } else { return null; } diff --git a/lucene/src/java/org/apache/lucene/index/MultiReader.java b/lucene/src/java/org/apache/lucene/index/MultiReader.java index a67470930ac..aa0df651c99 100644 --- a/lucene/src/java/org/apache/lucene/index/MultiReader.java +++ b/lucene/src/java/org/apache/lucene/index/MultiReader.java @@ -141,8 +141,8 @@ public class MultiReader extends IndexReader implements Cloneable { } @Override - public Bits getDeletedDocs() { - throw new UnsupportedOperationException("please use MultiFields.getDeletedDocs, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Bits deletedDocs"); + public Bits getLiveDocs() { + throw new UnsupportedOperationException("please use MultiFields.getLiveDocs, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Bits liveDocs"); } /** diff --git a/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java b/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java index 330b76d061b..b0771cb77c3 100644 --- a/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java +++ b/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java @@ -346,7 +346,7 @@ public final class MultiTermsEnum extends TermsEnum { } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { final MultiDocsEnum docsEnum; if (reuse != null) { docsEnum = (MultiDocsEnum) reuse; @@ -354,11 +354,11 @@ public final class MultiTermsEnum extends TermsEnum { docsEnum = new MultiDocsEnum(); } - final MultiBits multiSkipDocs; - if (skipDocs instanceof MultiBits) { - multiSkipDocs = (MultiBits) skipDocs; + final MultiBits multiLiveDocs; + if (liveDocs instanceof MultiBits) { + multiLiveDocs = (MultiBits) liveDocs; } else { - multiSkipDocs = null; + multiLiveDocs = null; } int upto = 0; @@ -369,22 +369,22 @@ public final class MultiTermsEnum extends TermsEnum { final Bits b; - if (multiSkipDocs != null) { + if (multiLiveDocs != null) { // optimize for common case: requested skip docs is a // congruent sub-slice of MultiBits: in this case, we - // just pull the skipDocs from the sub reader, rather + // just pull the liveDocs from the sub reader, rather // than making the inefficient // Slice(Multi(sub-readers)): - final MultiBits.SubResult sub = multiSkipDocs.getMatchingSub(entry.subSlice); + final MultiBits.SubResult sub = multiLiveDocs.getMatchingSub(entry.subSlice); if (sub.matches) { b = sub.result; } else { // custom case: requested skip docs is foreign: // must slice it on every access - b = new BitsSlice(skipDocs, entry.subSlice); + b = new BitsSlice(liveDocs, entry.subSlice); } - } else if (skipDocs != null) { - b = new BitsSlice(skipDocs, entry.subSlice); + } else if (liveDocs != null) { + b = new BitsSlice(liveDocs, entry.subSlice); } else { // no deletions b = null; @@ -407,7 +407,7 @@ public final class MultiTermsEnum extends TermsEnum { } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { final MultiDocsAndPositionsEnum docsAndPositionsEnum; if (reuse != null) { docsAndPositionsEnum = (MultiDocsAndPositionsEnum) reuse; @@ -415,11 +415,11 @@ public final class MultiTermsEnum extends TermsEnum { docsAndPositionsEnum = new MultiDocsAndPositionsEnum(); } - final MultiBits multiSkipDocs; - if (skipDocs instanceof MultiBits) { - multiSkipDocs = (MultiBits) skipDocs; + final MultiBits multiLiveDocs; + if (liveDocs instanceof MultiBits) { + multiLiveDocs = (MultiBits) liveDocs; } else { - multiSkipDocs = null; + multiLiveDocs = null; } int upto = 0; @@ -430,23 +430,23 @@ public final class MultiTermsEnum extends TermsEnum { final Bits b; - if (multiSkipDocs != null) { + if (multiLiveDocs != null) { // Optimize for common case: requested skip docs is a // congruent sub-slice of MultiBits: in this case, we - // just pull the skipDocs from the sub reader, rather + // just pull the liveDocs from the sub reader, rather // than making the inefficient // Slice(Multi(sub-readers)): - final MultiBits.SubResult sub = multiSkipDocs.getMatchingSub(top[i].subSlice); + final MultiBits.SubResult sub = multiLiveDocs.getMatchingSub(top[i].subSlice); if (sub.matches) { b = sub.result; } else { // custom case: requested skip docs is foreign: // must slice it on every access (very // inefficient) - b = new BitsSlice(skipDocs, top[i].subSlice); + b = new BitsSlice(liveDocs, top[i].subSlice); } - } else if (skipDocs != null) { - b = new BitsSlice(skipDocs, top[i].subSlice); + } else if (liveDocs != null) { + b = new BitsSlice(liveDocs, top[i].subSlice); } else { // no deletions b = null; diff --git a/lucene/src/java/org/apache/lucene/index/ParallelReader.java b/lucene/src/java/org/apache/lucene/index/ParallelReader.java index d587b4a7b51..a281abb8e43 100644 --- a/lucene/src/java/org/apache/lucene/index/ParallelReader.java +++ b/lucene/src/java/org/apache/lucene/index/ParallelReader.java @@ -206,9 +206,9 @@ public class ParallelReader extends IndexReader { } } - @Override - public Bits getDeletedDocs() { - return MultiFields.getDeletedDocs(readers.get(0)); + @Override + public Bits getLiveDocs() { + return MultiFields.getLiveDocs(readers.get(0)); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/src/java/org/apache/lucene/index/SegmentMerger.java index 93c32738762..30e02f11db4 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentMerger.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentMerger.java @@ -282,11 +282,12 @@ final class SegmentMerger { throws IOException, MergeAbortedException, CorruptIndexException { int docCount = 0; final int maxDoc = reader.maxDoc(); - final Bits delDocs = reader.getDeletedDocs(); + final Bits liveDocs = reader.getLiveDocs(); + assert liveDocs != null; if (matchingFieldsReader != null) { // We can bulk-copy because the fieldInfos are "congruent" for (int j = 0; j < maxDoc;) { - if (delDocs.get(j)) { + if (!liveDocs.get(j)) { // skip deleted docs ++j; continue; @@ -298,7 +299,7 @@ final class SegmentMerger { j++; numDocs++; if (j >= maxDoc) break; - if (delDocs.get(j)) { + if (!liveDocs.get(j)) { j++; break; } @@ -311,7 +312,7 @@ final class SegmentMerger { } } else { for (int j = 0; j < maxDoc; j++) { - if (delDocs.get(j)) { + if (!liveDocs.get(j)) { // skip deleted docs continue; } @@ -401,11 +402,11 @@ final class SegmentMerger { final IndexReader reader) throws IOException, MergeAbortedException { final int maxDoc = reader.maxDoc(); - final Bits delDocs = reader.getDeletedDocs(); + final Bits liveDocs = reader.getLiveDocs(); if (matchingVectorsReader != null) { // We can bulk-copy because the fieldInfos are "congruent" for (int docNum = 0; docNum < maxDoc;) { - if (delDocs.get(docNum)) { + if (!liveDocs.get(docNum)) { // skip deleted docs ++docNum; continue; @@ -417,7 +418,7 @@ final class SegmentMerger { docNum++; numDocs++; if (docNum >= maxDoc) break; - if (delDocs.get(docNum)) { + if (!liveDocs.get(docNum)) { docNum++; break; } @@ -429,7 +430,7 @@ final class SegmentMerger { } } else { for (int docNum = 0; docNum < maxDoc; docNum++) { - if (delDocs.get(docNum)) { + if (!liveDocs.get(docNum)) { // skip deleted docs continue; } @@ -499,14 +500,14 @@ final class SegmentMerger { if (f != null) { slices.add(new ReaderUtil.Slice(docBase, maxDoc, fields.size())); fields.add(f); - bits.add(r.getDeletedDocs()); + bits.add(r.getLiveDocs()); bitsStarts.add(docBase); } final PerDocValues producer = r.perDocValues(); if (producer != null) { perDocSlices.add(new ReaderUtil.Slice(docBase, maxDoc, fields.size())); perDocProducers.add(producer); - perDocBits.add(r.getDeletedDocs()); + perDocBits.add(r.getLiveDocs()); perDocBitsStarts.add(docBase); } docBase += maxDoc; @@ -544,13 +545,13 @@ final class SegmentMerger { inputDocBase += reader.maxDoc(); if (mergeState.delCounts[i] != 0) { int delCount = 0; - final Bits delDocs = reader.getDeletedDocs(); - assert delDocs != null; + final Bits liveDocs = reader.getLiveDocs(); + assert liveDocs != null; final int maxDoc = reader.maxDoc(); final int[] docMap = mergeState.docMaps[i] = new int[maxDoc]; int newDocID = 0; for(int j=0;j fieldsReaderLocal = new FieldsReaderLocal(); CloseableThreadLocal termVectorsLocal = new CloseableThreadLocal(); - volatile BitVector deletedDocs; - AtomicInteger deletedDocsRef = null; - private boolean deletedDocsDirty = false; + volatile BitVector liveDocs; + AtomicInteger liveDocsRef = null; + private boolean liveDocsDirty = false; private boolean normsDirty = false; // TODO: we should move this tracking into SegmentInfo; @@ -116,7 +116,7 @@ public class SegmentReader extends IndexReader implements Cloneable { if (doOpenStores) { instance.core.openDocStores(si); } - instance.loadDeletedDocs(); + instance.loadLiveDocs(); instance.openNorms(instance.core.cfsDir, readBufferSize); success = true; } finally { @@ -138,34 +138,37 @@ public class SegmentReader extends IndexReader implements Cloneable { } @Override - public Bits getDeletedDocs() { - return deletedDocs; + public Bits getLiveDocs() { + return liveDocs; } - private boolean checkDeletedCounts() throws IOException { - final int recomputedCount = deletedDocs.getRecomputedCount(); - - assert deletedDocs.count() == recomputedCount : "deleted count=" + deletedDocs.count() + " vs recomputed count=" + recomputedCount; + private boolean checkLiveCounts() throws IOException { + final int recomputedCount = liveDocs.getRecomputedCount(); + // First verify BitVector is self consistent: + assert liveDocs.count() == recomputedCount : "live count=" + liveDocs.count() + " vs recomputed count=" + recomputedCount; - assert si.getDelCount() == recomputedCount : - "delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + recomputedCount; + assert si.getDelCount() == si.docCount - recomputedCount : + "delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + (si.docCount-recomputedCount); // Verify # deletes does not exceed maxDoc for this // segment: assert si.getDelCount() <= maxDoc() : - "delete count mismatch: " + recomputedCount + ") exceeds max doc (" + maxDoc() + ") for segment " + si.name; + "delete count mismatch: " + recomputedCount + ") exceeds max doc (" + maxDoc() + ") for segment " + si.name; return true; } - private void loadDeletedDocs() throws IOException { + private void loadLiveDocs() throws IOException { // NOTE: the bitvector is stored using the regular directory, not cfs if (hasDeletions(si)) { - deletedDocs = new BitVector(directory(), si.getDelFileName()); - deletedDocsRef = new AtomicInteger(1); - assert checkDeletedCounts(); - if (deletedDocs.size() != si.docCount) { - throw new CorruptIndexException("document count mismatch: deleted docs count " + deletedDocs.size() + " vs segment doc count " + si.docCount + " segment=" + si.name); + liveDocs = new BitVector(directory(), si.getDelFileName()); + if (liveDocs.getVersion() < BitVector.VERSION_DGAPS_CLEARED) { + liveDocs.invertAll(); + } + liveDocsRef = new AtomicInteger(1); + assert checkLiveCounts(); + if (liveDocs.size() != si.docCount) { + throw new CorruptIndexException("document count mismatch: deleted docs count " + liveDocs.size() + " vs segment doc count " + si.docCount + " segment=" + si.name); } } else assert si.getDelCount() == 0; @@ -256,27 +259,27 @@ public class SegmentReader extends IndexReader implements Cloneable { if (!openReadOnly && hasChanges) { // My pending changes transfer to the new reader - clone.deletedDocsDirty = deletedDocsDirty; + clone.liveDocsDirty = liveDocsDirty; clone.normsDirty = normsDirty; clone.hasChanges = hasChanges; hasChanges = false; } if (doClone) { - if (deletedDocs != null) { - deletedDocsRef.incrementAndGet(); - clone.deletedDocs = deletedDocs; - clone.deletedDocsRef = deletedDocsRef; + if (liveDocs != null) { + liveDocsRef.incrementAndGet(); + clone.liveDocs = liveDocs; + clone.liveDocsRef = liveDocsRef; } } else { if (!deletionsUpToDate) { // load deleted docs - assert clone.deletedDocs == null; - clone.loadDeletedDocs(); - } else if (deletedDocs != null) { - deletedDocsRef.incrementAndGet(); - clone.deletedDocs = deletedDocs; - clone.deletedDocsRef = deletedDocsRef; + assert clone.liveDocs == null; + clone.loadLiveDocs(); + } else if (liveDocs != null) { + liveDocsRef.incrementAndGet(); + clone.liveDocs = liveDocs; + clone.liveDocsRef = liveDocsRef; } } @@ -326,10 +329,10 @@ public class SegmentReader extends IndexReader implements Cloneable { } private synchronized void commitChanges(Map commitUserData) throws IOException { - if (deletedDocsDirty) { // re-write deleted + if (liveDocsDirty) { // re-write deleted si.advanceDelGen(); - assert deletedDocs.length() == si.docCount; + assert liveDocs.length() == si.docCount; // We can write directly to the actual name (vs to a // .tmp & renaming it) because the file is not live @@ -337,7 +340,7 @@ public class SegmentReader extends IndexReader implements Cloneable { final String delFileName = si.getDelFileName(); boolean success = false; try { - deletedDocs.write(directory(), delFileName); + liveDocs.write(directory(), delFileName); success = true; } finally { if (!success) { @@ -349,10 +352,9 @@ public class SegmentReader extends IndexReader implements Cloneable { } } } - si.setDelCount(si.getDelCount()+pendingDeleteCount); pendingDeleteCount = 0; - assert deletedDocs.count() == si.getDelCount(): "delete count mismatch during commit: info=" + si.getDelCount() + " vs BitVector=" + deletedDocs.count(); + assert (maxDoc()-liveDocs.count()) == si.getDelCount(): "delete count mismatch during commit: info=" + si.getDelCount() + " vs BitVector=" + (maxDoc()-liveDocs.count()); } else { assert pendingDeleteCount == 0; } @@ -365,7 +367,7 @@ public class SegmentReader extends IndexReader implements Cloneable { } } } - deletedDocsDirty = false; + liveDocsDirty = false; normsDirty = false; hasChanges = false; } @@ -379,10 +381,10 @@ public class SegmentReader extends IndexReader implements Cloneable { termVectorsLocal.close(); fieldsReaderLocal.close(); - if (deletedDocs != null) { - deletedDocsRef.decrementAndGet(); + if (liveDocs != null) { + liveDocsRef.decrementAndGet(); // null so if an app hangs on to us we still free most ram - deletedDocs = null; + liveDocs = null; } for (final SegmentNorms norm : norms.values()) { @@ -401,7 +403,7 @@ public class SegmentReader extends IndexReader implements Cloneable { @Override public boolean hasDeletions() { // Don't call ensureOpen() here (it could affect performance) - return deletedDocs != null; + return liveDocs != null; } static boolean usesCompoundFile(SegmentInfo si) throws IOException { @@ -414,38 +416,39 @@ public class SegmentReader extends IndexReader implements Cloneable { @Override protected void doDelete(int docNum) { - if (deletedDocs == null) { - deletedDocs = new BitVector(maxDoc()); - deletedDocsRef = new AtomicInteger(1); + if (liveDocs == null) { + liveDocs = new BitVector(maxDoc()); + liveDocs.setAll(); + liveDocsRef = new AtomicInteger(1); } // there is more than 1 SegmentReader with a reference to this - // deletedDocs BitVector so decRef the current deletedDocsRef, - // clone the BitVector, create a new deletedDocsRef - if (deletedDocsRef.get() > 1) { - AtomicInteger oldRef = deletedDocsRef; - deletedDocs = cloneDeletedDocs(deletedDocs); - deletedDocsRef = new AtomicInteger(1); + // liveDocs BitVector so decRef the current liveDocsRef, + // clone the BitVector, create a new liveDocsRef + if (liveDocsRef.get() > 1) { + AtomicInteger oldRef = liveDocsRef; + liveDocs = cloneDeletedDocs(liveDocs); + liveDocsRef = new AtomicInteger(1); oldRef.decrementAndGet(); } - deletedDocsDirty = true; - if (!deletedDocs.getAndSet(docNum)) { + liveDocsDirty = true; + if (liveDocs.getAndClear(docNum)) { pendingDeleteCount++; } } @Override protected void doUndeleteAll() { - deletedDocsDirty = false; - if (deletedDocs != null) { - assert deletedDocsRef != null; - deletedDocsRef.decrementAndGet(); - deletedDocs = null; - deletedDocsRef = null; + liveDocsDirty = false; + if (liveDocs != null) { + assert liveDocsRef != null; + liveDocsRef.decrementAndGet(); + liveDocs = null; + liveDocsRef = null; pendingDeleteCount = 0; si.clearDelGen(); si.setDelCount(0); } else { - assert deletedDocsRef == null; + assert liveDocsRef == null; assert pendingDeleteCount == 0; } } @@ -484,10 +487,11 @@ public class SegmentReader extends IndexReader implements Cloneable { @Override public int numDocs() { // Don't call ensureOpen() here (it could affect performance) - int n = maxDoc(); - if (deletedDocs != null) - n -= deletedDocs.count(); - return n; + if (liveDocs != null) { + return liveDocs.count(); + } else { + return maxDoc(); + } } @Override @@ -790,7 +794,7 @@ public class SegmentReader extends IndexReader implements Cloneable { void startCommit() { rollbackSegmentInfo = (SegmentInfo) si.clone(); rollbackHasChanges = hasChanges; - rollbackDeletedDocsDirty = deletedDocsDirty; + rollbackDeletedDocsDirty = liveDocsDirty; rollbackNormsDirty = normsDirty; rollbackPendingDeleteCount = pendingDeleteCount; for (SegmentNorms norm : norms.values()) { @@ -801,7 +805,7 @@ public class SegmentReader extends IndexReader implements Cloneable { void rollbackCommit() { si.reset(rollbackSegmentInfo); hasChanges = rollbackHasChanges; - deletedDocsDirty = rollbackDeletedDocsDirty; + liveDocsDirty = rollbackDeletedDocsDirty; normsDirty = rollbackNormsDirty; pendingDeleteCount = rollbackPendingDeleteCount; for (SegmentNorms norm : norms.values()) { diff --git a/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java b/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java index 32fab036273..f9f4cc0d094 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java @@ -40,7 +40,7 @@ public class SegmentWriteState { public final BufferedDeletes segDeletes; // Lazily created: - public BitVector deletedDocs; + public BitVector liveDocs; final SegmentCodecs segmentCodecs; public final int codecId; diff --git a/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java b/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java index 3fc72663d54..e9432bd0bca 100644 --- a/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java +++ b/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java @@ -76,10 +76,9 @@ public final class SlowMultiReaderWrapper extends FilterIndexReader { } @Override - public Bits getDeletedDocs() { - return MultiFields.getDeletedDocs(in); + public Bits getLiveDocs() { + return MultiFields.getLiveDocs(in); } - @Override public IndexReader[] getSequentialSubReaders() { diff --git a/lucene/src/java/org/apache/lucene/index/Terms.java b/lucene/src/java/org/apache/lucene/index/Terms.java index 67e1c144bf5..b7600c8a784 100644 --- a/lucene/src/java/org/apache/lucene/index/Terms.java +++ b/lucene/src/java/org/apache/lucene/index/Terms.java @@ -74,10 +74,10 @@ public abstract class Terms { /** Get {@link DocsEnum} for the specified term. This * method may return null if the term does not exist. */ - public DocsEnum docs(Bits skipDocs, BytesRef text, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, BytesRef text, DocsEnum reuse) throws IOException { final TermsEnum termsEnum = getThreadTermsEnum(); if (termsEnum.seekExact(text, true)) { - return termsEnum.docs(skipDocs, reuse); + return termsEnum.docs(liveDocs, reuse); } else { return null; } @@ -86,10 +86,10 @@ public abstract class Terms { /** Get {@link DocsEnum} for the specified term. This * method will may return null if the term does not * exists, or positions were not indexed. */ - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, BytesRef text, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, BytesRef text, DocsAndPositionsEnum reuse) throws IOException { final TermsEnum termsEnum = getThreadTermsEnum(); if (termsEnum.seekExact(text, true)) { - return termsEnum.docsAndPositions(skipDocs, reuse); + return termsEnum.docsAndPositions(liveDocs, reuse); } else { return null; } @@ -101,10 +101,10 @@ public abstract class Terms { * * @see TermsEnum#termState() * @see TermsEnum#seekExact(BytesRef, TermState) */ - public DocsEnum docs(Bits skipDocs, BytesRef term, TermState termState, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, BytesRef term, TermState termState, DocsEnum reuse) throws IOException { final TermsEnum termsEnum = getThreadTermsEnum(); termsEnum.seekExact(term, termState); - return termsEnum.docs(skipDocs, reuse); + return termsEnum.docs(liveDocs, reuse); } /** @@ -114,10 +114,10 @@ public abstract class Terms { * * @see TermsEnum#termState() * @see TermsEnum#seekExact(BytesRef, TermState) */ - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, BytesRef term, TermState termState, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, BytesRef term, TermState termState, DocsAndPositionsEnum reuse) throws IOException { final TermsEnum termsEnum = getThreadTermsEnum(); termsEnum.seekExact(term, termState); - return termsEnum.docsAndPositions(skipDocs, reuse); + return termsEnum.docsAndPositions(liveDocs, reuse); } public long getUniqueTermCount() throws IOException { diff --git a/lucene/src/java/org/apache/lucene/index/TermsEnum.java b/lucene/src/java/org/apache/lucene/index/TermsEnum.java index 18918ab6e64..e1bd0e31239 100644 --- a/lucene/src/java/org/apache/lucene/index/TermsEnum.java +++ b/lucene/src/java/org/apache/lucene/index/TermsEnum.java @@ -147,16 +147,16 @@ public abstract class TermsEnum { * call this when the enum is unpositioned. This method * will not return null. * - * @param skipDocs set bits are documents that should not + * @param liveDocs set bits are documents that should not * be returned * @param reuse pass a prior DocsEnum for possible reuse */ - public abstract DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException; + public abstract DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException; /** Get {@link DocsAndPositionsEnum} for the current term. * Do not call this when the enum is unpositioned. * This method will only return null if positions were * not indexed into the postings by this codec. */ - public abstract DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException; + public abstract DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException; /** * Expert: Returns the TermsEnums internal state to position the TermsEnum @@ -224,12 +224,12 @@ public abstract class TermsEnum { } @Override - public DocsEnum docs(Bits bits, DocsEnum reuse) { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) { throw new IllegalStateException("this method should never be called"); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits bits, DocsAndPositionsEnum reuse) { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) { throw new IllegalStateException("this method should never be called"); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java index b03b639ee00..6e61e7e1ef7 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java @@ -688,23 +688,23 @@ public class BlockTermsReader extends FieldsProducer { } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { //System.out.println("BTR.docs this=" + this); decodeMetaData(); //System.out.println(" state.docFreq=" + state.docFreq); - final DocsEnum docsEnum = postingsReader.docs(fieldInfo, state, skipDocs, reuse); + final DocsEnum docsEnum = postingsReader.docs(fieldInfo, state, liveDocs, reuse); assert docsEnum != null; return docsEnum; } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { //System.out.println("BTR.d&p this=" + this); decodeMetaData(); if (fieldInfo.omitTermFreqAndPositions) { return null; } else { - DocsAndPositionsEnum dpe = postingsReader.docsAndPositions(fieldInfo, state, skipDocs, reuse); + DocsAndPositionsEnum dpe = postingsReader.docsAndPositions(fieldInfo, state, liveDocs, reuse); //System.out.println(" return d&pe=" + dpe); return dpe; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java index 5e66eb1434e..591399ec690 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java @@ -114,8 +114,8 @@ public abstract class DocValuesConsumer { final IndexDocValues r = reader.docValues(mergeState.fieldInfo.name); if (r != null) { merged = true; - merge(new Writer.MergeState(r, docBase, reader.maxDoc(), reader - .getDeletedDocs())); + merge(new Writer.MergeState(r, docBase, reader.maxDoc(), + reader.getLiveDocs())); } docBase += reader.numDocs(); } @@ -152,15 +152,15 @@ public abstract class DocValuesConsumer { public final int docBase; /** the number of documents in this MergeState */ public final int docCount; - /** the deleted bits for this MergeState */ - public final Bits bits; + /** the not deleted bits for this MergeState */ + public final Bits liveDocs; - public MergeState(IndexDocValues reader, int docBase, int docCount, Bits bits) { + public MergeState(IndexDocValues reader, int docBase, int docCount, Bits liveDocs) { assert reader != null; this.reader = reader; this.docBase = docBase; this.docCount = docCount; - this.bits = bits; + this.liveDocs = liveDocs; } } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/MergeState.java b/lucene/src/java/org/apache/lucene/index/codecs/MergeState.java index ad29d1c9b1d..fa73f47a8be 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/MergeState.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/MergeState.java @@ -39,7 +39,7 @@ public class MergeState { public int[] delCounts; // Deletion count per reader public int[] docBase; // New docID base per reader public int mergedDocCount; // Total # merged docs - public Bits multiDeletedDocs; + public Bits multiLiveDocs; public CheckAbort checkAbort; // Updated per field; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java b/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java index b86c009f474..ffdfdc38616 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java @@ -49,11 +49,11 @@ public abstract class PostingsReaderBase implements Closeable { /** Must fully consume state, since after this call that * TermState may be reused. */ - public abstract DocsEnum docs(FieldInfo fieldInfo, BlockTermState state, Bits skipDocs, DocsEnum reuse) throws IOException; + public abstract DocsEnum docs(FieldInfo fieldInfo, BlockTermState state, Bits liveDocs, DocsEnum reuse) throws IOException; /** Must fully consume state, since after this call that * TermState may be reused. */ - public abstract DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState state, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException; + public abstract DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState state, Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException; public abstract void close() throws IOException; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java b/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java index 93b578ce17c..875da3bbb63 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java @@ -67,7 +67,7 @@ public abstract class TermsConsumer { MultiDocsEnum docsEnumIn = null; while((term = termsEnum.next()) != null) { - docsEnumIn = (MultiDocsEnum) termsEnum.docs(mergeState.multiDeletedDocs, docsEnumIn); + docsEnumIn = (MultiDocsEnum) termsEnum.docs(mergeState.multiLiveDocs, docsEnumIn); if (docsEnumIn != null) { docsEnum.reset(docsEnumIn); final PostingsConsumer postingsConsumer = startTerm(term); @@ -89,7 +89,7 @@ public abstract class TermsConsumer { postingsEnum.setMergeState(mergeState); MultiDocsAndPositionsEnum postingsEnumIn = null; while((term = termsEnum.next()) != null) { - postingsEnumIn = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(mergeState.multiDeletedDocs, postingsEnumIn); + postingsEnumIn = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(mergeState.multiLiveDocs, postingsEnumIn); if (postingsEnumIn != null) { postingsEnum.reset(postingsEnumIn); // set PayloadProcessor diff --git a/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java b/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java index 344810f8bc5..f84237160a5 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java @@ -269,7 +269,7 @@ public class MemoryCodec extends Codec { private byte[] buffer = new byte[16]; private final ByteArrayDataInput in = new ByteArrayDataInput(buffer); - private Bits skipDocs; + private Bits liveDocs; private int docUpto; private int docID; private int freq; @@ -285,14 +285,14 @@ public class MemoryCodec extends Codec { return omitTFAP == this.omitTFAP && storePayloads == this.storePayloads; } - public FSTDocsEnum reset(BytesRef bufferIn, Bits skipDocs, int numDocs) { + public FSTDocsEnum reset(BytesRef bufferIn, Bits liveDocs, int numDocs) { assert numDocs > 0; if (buffer.length < bufferIn.length - bufferIn.offset) { buffer = ArrayUtil.grow(buffer, bufferIn.length - bufferIn.offset); } in.reset(buffer, 0, bufferIn.length - bufferIn.offset); System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length - bufferIn.offset); - this.skipDocs = skipDocs; + this.liveDocs = liveDocs; docID = 0; docUpto = 0; payloadLen = 0; @@ -339,7 +339,7 @@ public class MemoryCodec extends Codec { } } - if (skipDocs == null || !skipDocs.get(docID)) { + if (liveDocs == null || liveDocs.get(docID)) { if (VERBOSE) System.out.println(" return docID=" + docID + " freq=" + freq); return docID; } @@ -375,7 +375,7 @@ public class MemoryCodec extends Codec { private byte[] buffer = new byte[16]; private final ByteArrayDataInput in = new ByteArrayDataInput(buffer); - private Bits skipDocs; + private Bits liveDocs; private int docUpto; private int docID; private int freq; @@ -396,7 +396,7 @@ public class MemoryCodec extends Codec { return omitTFAP == this.omitTFAP && storePayloads == this.storePayloads; } - public FSTDocsAndPositionsEnum reset(BytesRef bufferIn, Bits skipDocs, int numDocs) { + public FSTDocsAndPositionsEnum reset(BytesRef bufferIn, Bits liveDocs, int numDocs) { assert numDocs > 0; if (VERBOSE) { System.out.println("D&P reset bytes this=" + this); @@ -409,7 +409,7 @@ public class MemoryCodec extends Codec { } in.reset(buffer, 0, bufferIn.length - bufferIn.offset); System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length - bufferIn.offset); - this.skipDocs = skipDocs; + this.liveDocs = liveDocs; docID = 0; docUpto = 0; payload.bytes = buffer; @@ -446,7 +446,7 @@ public class MemoryCodec extends Codec { } } - if (skipDocs == null || !skipDocs.get(docID)) { + if (liveDocs == null || liveDocs.get(docID)) { pos = 0; posPending = freq; if (VERBOSE) System.out.println(" return docID=" + docID + " freq=" + freq); @@ -598,7 +598,7 @@ public class MemoryCodec extends Codec { } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { decodeMetaData(); FSTDocsEnum docsEnum; if (reuse == null || !(reuse instanceof FSTDocsEnum)) { @@ -609,11 +609,11 @@ public class MemoryCodec extends Codec { docsEnum = new FSTDocsEnum(field.omitTermFreqAndPositions, field.storePayloads); } } - return docsEnum.reset(current.output, skipDocs, docFreq); + return docsEnum.reset(current.output, liveDocs, docFreq); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { if (field.omitTermFreqAndPositions) { return null; } @@ -628,7 +628,7 @@ public class MemoryCodec extends Codec { } } if (VERBOSE) System.out.println("D&P reset this=" + this); - return docsAndPositionsEnum.reset(current.output, skipDocs, docFreq); + return docsAndPositionsEnum.reset(current.output, liveDocs, docFreq); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java index 61e26e7a60d..78253acdd13 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java @@ -951,7 +951,7 @@ public class PreFlexFields extends FieldsProducer { } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { PreDocsEnum docsEnum; if (reuse == null || !(reuse instanceof PreDocsEnum)) { docsEnum = new PreDocsEnum(); @@ -961,11 +961,11 @@ public class PreFlexFields extends FieldsProducer { docsEnum = new PreDocsEnum(); } } - return docsEnum.reset(termEnum, skipDocs); + return docsEnum.reset(termEnum, liveDocs); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { PreDocsAndPositionsEnum docsPosEnum; if (fieldInfo.omitTermFreqAndPositions) { return null; @@ -977,7 +977,7 @@ public class PreFlexFields extends FieldsProducer { docsPosEnum = new PreDocsAndPositionsEnum(); } } - return docsPosEnum.reset(termEnum, skipDocs); + return docsPosEnum.reset(termEnum, liveDocs); } } @@ -992,8 +992,8 @@ public class PreFlexFields extends FieldsProducer { return freqStream; } - public PreDocsEnum reset(SegmentTermEnum termEnum, Bits skipDocs) throws IOException { - docs.setSkipDocs(skipDocs); + public PreDocsEnum reset(SegmentTermEnum termEnum, Bits liveDocs) throws IOException { + docs.setLiveDocs(liveDocs); docs.seek(termEnum); return this; } @@ -1048,8 +1048,8 @@ public class PreFlexFields extends FieldsProducer { return freqStream; } - public DocsAndPositionsEnum reset(SegmentTermEnum termEnum, Bits skipDocs) throws IOException { - pos.setSkipDocs(skipDocs); + public DocsAndPositionsEnum reset(SegmentTermEnum termEnum, Bits liveDocs) throws IOException { + pos.setLiveDocs(liveDocs); pos.seek(termEnum); return this; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java index ac483f93e4f..c0616aa9007 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java @@ -33,7 +33,7 @@ public class SegmentTermDocs { //protected SegmentReader parent; private final FieldInfos fieldInfos; private final TermInfosReader tis; - protected Bits skipDocs; + protected Bits liveDocs; protected IndexInput freqStream; protected int count; protected int df; @@ -53,18 +53,6 @@ public class SegmentTermDocs { protected boolean currentFieldStoresPayloads; protected boolean currentFieldOmitTermFreqAndPositions; - /* - protected SegmentTermDocs(SegmentReader parent) { - this.parent = parent; - this.freqStream = (IndexInput) parent.core.freqStream.clone(); - synchronized (parent) { - this.deletedDocs = parent.deletedDocs; - } - this.skipInterval = parent.core.getTermsReader().getSkipInterval(); - this.maxSkipLevels = parent.core.getTermsReader().getMaxSkipLevels(); - } - */ - public SegmentTermDocs(IndexInput freqStream, TermInfosReader tis, FieldInfos fieldInfos) { this.freqStream = (IndexInput) freqStream.clone(); this.tis = tis; @@ -78,8 +66,8 @@ public class SegmentTermDocs { seek(ti, term); } - public void setSkipDocs(Bits skipDocs) { - this.skipDocs = skipDocs; + public void setLiveDocs(Bits liveDocs) { + this.liveDocs = liveDocs; } public void seek(SegmentTermEnum segmentTermEnum) throws IOException { @@ -149,7 +137,7 @@ public class SegmentTermDocs { count++; - if (skipDocs == null || !skipDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(doc)) { break; } skippingDoc(); @@ -175,7 +163,7 @@ public class SegmentTermDocs { freq = freqStream.readVInt(); // else read freq count++; - if (skipDocs == null || !skipDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(doc)) { docs[i] = doc; freqs[i] = freq; ++i; @@ -192,7 +180,7 @@ public class SegmentTermDocs { doc += freqStream.readVInt(); count++; - if (skipDocs == null || !skipDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(doc)) { docs[i] = doc; // Hardware freq to 1 when term freqs were not // stored in the index diff --git a/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java index eeb2192ed94..18ba6f2588f 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java @@ -167,7 +167,7 @@ public class PulsingPostingsReaderImpl extends PostingsReaderBase { // TODO: we could actually reuse, by having TL that // holds the last wrapped reuse, and vice-versa @Override - public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse) throws IOException { PulsingTermState termState = (PulsingTermState) _termState; if (termState.postingsSize != -1) { PulsingDocsEnum postings; @@ -179,20 +179,20 @@ public class PulsingPostingsReaderImpl extends PostingsReaderBase { } else { postings = new PulsingDocsEnum(field); } - return postings.reset(skipDocs, termState); + return postings.reset(liveDocs, termState); } else { // TODO: not great that we lose reuse of PulsingDocsEnum in this case: if (reuse instanceof PulsingDocsEnum) { - return wrappedPostingsReader.docs(field, termState.wrappedTermState, skipDocs, null); + return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, null); } else { - return wrappedPostingsReader.docs(field, termState.wrappedTermState, skipDocs, reuse); + return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse); } } } // TODO: -- not great that we can't always reuse @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo field, BlockTermState _termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { if (field.omitTermFreqAndPositions) { return null; } @@ -211,12 +211,12 @@ public class PulsingPostingsReaderImpl extends PostingsReaderBase { postings = new PulsingDocsAndPositionsEnum(field); } - return postings.reset(skipDocs, termState); + return postings.reset(liveDocs, termState); } else { if (reuse instanceof PulsingDocsAndPositionsEnum) { - return wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, skipDocs, null); + return wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, null); } else { - return wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, skipDocs, reuse); + return wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, reuse); } } } @@ -225,7 +225,7 @@ public class PulsingPostingsReaderImpl extends PostingsReaderBase { private final ByteArrayDataInput postings = new ByteArrayDataInput(); private final boolean omitTF; private final boolean storePayloads; - private Bits skipDocs; + private Bits liveDocs; private int docID; private int freq; private int payloadLength; @@ -235,7 +235,7 @@ public class PulsingPostingsReaderImpl extends PostingsReaderBase { storePayloads = fieldInfo.storePayloads; } - public PulsingDocsEnum reset(Bits skipDocs, PulsingTermState termState) { + public PulsingDocsEnum reset(Bits liveDocs, PulsingTermState termState) { //System.out.println("PR docsEnum termState=" + termState + " docFreq=" + termState.docFreq); assert termState.postingsSize != -1; final byte[] bytes = new byte[termState.postingsSize]; @@ -244,7 +244,7 @@ public class PulsingPostingsReaderImpl extends PostingsReaderBase { docID = 0; payloadLength = 0; freq = 1; - this.skipDocs = skipDocs; + this.liveDocs = liveDocs; return this; } @@ -291,7 +291,7 @@ public class PulsingPostingsReaderImpl extends PostingsReaderBase { } } - if (skipDocs == null || !skipDocs.get(docID)) { + if (liveDocs == null || liveDocs.get(docID)) { //System.out.println(" return docID=" + docID + " freq=" + freq); return docID; } @@ -323,7 +323,7 @@ public class PulsingPostingsReaderImpl extends PostingsReaderBase { private final ByteArrayDataInput postings = new ByteArrayDataInput(); private final boolean storePayloads; - private Bits skipDocs; + private Bits liveDocs; private int docID; private int freq; private int posPending; @@ -341,12 +341,12 @@ public class PulsingPostingsReaderImpl extends PostingsReaderBase { return storePayloads == fieldInfo.storePayloads; } - public PulsingDocsAndPositionsEnum reset(Bits skipDocs, PulsingTermState termState) { + public PulsingDocsAndPositionsEnum reset(Bits liveDocs, PulsingTermState termState) { assert termState.postingsSize != -1; final byte[] bytes = new byte[termState.postingsSize]; System.arraycopy(termState.postings, 0, bytes, 0, termState.postingsSize); postings.reset(bytes); - this.skipDocs = skipDocs; + this.liveDocs = liveDocs; payloadLength = 0; posPending = 0; docID = 0; @@ -378,7 +378,7 @@ public class PulsingPostingsReaderImpl extends PostingsReaderBase { } posPending = freq; - if (skipDocs == null || !skipDocs.get(docID)) { + if (liveDocs == null || liveDocs.get(docID)) { //System.out.println(" return docID=" + docID + " freq=" + freq); position = 0; return docID; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java index a6588d5c217..febb756a34b 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java @@ -256,7 +256,7 @@ public class SepPostingsReaderImpl extends PostingsReaderBase { } @Override - public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse) throws IOException { final SepTermState termState = (SepTermState) _termState; SepDocsEnum docsEnum; if (reuse == null || !(reuse instanceof SepDocsEnum)) { @@ -271,11 +271,11 @@ public class SepPostingsReaderImpl extends PostingsReaderBase { } } - return docsEnum.init(fieldInfo, termState, skipDocs); + return docsEnum.init(fieldInfo, termState, liveDocs); } @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState _termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { assert !fieldInfo.omitTermFreqAndPositions; final SepTermState termState = (SepTermState) _termState; SepDocsAndPositionsEnum postingsEnum; @@ -291,7 +291,7 @@ public class SepPostingsReaderImpl extends PostingsReaderBase { } } - return postingsEnum.init(fieldInfo, termState, skipDocs); + return postingsEnum.init(fieldInfo, termState, liveDocs); } class SepDocsEnum extends DocsEnum { @@ -304,7 +304,7 @@ public class SepPostingsReaderImpl extends PostingsReaderBase { // TODO: -- should we do omitTF with 2 different enum classes? private boolean omitTF; private boolean storePayloads; - private Bits skipDocs; + private Bits liveDocs; private final IntIndexInput.Reader docReader; private final IntIndexInput.Reader freqReader; private long skipFP; @@ -337,8 +337,8 @@ public class SepPostingsReaderImpl extends PostingsReaderBase { } } - SepDocsEnum init(FieldInfo fieldInfo, SepTermState termState, Bits skipDocs) throws IOException { - this.skipDocs = skipDocs; + SepDocsEnum init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs) throws IOException { + this.liveDocs = liveDocs; omitTF = fieldInfo.omitTermFreqAndPositions; storePayloads = fieldInfo.storePayloads; @@ -383,7 +383,7 @@ public class SepPostingsReaderImpl extends PostingsReaderBase { freq = freqReader.next(); } - if (skipDocs == null || !skipDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(doc)) { break; } } @@ -408,7 +408,7 @@ public class SepPostingsReaderImpl extends PostingsReaderBase { freq = freqReader.next(); } - if (skipDocs == null || !skipDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(doc)) { docs[i] = doc; freqs[i] = freq; //System.out.println(" docs[" + i + "]=" + doc + " count=" + count + " dF=" + docFreq); @@ -493,7 +493,7 @@ public class SepPostingsReaderImpl extends PostingsReaderBase { long freqStart; private boolean storePayloads; - private Bits skipDocs; + private Bits liveDocs; private final IntIndexInput.Reader docReader; private final IntIndexInput.Reader freqReader; private final IntIndexInput.Reader posReader; @@ -528,8 +528,8 @@ public class SepPostingsReaderImpl extends PostingsReaderBase { payloadIn = (IndexInput) SepPostingsReaderImpl.this.payloadIn.clone(); } - SepDocsAndPositionsEnum init(FieldInfo fieldInfo, SepTermState termState, Bits skipDocs) throws IOException { - this.skipDocs = skipDocs; + SepDocsAndPositionsEnum init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs) throws IOException { + this.liveDocs = liveDocs; storePayloads = fieldInfo.storePayloads; //System.out.println("Sep D&P init"); @@ -584,7 +584,7 @@ public class SepPostingsReaderImpl extends PostingsReaderBase { pendingPosCount += freq; - if (skipDocs == null || !skipDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(doc)) { break; } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java index dd4c8a69b6a..9d34b037c23 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java @@ -211,18 +211,18 @@ class SimpleTextFieldsReader extends FieldsProducer { } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { SimpleTextDocsEnum docsEnum; if (reuse != null && reuse instanceof SimpleTextDocsEnum && ((SimpleTextDocsEnum) reuse).canReuse(in)) { docsEnum = (SimpleTextDocsEnum) reuse; } else { docsEnum = new SimpleTextDocsEnum(); } - return docsEnum.reset(docsStart, skipDocs, omitTF); + return docsEnum.reset(docsStart, liveDocs, omitTF); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { if (omitTF) { return null; } @@ -233,7 +233,7 @@ class SimpleTextFieldsReader extends FieldsProducer { } else { docsAndPositionsEnum = new SimpleTextDocsAndPositionsEnum(); } - return docsAndPositionsEnum.reset(docsStart, skipDocs); + return docsAndPositionsEnum.reset(docsStart, liveDocs); } @Override @@ -248,7 +248,7 @@ class SimpleTextFieldsReader extends FieldsProducer { private boolean omitTF; private int docID; private int tf; - private Bits skipDocs; + private Bits liveDocs; private final BytesRef scratch = new BytesRef(10); private final CharsRef scratchUTF16 = new CharsRef(10); @@ -261,8 +261,8 @@ class SimpleTextFieldsReader extends FieldsProducer { return in == inStart; } - public SimpleTextDocsEnum reset(long fp, Bits skipDocs, boolean omitTF) throws IOException { - this.skipDocs = skipDocs; + public SimpleTextDocsEnum reset(long fp, Bits liveDocs, boolean omitTF) throws IOException { + this.liveDocs = liveDocs; in.seek(fp); this.omitTF = omitTF; if (omitTF) { @@ -292,7 +292,7 @@ class SimpleTextFieldsReader extends FieldsProducer { final long lineStart = in.getFilePointer(); readLine(in, scratch); if (scratch.startsWith(DOC)) { - if (!first && (skipDocs == null || !skipDocs.get(docID))) { + if (!first && (liveDocs == null || liveDocs.get(docID))) { in.seek(lineStart); if (!omitTF) { tf = termFreq; @@ -309,7 +309,7 @@ class SimpleTextFieldsReader extends FieldsProducer { // skip } else { assert scratch.startsWith(TERM) || scratch.startsWith(FIELD) || scratch.startsWith(END): "scratch=" + scratch.utf8ToString(); - if (!first && (skipDocs == null || !skipDocs.get(docID))) { + if (!first && (liveDocs == null || liveDocs.get(docID))) { in.seek(lineStart); if (!omitTF) { tf = termFreq; @@ -334,7 +334,7 @@ class SimpleTextFieldsReader extends FieldsProducer { private final IndexInput in; private int docID; private int tf; - private Bits skipDocs; + private Bits liveDocs; private final BytesRef scratch = new BytesRef(10); private final BytesRef scratch2 = new BytesRef(10); private final CharsRef scratchUTF16 = new CharsRef(10); @@ -351,8 +351,8 @@ class SimpleTextFieldsReader extends FieldsProducer { return in == inStart; } - public SimpleTextDocsAndPositionsEnum reset(long fp, Bits skipDocs) { - this.skipDocs = skipDocs; + public SimpleTextDocsAndPositionsEnum reset(long fp, Bits liveDocs) { + this.liveDocs = liveDocs; nextDocStart = fp; return this; } @@ -376,7 +376,7 @@ class SimpleTextFieldsReader extends FieldsProducer { final long lineStart = in.getFilePointer(); readLine(in, scratch); if (scratch.startsWith(DOC)) { - if (!first && (skipDocs == null || !skipDocs.get(docID))) { + if (!first && (liveDocs == null || liveDocs.get(docID))) { nextDocStart = lineStart; in.seek(posStart); return docID; @@ -392,7 +392,7 @@ class SimpleTextFieldsReader extends FieldsProducer { // skip } else { assert scratch.startsWith(TERM) || scratch.startsWith(FIELD) || scratch.startsWith(END); - if (!first && (skipDocs == null || !skipDocs.get(docID))) { + if (!first && (liveDocs == null || liveDocs.get(docID))) { nextDocStart = lineStart; in.seek(posStart); return docID; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java index 52717615b22..51243c93365 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java @@ -200,7 +200,7 @@ public class StandardPostingsReader extends PostingsReaderBase { } @Override - public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsEnum reuse) throws IOException { SegmentDocsEnum docsEnum; if (reuse == null || !(reuse instanceof SegmentDocsEnum)) { docsEnum = new SegmentDocsEnum(freqIn); @@ -213,11 +213,11 @@ public class StandardPostingsReader extends PostingsReaderBase { docsEnum = new SegmentDocsEnum(freqIn); } } - return docsEnum.reset(fieldInfo, (StandardTermState) termState, skipDocs); + return docsEnum.reset(fieldInfo, (StandardTermState) termState, liveDocs); } @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { if (fieldInfo.omitTermFreqAndPositions) { return null; } @@ -236,7 +236,7 @@ public class StandardPostingsReader extends PostingsReaderBase { docsEnum = new SegmentDocsAndPositionsAndPayloadsEnum(freqIn, proxIn); } } - return docsEnum.reset(fieldInfo, (StandardTermState) termState, skipDocs); + return docsEnum.reset(fieldInfo, (StandardTermState) termState, liveDocs); } else { SegmentDocsAndPositionsEnum docsEnum; if (reuse == null || !(reuse instanceof SegmentDocsAndPositionsEnum)) { @@ -250,7 +250,7 @@ public class StandardPostingsReader extends PostingsReaderBase { docsEnum = new SegmentDocsAndPositionsEnum(freqIn, proxIn); } } - return docsEnum.reset(fieldInfo, (StandardTermState) termState, skipDocs); + return docsEnum.reset(fieldInfo, (StandardTermState) termState, liveDocs); } } @@ -267,7 +267,7 @@ public class StandardPostingsReader extends PostingsReaderBase { int doc; // doc we last read int freq; // freq we last read - Bits skipDocs; + Bits liveDocs; long freqOffset; int skipOffset; @@ -280,13 +280,13 @@ public class StandardPostingsReader extends PostingsReaderBase { this.freqIn = (IndexInput) freqIn.clone(); } - public SegmentDocsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits skipDocs) throws IOException { + public SegmentDocsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException { omitTF = fieldInfo.omitTermFreqAndPositions; if (omitTF) { freq = 1; } storePayloads = fieldInfo.storePayloads; - this.skipDocs = skipDocs; + this.liveDocs = liveDocs; freqOffset = termState.freqOffset; skipOffset = termState.skipOffset; @@ -327,7 +327,7 @@ public class StandardPostingsReader extends PostingsReaderBase { } } - if (skipDocs == null || !skipDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(doc)) { break; } } @@ -357,7 +357,7 @@ public class StandardPostingsReader extends PostingsReaderBase { } } - if (skipDocs == null || !skipDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(doc)) { docs[i] = doc; freqs[i] = freq; ++i; @@ -435,7 +435,7 @@ public class StandardPostingsReader extends PostingsReaderBase { int freq; // freq we last read int position; - Bits skipDocs; + Bits liveDocs; long freqOffset; int skipOffset; @@ -453,11 +453,11 @@ public class StandardPostingsReader extends PostingsReaderBase { this.proxIn = (IndexInput) proxIn.clone(); } - public SegmentDocsAndPositionsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits skipDocs) throws IOException { + public SegmentDocsAndPositionsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException { assert !fieldInfo.omitTermFreqAndPositions; assert !fieldInfo.storePayloads; - this.skipDocs = skipDocs; + this.liveDocs = liveDocs; // TODO: for full enum case (eg segment merging) this // seek is unnecessary; maybe we can avoid in such @@ -504,7 +504,7 @@ public class StandardPostingsReader extends PostingsReaderBase { } posPendingCount += freq; - if (skipDocs == null || !skipDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(doc)) { break; } } @@ -626,7 +626,7 @@ public class StandardPostingsReader extends PostingsReaderBase { int freq; // freq we last read int position; - Bits skipDocs; + Bits liveDocs; long freqOffset; int skipOffset; @@ -647,7 +647,7 @@ public class StandardPostingsReader extends PostingsReaderBase { this.proxIn = (IndexInput) proxIn.clone(); } - public SegmentDocsAndPositionsAndPayloadsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits skipDocs) throws IOException { + public SegmentDocsAndPositionsAndPayloadsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException { assert !fieldInfo.omitTermFreqAndPositions; assert fieldInfo.storePayloads; if (payload == null) { @@ -655,7 +655,7 @@ public class StandardPostingsReader extends PostingsReaderBase { payload.bytes = new byte[1]; } - this.skipDocs = skipDocs; + this.liveDocs = liveDocs; // TODO: for full enum case (eg segment merging) this // seek is unnecessary; maybe we can avoid in such @@ -701,7 +701,7 @@ public class StandardPostingsReader extends PostingsReaderBase { } posPendingCount += freq; - if (skipDocs == null || !skipDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(doc)) { break; } } diff --git a/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java index 5e56cda08ca..a50002762b1 100644 --- a/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java @@ -103,7 +103,7 @@ class FixedStraightBytesImpl { datOut = getDataOut(); boolean success = false; try { - if (state.bits == null && state.reader instanceof Reader) { + if (state.liveDocs == null && state.reader instanceof Reader) { Reader reader = (Reader) state.reader; final int maxDocs = reader.maxDoc; if (maxDocs == 0) { diff --git a/lucene/src/java/org/apache/lucene/index/values/Floats.java b/lucene/src/java/org/apache/lucene/index/values/Floats.java index ae5287b9fb3..6a35a855bca 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Floats.java +++ b/lucene/src/java/org/apache/lucene/index/values/Floats.java @@ -131,7 +131,7 @@ public class Floats { if (datOut == null) { initDataOut(); } - if (state.bits == null && state.reader instanceof FloatsReader) { + if (state.liveDocs == null && state.reader instanceof FloatsReader) { // no deletes - bulk copy final FloatsReader reader = (FloatsReader) state.reader; assert reader.precisionBytes == (int) precision; diff --git a/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java b/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java index ac19d2ec761..0e64e25f18a 100644 --- a/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java @@ -182,7 +182,7 @@ class IntsImpl { merging = true; if (typeOrd != PACKED) { initDataOut(typeOrd); // init datOut since we merge directly - if (state.bits == null && state.reader instanceof IntsReader) { + if (state.liveDocs == null && state.reader instanceof IntsReader) { // no deleted docs - try bulk copy final IntsReader reader = (IntsReader) state.reader; if (reader.type == typeOrd) { diff --git a/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java index 88e68a56f59..6128100dc8f 100644 --- a/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java @@ -97,7 +97,7 @@ class VarStraightBytesImpl { datOut = getDataOut(); boolean success = false; try { - if (state.bits == null && state.reader instanceof Reader) { + if (state.liveDocs == null && state.reader instanceof Reader) { // bulk merge since we don't have any deletes Reader reader = (Reader) state.reader; final int maxDocs = reader.maxDoc; diff --git a/lucene/src/java/org/apache/lucene/index/values/Writer.java b/lucene/src/java/org/apache/lucene/index/values/Writer.java index 5b12082f688..abcbbcef87c 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Writer.java +++ b/lucene/src/java/org/apache/lucene/index/values/Writer.java @@ -147,12 +147,12 @@ public abstract class Writer extends DocValuesConsumer { // impl. will get the correct reference for the type // it supports int docID = state.docBase; - final Bits bits = state.bits; + final Bits liveDocs = state.liveDocs; final int docCount = state.docCount; int currentDocId; if ((currentDocId = valEnum.advance(0)) != ValuesEnum.NO_MORE_DOCS) { for (int i = 0; i < docCount; i++) { - if (bits == null || !bits.get(i)) { + if (liveDocs == null || liveDocs.get(i)) { if (currentDocId < i) { if ((currentDocId = valEnum.advance(i)) == ValuesEnum.NO_MORE_DOCS) { break; // advance can jump over default values diff --git a/lucene/src/java/org/apache/lucene/search/CachingSpanFilter.java b/lucene/src/java/org/apache/lucene/search/CachingSpanFilter.java index e1341fe493a..3fe79d71564 100644 --- a/lucene/src/java/org/apache/lucene/search/CachingSpanFilter.java +++ b/lucene/src/java/org/apache/lucene/search/CachingSpanFilter.java @@ -54,7 +54,7 @@ public class CachingSpanFilter extends SpanFilter { } this.cache = new CachingWrapperFilter.FilterCache(deletesMode) { @Override - protected SpanFilterResult mergeDeletes(final Bits delDocs, final SpanFilterResult value) { + protected SpanFilterResult mergeLiveDocs(final Bits liveDocs, final SpanFilterResult value) { throw new IllegalStateException("DeletesMode.DYNAMIC is not supported"); } }; @@ -73,7 +73,7 @@ public class CachingSpanFilter extends SpanFilter { final IndexReader reader = context.reader; final Object coreKey = reader.getCoreCacheKey(); - final Object delCoreKey = reader.hasDeletions() ? reader.getDeletedDocs() : coreKey; + final Object delCoreKey = reader.hasDeletions() ? reader.getLiveDocs() : coreKey; SpanFilterResult result = cache.get(reader, coreKey, delCoreKey); if (result != null) { diff --git a/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java b/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java index e726aa8d268..ab57997f731 100644 --- a/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java +++ b/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java @@ -103,13 +103,13 @@ public class CachingWrapperFilter extends Filter { value = cache.get(delCoreKey); if (value == null) { - // now for core match, but dynamically AND NOT - // deletions + // now for core match, but dynamically AND + // live docs value = cache.get(coreKey); if (value != null) { - final Bits delDocs = reader.getDeletedDocs(); - if (delDocs != null) { - value = mergeDeletes(delDocs, value); + final Bits liveDocs = reader.getLiveDocs(); + if (liveDocs != null) { + value = mergeLiveDocs(liveDocs, value); } } } @@ -118,7 +118,7 @@ public class CachingWrapperFilter extends Filter { return value; } - protected abstract T mergeDeletes(Bits delDocs, T value); + protected abstract T mergeLiveDocs(Bits liveDocs, T value); public synchronized void put(Object coreKey, Object delCoreKey, T value) { if (deletesMode == DeletesMode.IGNORE) { @@ -158,11 +158,11 @@ public class CachingWrapperFilter extends Filter { this.filter = filter; cache = new FilterCache(deletesMode) { @Override - public DocIdSet mergeDeletes(final Bits delDocs, final DocIdSet docIdSet) { + public DocIdSet mergeLiveDocs(final Bits liveDocs, final DocIdSet docIdSet) { return new FilteredDocIdSet(docIdSet) { @Override - protected boolean match(int docID) { - return !delDocs.get(docID); + protected boolean match(int docID) { + return liveDocs.get(docID); } }; } @@ -197,7 +197,7 @@ public class CachingWrapperFilter extends Filter { public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException { final IndexReader reader = context.reader; final Object coreKey = reader.getCoreCacheKey(); - final Object delCoreKey = reader.hasDeletions() ? reader.getDeletedDocs() : coreKey; + final Object delCoreKey = reader.hasDeletions() ? reader.getLiveDocs() : coreKey; DocIdSet docIdSet = cache.get(reader, coreKey, delCoreKey); if (docIdSet != null) { diff --git a/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java b/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java index d5f5f50389b..85007dfe74b 100644 --- a/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java +++ b/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java @@ -136,16 +136,18 @@ public class ConstantScoreQuery extends Query { if (filter != null) { assert query == null; final DocIdSet dis = filter.getDocIdSet(context); - if (dis == null) + if (dis == null) { return null; + } disi = dis.iterator(); } else { assert query != null && innerWeight != null; - disi = - innerWeight.scorer(context, scorerContext); + disi = innerWeight.scorer(context, scorerContext); } - if (disi == null) + + if (disi == null) { return null; + } return new ConstantScorer(disi, this); } diff --git a/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java b/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java index 9293e509608..e14ded87e3e 100644 --- a/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java +++ b/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java @@ -20,7 +20,6 @@ import java.io.IOException; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.MultiFields; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -533,9 +532,9 @@ public abstract class FieldCacheRangeFilter extends Filter { @Override public DocIdSetIterator iterator() throws IOException { - final Bits skipDocs = canIgnoreDeletedDocs ? null : reader.getDeletedDocs(); + final Bits liveDocs = canIgnoreDeletedDocs ? null : reader.getLiveDocs(); - if (skipDocs == null) { + if (liveDocs == null) { // Specialization optimization disregard deletions return new DocIdSetIterator() { private int doc = -1; @@ -575,7 +574,7 @@ public abstract class FieldCacheRangeFilter extends Filter { final int maxDoc = reader.maxDoc(); // a DocIdSetIterator generating docIds by - // incrementing a variable & checking skipDocs - + // incrementing a variable & checking liveDocs - return new DocIdSetIterator() { private int doc = -1; @Override @@ -590,14 +589,14 @@ public abstract class FieldCacheRangeFilter extends Filter { if (doc >= maxDoc) { return doc = NO_MORE_DOCS; } - } while (skipDocs.get(doc) || !matchDoc(doc)); + } while (!liveDocs.get(doc) || !matchDoc(doc)); return doc; } @Override public int advance(int target) { for(doc=target;doc docsEnums = new LinkedList(); - final Bits delDocs = indexReader.getDeletedDocs(); + final Bits liveDocs = indexReader.getLiveDocs(); for (int i = 0; i < terms.length; i++) { - DocsAndPositionsEnum postings = indexReader.termPositionsEnum(delDocs, + DocsAndPositionsEnum postings = indexReader.termPositionsEnum(liveDocs, terms[i].field(), terms[i].bytes()); if (postings != null) { docsEnums.add(postings); } else { - if (indexReader.termDocsEnum(delDocs, terms[i].field(), terms[i].bytes()) != null) { + if (indexReader.termDocsEnum(liveDocs, terms[i].field(), terms[i].bytes()) != null) { // term does exist, but has no positions throw new IllegalStateException("field \"" + terms[i].field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + terms[i].text() + ")"); } diff --git a/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java b/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java index 8a6df063b83..3b16f64e97e 100644 --- a/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java +++ b/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java @@ -19,14 +19,14 @@ package org.apache.lucene.search; import java.io.IOException; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Fields; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.util.OpenBitSet; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.OpenBitSet; /** * A wrapper for {@link MultiTermQuery}, that exposes its @@ -123,15 +123,15 @@ public class MultiTermQueryWrapperFilter extends Filte assert termsEnum != null; if (termsEnum.next() != null) { // fill into a OpenBitSet - final OpenBitSet bitSet = new OpenBitSet(context.reader.maxDoc()); + final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc()); int termCount = 0; - final Bits delDocs = reader.getDeletedDocs(); + final Bits liveDocs = reader.getLiveDocs(); DocsEnum docsEnum = null; do { termCount++; // System.out.println(" iter termCount=" + termCount + " term=" + // enumerator.term().toBytesString()); - docsEnum = termsEnum.docs(delDocs, docsEnum); + docsEnum = termsEnum.docs(liveDocs, docsEnum); final DocsEnum.BulkReadResult result = docsEnum.getBulkResult(); while (true) { final int count = docsEnum.read(); diff --git a/lucene/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/src/java/org/apache/lucene/search/PhraseQuery.java index 2ba8d880328..300e63f30e1 100644 --- a/lucene/src/java/org/apache/lucene/search/PhraseQuery.java +++ b/lucene/src/java/org/apache/lucene/search/PhraseQuery.java @@ -212,17 +212,17 @@ public class PhraseQuery extends Query { if (terms.size() == 0) // optimize zero-term case return null; final IndexReader reader = context.reader; + final Bits liveDocs = reader.getLiveDocs(); PostingsAndFreq[] postingsFreqs = new PostingsAndFreq[terms.size()]; - final Bits delDocs = reader.getDeletedDocs(); for (int i = 0; i < terms.size(); i++) { final Term t = terms.get(i); - DocsAndPositionsEnum postingsEnum = reader.termPositionsEnum(delDocs, + DocsAndPositionsEnum postingsEnum = reader.termPositionsEnum(liveDocs, t.field(), t.bytes()); // PhraseQuery on a field that did not index // positions. if (postingsEnum == null) { - if (reader.termDocsEnum(delDocs, t.field(), t.bytes()) != null) { + if (reader.termDocsEnum(liveDocs, t.field(), t.bytes()) != null) { // term does exist, but has no positions throw new IllegalStateException("field \"" + t.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + t.text() + ")"); } else { diff --git a/lucene/src/java/org/apache/lucene/search/TermQuery.java b/lucene/src/java/org/apache/lucene/search/TermQuery.java index 290d106c0f9..078d02f7089 100644 --- a/lucene/src/java/org/apache/lucene/search/TermQuery.java +++ b/lucene/src/java/org/apache/lucene/search/TermQuery.java @@ -90,13 +90,12 @@ public class TermQuery extends Query { final String field = term.field(); final IndexReader reader = context.reader; assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termStates.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context); - final TermState state = termStates - .get(context.ord); + final TermState state = termStates.get(context.ord); if (state == null) { // term is not present in that reader assert termNotInReader(reader, field, term.bytes()) : "no termstate found but term exists in reader"; return null; } - final DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), field, term.bytes(), state); + final DocsEnum docs = reader.termDocsEnum(reader.getLiveDocs(), field, term.bytes(), state); assert docs != null; return new TermScorer(this, docs, similarity, context.reader.norms(field)); } @@ -143,7 +142,7 @@ public class TermQuery extends Query { Explanation tfExplanation = new Explanation(); int tf = 0; - DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), term.field(), term.bytes()); + DocsEnum docs = reader.termDocsEnum(context.reader.getLiveDocs(), term.field(), term.bytes()); if (docs != null) { int newDoc = docs.advance(doc); if (newDoc == doc) { diff --git a/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java b/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java index 5cadcbd54b1..7505ea6b858 100644 --- a/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java +++ b/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java @@ -105,7 +105,7 @@ public class DocTermsCreator extends EntryCreatorWithOptions if (terms != null) { int termCount = 0; final TermsEnum termsEnum = terms.iterator(); - final Bits delDocs = MultiFields.getDeletedDocs(reader); + final Bits liveDocs = MultiFields.getLiveDocs(reader); DocsEnum docs = null; while(true) { if (termCount++ == termCountHardLimit) { @@ -120,7 +120,7 @@ public class DocTermsCreator extends EntryCreatorWithOptions break; } final long pointer = bytes.copyUsingLengthPrefix(term); - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(liveDocs, docs); while (true) { final int docID = docs.nextDoc(); if (docID == DocIdSetIterator.NO_MORE_DOCS) { diff --git a/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java b/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java index 1cc92e95292..dc35006bffc 100644 --- a/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java +++ b/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java @@ -322,12 +322,12 @@ public class DocTermsIndexCreator extends EntryCreatorWithOptions } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { throw new UnsupportedOperationException(); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { throw new UnsupportedOperationException(); } diff --git a/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java b/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java index 33347861fda..8543627c89a 100644 --- a/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java +++ b/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java @@ -83,14 +83,14 @@ public class SpanTermQuery extends SpanQuery { @Override public Spans getSpans(final AtomicReaderContext context) throws IOException { final IndexReader reader = context.reader; - final DocsAndPositionsEnum postings = reader.termPositionsEnum(reader.getDeletedDocs(), + final DocsAndPositionsEnum postings = reader.termPositionsEnum(reader.getLiveDocs(), term.field(), term.bytes()); if (postings != null) { return new TermSpans(postings, term); } else { - if (reader.termDocsEnum(reader.getDeletedDocs(), term.field(), term.bytes()) != null) { + if (reader.termDocsEnum(reader.getLiveDocs(), term.field(), term.bytes()) != null) { // term does exist, but has no positions throw new IllegalStateException("field \"" + term.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run SpanTermQuery (term=" + term.text() + ")"); } else { diff --git a/lucene/src/java/org/apache/lucene/util/BitVector.java b/lucene/src/java/org/apache/lucene/util/BitVector.java index 505eca22484..c664df06782 100644 --- a/lucene/src/java/org/apache/lucene/util/BitVector.java +++ b/lucene/src/java/org/apache/lucene/util/BitVector.java @@ -18,6 +18,7 @@ package org.apache.lucene.util; */ import java.io.IOException; +import java.util.Arrays; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -39,6 +40,7 @@ public final class BitVector implements Cloneable, Bits { private byte[] bits; private int size; private int count; + private int version; /** Constructs a vector capable of holding n bits. */ public BitVector(int n) { @@ -92,8 +94,10 @@ public final class BitVector implements Cloneable, Bits { return true; else { bits[pos] = (byte) (v | flag); - if (count != -1) + if (count != -1) { count++; + assert count <= size; + } return false; } } @@ -107,6 +111,25 @@ public final class BitVector implements Cloneable, Bits { count = -1; } + public final boolean getAndClear(int bit) { + if (bit >= size) { + throw new ArrayIndexOutOfBoundsException(bit); + } + final int pos = bit >> 3; + final int v = bits[pos]; + final int flag = 1 << (bit & 7); + if ((flag & v) == 0) { + return false; + } else { + bits[pos] &= ~flag; + if (count != -1) { + count--; + assert count >= 0; + } + return true; + } + } + /** Returns true if bit is one and false if it is zero. */ public final boolean get(int bit) { @@ -133,8 +156,9 @@ public final class BitVector implements Cloneable, Bits { if (count == -1) { int c = 0; int end = bits.length; - for (int i = 0; i < end; i++) + for (int i = 0; i < end; i++) { c += BYTE_COUNTS[bits[i] & 0xFF]; // sum bits per byte + } count = c; } return count; @@ -144,8 +168,9 @@ public final class BitVector implements Cloneable, Bits { public final int getRecomputedCount() { int c = 0; int end = bits.length; - for (int i = 0; i < end; i++) + for (int i = 0; i < end; i++) { c += BYTE_COUNTS[bits[i] & 0xFF]; // sum bits per byte + } return c; } @@ -171,13 +196,21 @@ public final class BitVector implements Cloneable, Bits { private static String CODEC = "BitVector"; // Version before version tracking was added: - private final static int VERSION_PRE = -1; + public final static int VERSION_PRE = -1; // First version: - private final static int VERSION_START = 0; + public final static int VERSION_START = 0; + + // Changed DGaps to encode gaps between cleared bits, not + // set: + public final static int VERSION_DGAPS_CLEARED = 1; // Increment version to change it: - private final static int VERSION_CURRENT = VERSION_START; + public final static int VERSION_CURRENT = VERSION_DGAPS_CLEARED; + + public int getVersion() { + return version; + } /** Writes this vector to the file name in Directory d, in a format that can be read by the constructor {@link @@ -188,7 +221,8 @@ public final class BitVector implements Cloneable, Bits { output.writeInt(-2); CodecUtil.writeHeader(output, CODEC, VERSION_CURRENT); if (isSparse()) { - writeDgaps(output); // sparse bit-set more efficiently saved as d-gaps. + // sparse bit-set more efficiently saved as d-gaps. + writeClearedDgaps(output); } else { writeBits(output); } @@ -196,6 +230,38 @@ public final class BitVector implements Cloneable, Bits { output.close(); } } + + /** Invert all bits */ + public void invertAll() { + if (count != -1) { + count = size - count; + } + if (bits.length > 0) { + for(int idx=0;idx 0) { + final int lastNBits = size & 7; + if (lastNBits != 0) { + final int mask = (1 << lastNBits)-1; + bits[bits.length-1] &= mask; + } + } + } + + /** Set all bits */ + public void setAll() { + Arrays.fill(bits, (byte) 0xff); + clearUnusedBits(); + count = size; + } /** Write as a bit set */ private void writeBits(IndexOutput output) throws IOException { @@ -205,19 +271,20 @@ public final class BitVector implements Cloneable, Bits { } /** Write as a d-gaps list */ - private void writeDgaps(IndexOutput output) throws IOException { + private void writeClearedDgaps(IndexOutput output) throws IOException { output.writeInt(-1); // mark using d-gaps output.writeInt(size()); // write size output.writeInt(count()); // write count int last=0; - int n = count(); + int numCleared = size()-count(); int m = bits.length; - for (int i=0; i0; i++) { - if (bits[i]!=0) { + for (int i=0; i0; i++) { + if (bits[i]!=0xff) { output.writeVInt(i-last); output.writeByte(bits[i]); last = i; - n -= BYTE_COUNTS[bits[i] & 0xFF]; + numCleared -= (8-BYTE_COUNTS[bits[i] & 0xFF]); + assert numCleared >= 0; } } } @@ -225,12 +292,12 @@ public final class BitVector implements Cloneable, Bits { /** Indicates if the bit vector is sparse and should be saved as a d-gaps list, or dense, and should be saved as a bit set. */ private boolean isSparse() { - final int setCount = count(); - if (setCount == 0) { + final int clearedCount = size() - count(); + if (clearedCount == 0) { return true; } - final int avgGapLength = bits.length / setCount; + final int avgGapLength = bits.length / clearedCount; // expected number of bytes for vInt encoding of each gap final int expectedDGapBytes; @@ -266,17 +333,21 @@ public final class BitVector implements Cloneable, Bits { try { final int firstInt = input.readInt(); - final int version; + if (firstInt == -2) { // New format, with full header & version: - version = CodecUtil.checkHeader(input, CODEC, VERSION_START, VERSION_START); + version = CodecUtil.checkHeader(input, CODEC, VERSION_START, VERSION_CURRENT); size = input.readInt(); } else { version = VERSION_PRE; size = firstInt; } if (size == -1) { - readDgaps(input); + if (version >= VERSION_DGAPS_CLEARED) { + readClearedDgaps(input); + } else { + readSetDgaps(input); + } } else { readBits(input); } @@ -293,7 +364,7 @@ public final class BitVector implements Cloneable, Bits { } /** read as a d-gaps list */ - private void readDgaps(IndexInput input) throws IOException { + private void readSetDgaps(IndexInput input) throws IOException { size = input.readInt(); // (re)read size count = input.readInt(); // read count bits = new byte[(size >> 3) + 1]; // allocate bits @@ -303,6 +374,24 @@ public final class BitVector implements Cloneable, Bits { last += input.readVInt(); bits[last] = input.readByte(); n -= BYTE_COUNTS[bits[last] & 0xFF]; + assert n >= 0; } } + + /** read as a d-gaps cleared bits list */ + private void readClearedDgaps(IndexInput input) throws IOException { + size = input.readInt(); // (re)read size + count = input.readInt(); // read count + bits = new byte[(size >> 3) + 1]; // allocate bits + Arrays.fill(bits, (byte) 0xff); + clearUnusedBits(); + int last=0; + int numCleared = size()-count(); + while (numCleared>0) { + last += input.readVInt(); + bits[last] = input.readByte(); + numCleared -= 8-BYTE_COUNTS[bits[last] & 0xFF]; + assert numCleared >= 0; + } + } } diff --git a/lucene/src/java/org/apache/lucene/util/MultiBits.java b/lucene/src/java/org/apache/lucene/util/MultiBits.java index c31698856c4..e8bd59461a9 100644 --- a/lucene/src/java/org/apache/lucene/util/MultiBits.java +++ b/lucene/src/java/org/apache/lucene/util/MultiBits.java @@ -34,13 +34,16 @@ public final class MultiBits implements Bits { // length is 1+subs.length (the last entry has the maxDoc): private final int[] starts; - public MultiBits(List bits, List starts) { + private final boolean defaultValue; + + public MultiBits(List bits, List starts, boolean defaultValue) { assert starts.size() == 1+bits.size(); this.subs = bits.toArray(Bits.EMPTY_ARRAY); this.starts = new int[starts.size()]; for(int i=0;i>32) ^ h) + 0x98761234; } - } diff --git a/lucene/src/test/org/apache/lucene/TestExternalCodecs.java b/lucene/src/test/org/apache/lucene/TestExternalCodecs.java index e5c821f3102..c3625abc4cf 100644 --- a/lucene/src/test/org/apache/lucene/TestExternalCodecs.java +++ b/lucene/src/test/org/apache/lucene/TestExternalCodecs.java @@ -346,26 +346,26 @@ public class TestExternalCodecs extends LuceneTestCase { } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) { - return new RAMDocsEnum(ramField.termToDocs.get(current), skipDocs); + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) { + return new RAMDocsEnum(ramField.termToDocs.get(current), liveDocs); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) { - return new RAMDocsAndPositionsEnum(ramField.termToDocs.get(current), skipDocs); + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) { + return new RAMDocsAndPositionsEnum(ramField.termToDocs.get(current), liveDocs); } } private static class RAMDocsEnum extends DocsEnum { private final RAMTerm ramTerm; - private final Bits skipDocs; + private final Bits liveDocs; private RAMDoc current; int upto = -1; int posUpto = 0; - public RAMDocsEnum(RAMTerm ramTerm, Bits skipDocs) { + public RAMDocsEnum(RAMTerm ramTerm, Bits liveDocs) { this.ramTerm = ramTerm; - this.skipDocs = skipDocs; + this.liveDocs = liveDocs; } @Override @@ -383,7 +383,7 @@ public class TestExternalCodecs extends LuceneTestCase { upto++; if (upto < ramTerm.docs.size()) { current = ramTerm.docs.get(upto); - if (skipDocs == null || !skipDocs.get(current.docID)) { + if (liveDocs == null || liveDocs.get(current.docID)) { posUpto = 0; return current.docID; } @@ -406,14 +406,14 @@ public class TestExternalCodecs extends LuceneTestCase { private static class RAMDocsAndPositionsEnum extends DocsAndPositionsEnum { private final RAMTerm ramTerm; - private final Bits skipDocs; + private final Bits liveDocs; private RAMDoc current; int upto = -1; int posUpto = 0; - public RAMDocsAndPositionsEnum(RAMTerm ramTerm, Bits skipDocs) { + public RAMDocsAndPositionsEnum(RAMTerm ramTerm, Bits liveDocs) { this.ramTerm = ramTerm; - this.skipDocs = skipDocs; + this.liveDocs = liveDocs; } @Override @@ -431,7 +431,7 @@ public class TestExternalCodecs extends LuceneTestCase { upto++; if (upto < ramTerm.docs.size()) { current = ramTerm.docs.get(upto); - if (skipDocs == null || !skipDocs.get(current.docID)) { + if (liveDocs == null || liveDocs.get(current.docID)) { posUpto = 0; return current.docID; } @@ -560,6 +560,9 @@ public class TestExternalCodecs extends LuceneTestCase { r.close(); s.close(); + if (VERBOSE) { + System.out.println("\nTEST: now delete 2nd doc"); + } w.deleteDocuments(new Term("id", "44")); w.optimize(); r = IndexReader.open(w, true); diff --git a/lucene/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java b/lucene/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java index 2e058770d45..3f68400f7d4 100644 --- a/lucene/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java +++ b/lucene/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java @@ -73,7 +73,7 @@ public class TestCachingTokenFilter extends BaseTokenStreamTestCase { IndexReader reader = writer.getReader(); DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getLiveDocs(reader), "preanalyzed", new BytesRef("term1")); assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS); @@ -81,7 +81,7 @@ public class TestCachingTokenFilter extends BaseTokenStreamTestCase { assertEquals(0, termPositions.nextPosition()); termPositions = MultiFields.getTermPositionsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getLiveDocs(reader), "preanalyzed", new BytesRef("term2")); assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS); @@ -90,7 +90,7 @@ public class TestCachingTokenFilter extends BaseTokenStreamTestCase { assertEquals(3, termPositions.nextPosition()); termPositions = MultiFields.getTermPositionsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getLiveDocs(reader), "preanalyzed", new BytesRef("term3")); assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS); diff --git a/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index ba5f4f5917b..bdd53b3e9aa 100644 --- a/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -281,10 +281,10 @@ public class TestBackwardsCompatibility extends LuceneTestCase { _TestUtil.checkIndex(dir); - final Bits delDocs = MultiFields.getDeletedDocs(reader); + final Bits liveDocs = MultiFields.getLiveDocs(reader); for(int i=0;i<35;i++) { - if (!delDocs.get(i)) { + if (liveDocs.get(i)) { Document d = reader.document(i); List fields = d.getFields(); if (d.getField("content3") == null) { diff --git a/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java b/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java index 8e71ca8d19e..f46fd29fbe8 100644 --- a/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java +++ b/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java @@ -611,6 +611,9 @@ public class TestDeletionPolicy extends LuceneTestCase { final int N = 10; for(int pass=0;pass<2;pass++) { + if (VERBOSE) { + System.out.println("TEST: pass=" + pass); + } boolean useCompoundFile = (pass % 2) != 0; @@ -631,7 +634,7 @@ public class TestDeletionPolicy extends LuceneTestCase { for(int i=0;i fieldable1 = doc1.getFields(); @@ -880,15 +880,15 @@ public class TestIndexReader extends LuceneTestCase FieldsEnum fenum1 = MultiFields.getFields(index1).iterator(); FieldsEnum fenum2 = MultiFields.getFields(index1).iterator(); String field1 = null; - Bits delDocs = MultiFields.getDeletedDocs(index1); + Bits liveDocs = MultiFields.getLiveDocs(index1); while((field1=fenum1.next()) != null) { assertEquals("Different fields", field1, fenum2.next()); TermsEnum enum1 = fenum1.terms(); TermsEnum enum2 = fenum2.terms(); while(enum1.next() != null) { assertEquals("Different terms", enum1.term(), enum2.next()); - DocsAndPositionsEnum tp1 = enum1.docsAndPositions(delDocs, null); - DocsAndPositionsEnum tp2 = enum2.docsAndPositions(delDocs, null); + DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null); + DocsAndPositionsEnum tp2 = enum2.docsAndPositions(liveDocs, null); while(tp1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { assertTrue(tp2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java b/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java index 87e866829b3..bcbf857a195 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java @@ -29,8 +29,8 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.Bits; /** - * Tests cloning multiple types of readers, modifying the deletedDocs and norms - * and verifies copy on write semantics of the deletedDocs and norms is + * Tests cloning multiple types of readers, modifying the liveDocs and norms + * and verifies copy on write semantics of the liveDocs and norms is * implemented properly */ public class TestIndexReaderClone extends LuceneTestCase { @@ -282,9 +282,9 @@ public class TestIndexReaderClone extends LuceneTestCase { assertTrue(sim.decodeNormValue(MultiNorms.norms(r1, "field1")[4]) == norm1); assertTrue(sim.decodeNormValue(MultiNorms.norms(pr1Clone, "field1")[4]) != norm1); - final Bits delDocs = MultiFields.getDeletedDocs(r1); - assertTrue(delDocs == null || !delDocs.get(10)); - assertTrue(MultiFields.getDeletedDocs(pr1Clone).get(10)); + final Bits liveDocs = MultiFields.getLiveDocs(r1); + assertTrue(liveDocs == null || liveDocs.get(10)); + assertFalse(MultiFields.getLiveDocs(pr1Clone).get(10)); // try to update the original reader, which should throw an exception try { @@ -318,7 +318,7 @@ public class TestIndexReaderClone extends LuceneTestCase { origSegmentReader.deleteDocument(10); assertDelDocsRefCountEquals(1, origSegmentReader); origSegmentReader.undeleteAll(); - assertNull(origSegmentReader.deletedDocsRef); + assertNull(origSegmentReader.liveDocsRef); origSegmentReader.close(); // need to test norms? dir1.close(); @@ -350,10 +350,10 @@ public class TestIndexReaderClone extends LuceneTestCase { IndexReader origReader = IndexReader.open(dir1, false); SegmentReader origSegmentReader = getOnlySegmentReader(origReader); - // deletedDocsRef should be null because nothing has updated yet - assertNull(origSegmentReader.deletedDocsRef); + // liveDocsRef should be null because nothing has updated yet + assertNull(origSegmentReader.liveDocsRef); - // we deleted a document, so there is now a deletedDocs bitvector and a + // we deleted a document, so there is now a liveDocs bitvector and a // reference to it origReader.deleteDocument(1); assertDelDocsRefCountEquals(1, origSegmentReader); @@ -363,7 +363,7 @@ public class TestIndexReaderClone extends LuceneTestCase { IndexReader clonedReader = (IndexReader) origReader.clone(); SegmentReader clonedSegmentReader = getOnlySegmentReader(clonedReader); assertDelDocsRefCountEquals(2, origSegmentReader); - // deleting a document creates a new deletedDocs bitvector, the refs goes to + // deleting a document creates a new liveDocs bitvector, the refs goes to // 1 clonedReader.deleteDocument(2); assertDelDocsRefCountEquals(1, origSegmentReader); @@ -371,13 +371,13 @@ public class TestIndexReaderClone extends LuceneTestCase { // make sure the deletedocs objects are different (copy // on write) - assertTrue(origSegmentReader.deletedDocs != clonedSegmentReader.deletedDocs); + assertTrue(origSegmentReader.liveDocs != clonedSegmentReader.liveDocs); assertDocDeleted(origSegmentReader, clonedSegmentReader, 1); - final Bits delDocs = origSegmentReader.getDeletedDocs(); - assertTrue(delDocs == null || !delDocs.get(2)); // doc 2 should not be deleted + final Bits liveDocs = origSegmentReader.getLiveDocs(); + assertTrue(liveDocs == null || liveDocs.get(2)); // doc 2 should not be deleted // in original segmentreader - assertTrue(clonedSegmentReader.getDeletedDocs().get(2)); // doc 2 should be deleted in + assertFalse(clonedSegmentReader.getLiveDocs().get(2)); // doc 2 should be deleted in // cloned segmentreader // deleting a doc from the original segmentreader should throw an exception @@ -419,7 +419,7 @@ public class TestIndexReaderClone extends LuceneTestCase { clonedReader.close(); IndexReader r = IndexReader.open(dir1, false); - assertTrue(MultiFields.getDeletedDocs(r).get(1)); + assertFalse(MultiFields.getLiveDocs(r).get(1)); r.close(); dir1.close(); } @@ -448,11 +448,11 @@ public class TestIndexReaderClone extends LuceneTestCase { private void assertDocDeleted(SegmentReader reader, SegmentReader reader2, int doc) { - assertEquals(reader.getDeletedDocs().get(doc), reader2.getDeletedDocs().get(doc)); + assertEquals(reader.getLiveDocs().get(doc), reader2.getLiveDocs().get(doc)); } private void assertDelDocsRefCountEquals(int refCount, SegmentReader reader) { - assertEquals(refCount, reader.deletedDocsRef.get()); + assertEquals(refCount, reader.liveDocsRef.get()); } public void testCloneSubreaders() throws Exception { diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java b/lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java index a322fe01f50..9af326f3feb 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java @@ -285,17 +285,17 @@ public class TestIndexReaderDelete extends LuceneTestCase { IndexReader r = new SlowMultiReaderWrapper(w.getReader()); w.close(); - assertNull(r.getDeletedDocs()); + assertNull(r.getLiveDocs()); r.close(); r = new SlowMultiReaderWrapper(IndexReader.open(dir, false)); - assertNull(r.getDeletedDocs()); + assertNull(r.getLiveDocs()); assertEquals(1, r.deleteDocuments(new Term("f", "doctor"))); - assertNotNull(r.getDeletedDocs()); - assertTrue(r.getDeletedDocs().get(0)); + assertNotNull(r.getLiveDocs()); + assertFalse(r.getLiveDocs().get(0)); assertEquals(1, r.deleteDocuments(new Term("f", "who"))); - assertTrue(r.getDeletedDocs().get(1)); + assertFalse(r.getLiveDocs().get(1)); r.close(); dir.close(); } diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java b/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java index a421550de05..f9277eab9a9 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java @@ -1115,16 +1115,16 @@ public class TestIndexReaderReopen extends LuceneTestCase { SegmentReader sr2 = (SegmentReader) r2.getSequentialSubReaders()[0]; // and reopened IRs // At this point they share the same BitVector - assertTrue(sr1.deletedDocs==sr2.deletedDocs); + assertTrue(sr1.liveDocs==sr2.liveDocs); r2.deleteDocument(0); // r1 should not see the delete - final Bits r1DelDocs = MultiFields.getDeletedDocs(r1); - assertFalse(r1DelDocs != null && r1DelDocs.get(0)); + final Bits r1LiveDocs = MultiFields.getLiveDocs(r1); + assertFalse(r1LiveDocs != null && !r1LiveDocs.get(0)); // Now r2 should have made a private copy of deleted docs: - assertTrue(sr1.deletedDocs!=sr2.deletedDocs); + assertTrue(sr1.liveDocs!=sr2.liveDocs); r1.close(); r2.close(); @@ -1150,12 +1150,12 @@ public class TestIndexReaderReopen extends LuceneTestCase { SegmentReader sr2 = (SegmentReader) rs2[0]; // At this point they share the same BitVector - assertTrue(sr1.deletedDocs==sr2.deletedDocs); - final BitVector delDocs = sr1.deletedDocs; + assertTrue(sr1.liveDocs==sr2.liveDocs); + final BitVector liveDocs = sr1.liveDocs; r1.close(); r2.deleteDocument(0); - assertTrue(delDocs==sr2.deletedDocs); + assertTrue(liveDocs==sr2.liveDocs); r2.close(); dir.close(); } diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java index a52ab279d7a..87f307f9609 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -536,7 +536,7 @@ public class TestIndexWriter extends LuceneTestCase { Term t = new Term("field", "a"); assertEquals(1, reader.docFreq(t)); DocsEnum td = MultiFields.getTermDocsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getLiveDocs(reader), "field", new BytesRef("a")); td.nextDoc(); @@ -947,7 +947,7 @@ public class TestIndexWriter extends LuceneTestCase { assertEquals(1, hits.length); DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(s.getIndexReader(), - MultiFields.getDeletedDocs(s.getIndexReader()), + MultiFields.getLiveDocs(s.getIndexReader()), "field", new BytesRef("a")); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java index bf1c61ee817..a112e2124bb 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java @@ -484,7 +484,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { // Make sure the doc that hit the exception was marked // as deleted: DocsEnum tdocs = MultiFields.getTermDocsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getLiveDocs(reader), t.field(), new BytesRef(t.text())); @@ -624,10 +624,10 @@ public class TestIndexWriterExceptions extends LuceneTestCase { assertEquals(expected, reader.docFreq(new Term("contents", "here"))); assertEquals(expected, reader.maxDoc()); int numDel = 0; - final Bits delDocs = MultiFields.getDeletedDocs(reader); - assertNotNull(delDocs); + final Bits liveDocs = MultiFields.getLiveDocs(reader); + assertNotNull(liveDocs); for(int j=0;j=maxDoc) { return doc=NO_MORE_DOCS; } - if (hasDeletions && delDocs.get(doc)) continue; + if (hasDeletions && !liveDocs.get(doc)) continue; return doc; } } diff --git a/modules/queries/src/java/org/apache/lucene/queries/function/ValueSource.java b/modules/queries/src/java/org/apache/lucene/queries/function/ValueSource.java index e3656af27c7..ee108aded06 100644 --- a/modules/queries/src/java/org/apache/lucene/queries/function/ValueSource.java +++ b/modules/queries/src/java/org/apache/lucene/queries/function/ValueSource.java @@ -191,5 +191,3 @@ public abstract class ValueSource implements Serializable { } } } - - diff --git a/modules/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java b/modules/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java index b6b479d856f..a856d9fc7b9 100644 --- a/modules/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java +++ b/modules/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java @@ -30,7 +30,7 @@ public class ValueSourceScorer extends Scorer { protected final int maxDoc; protected final DocValues values; protected boolean checkDeletes; - private final Bits delDocs; + private final Bits liveDocs; protected ValueSourceScorer(IndexReader reader, DocValues values) { super(null); @@ -38,7 +38,7 @@ public class ValueSourceScorer extends Scorer { this.maxDoc = reader.maxDoc(); this.values = values; setCheckDeletes(true); - this.delDocs = MultiFields.getDeletedDocs(reader); + this.liveDocs = MultiFields.getLiveDocs(reader); } public IndexReader getReader() { @@ -50,7 +50,7 @@ public class ValueSourceScorer extends Scorer { } public boolean matches(int doc) { - return (!checkDeletes || !delDocs.get(doc)) && matchesValue(doc); + return (!checkDeletes || liveDocs.get(doc)) && matchesValue(doc); } public boolean matchesValue(int doc) { diff --git a/solr/src/java/org/apache/solr/request/SimpleFacets.java b/solr/src/java/org/apache/solr/request/SimpleFacets.java index ff0349f1728..7eb70374342 100644 --- a/solr/src/java/org/apache/solr/request/SimpleFacets.java +++ b/solr/src/java/org/apache/solr/request/SimpleFacets.java @@ -673,7 +673,7 @@ public class SimpleFacets { if (deState==null) { deState = new SolrIndexSearcher.DocsEnumState(); deState.fieldName = field; - deState.deletedDocs = MultiFields.getDeletedDocs(r); + deState.liveDocs = MultiFields.getLiveDocs(r); deState.termsEnum = termsEnum; deState.docsEnum = docsEnum; } diff --git a/solr/src/java/org/apache/solr/schema/LatLonType.java b/solr/src/java/org/apache/solr/schema/LatLonType.java index 1782fa945b4..dbcd9588c8a 100644 --- a/solr/src/java/org/apache/solr/schema/LatLonType.java +++ b/solr/src/java/org/apache/solr/schema/LatLonType.java @@ -389,7 +389,7 @@ class SpatialDistanceQuery extends Query { int doc=-1; final DocValues latVals; final DocValues lonVals; - final Bits delDocs; + final Bits liveDocs; final double lonMin, lonMax, lon2Min, lon2Max, latMin, latMax; @@ -411,7 +411,7 @@ class SpatialDistanceQuery extends Query { this.qWeight = w.getValue(); this.reader = readerContext.reader; this.maxDoc = reader.maxDoc(); - this.delDocs = reader.getDeletedDocs(); + this.liveDocs = reader.getLiveDocs(); latVals = latSource.getValues(weight.latContext, readerContext); lonVals = lonSource.getValues(weight.lonContext, readerContext); @@ -489,7 +489,7 @@ class SpatialDistanceQuery extends Query { if (doc>=maxDoc) { return doc=NO_MORE_DOCS; } - if (delDocs != null && delDocs.get(doc)) continue; + if (liveDocs != null && !liveDocs.get(doc)) continue; if (!match()) continue; return doc; } diff --git a/solr/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/src/java/org/apache/solr/search/JoinQParserPlugin.java index a38b1a8b673..4f188ea15a1 100644 --- a/solr/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -285,19 +285,19 @@ class JoinQuery extends Query { } } - Bits fromDeletedDocs = MultiFields.getDeletedDocs(fromSearcher.getIndexReader()); - Bits toDeletedDocs = fromSearcher == toSearcher ? fromDeletedDocs : MultiFields.getDeletedDocs(toSearcher.getIndexReader()); + Bits fromLiveDocs = MultiFields.getLiveDocs(fromSearcher.getIndexReader()); + Bits toLiveDocs = fromSearcher == toSearcher ? fromLiveDocs : MultiFields.getLiveDocs(toSearcher.getIndexReader()); fromDeState = new SolrIndexSearcher.DocsEnumState(); fromDeState.fieldName = fromField; - fromDeState.deletedDocs = fromDeletedDocs; + fromDeState.liveDocs = fromLiveDocs; fromDeState.termsEnum = termsEnum; fromDeState.docsEnum = null; fromDeState.minSetSizeCached = minDocFreqFrom; toDeState = new SolrIndexSearcher.DocsEnumState(); toDeState.fieldName = toField; - toDeState.deletedDocs = toDeletedDocs; + toDeState.liveDocs = toLiveDocs; toDeState.termsEnum = toTermsEnum; toDeState.docsEnum = null; toDeState.minSetSizeCached = minDocFreqTo; @@ -314,7 +314,7 @@ class JoinQuery extends Query { if (freq < minDocFreqFrom) { fromTermDirectCount++; - // OK to skip deletedDocs, since we check for intersection with docs matching query + // OK to skip liveDocs, since we check for intersection with docs matching query fromDeState.docsEnum = fromDeState.termsEnum.docs(null, fromDeState.docsEnum); DocsEnum docsEnum = fromDeState.docsEnum; @@ -393,8 +393,8 @@ class JoinQuery extends Query { } else { toTermDirectCount++; - // need to use deletedDocs here so we don't map to any deleted ones - toDeState.docsEnum = toDeState.termsEnum.docs(toDeState.deletedDocs, toDeState.docsEnum); + // need to use liveDocs here so we don't map to any deleted ones + toDeState.docsEnum = toDeState.termsEnum.docs(toDeState.liveDocs, toDeState.docsEnum); DocsEnum docsEnum = toDeState.docsEnum; if (docsEnum instanceof MultiDocsEnum) { diff --git a/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java index 6fa9d1059a2..513fcab7b09 100644 --- a/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -493,7 +493,7 @@ public class SolrIndexSearcher extends IndexSearcher implements SolrInfoMBean { Terms terms = fields.terms(t.field()); if (terms == null) return -1; BytesRef termBytes = t.bytes(); - DocsEnum docs = terms.docs(MultiFields.getDeletedDocs(reader), termBytes, null); + DocsEnum docs = terms.docs(MultiFields.getLiveDocs(reader), termBytes, null); if (docs == null) return -1; int id = docs.nextDoc(); return id == DocIdSetIterator.NO_MORE_DOCS ? -1 : id; @@ -646,12 +646,12 @@ public class SolrIndexSearcher extends IndexSearcher implements SolrInfoMBean { } collector.setNextReader(leaf); - Bits skipDocs = reader.getDeletedDocs(); + Bits liveDocs = reader.getLiveDocs(); int max = reader.maxDoc(); if (idIter == null) { for (int docid = 0; docid