LUCENE-2803: ignore deleted docs when populating field cache

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1042888 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2010-12-07 01:46:20 +00:00
parent 7aa9193885
commit 5b2e0f786b
9 changed files with 25 additions and 36 deletions

View File

@ -339,6 +339,12 @@ Bug fixes
* LUCENE-2633: PackedInts Packed32 and Packed64 did not support internal * LUCENE-2633: PackedInts Packed32 and Packed64 did not support internal
structures larger than 256MB (Toke Eskildsen via Mike McCandless) structures larger than 256MB (Toke Eskildsen via Mike McCandless)
* LUCENE-2803: The FieldCache can miss values if an entry for a reader
with more document deletions is requested before a reader with fewer
deletions, provided they share some segments. (yonik)
======================= Lucene 3.x (not yet released) ======================= ======================= Lucene 3.x (not yet released) =======================
Changes in backwards compatibility policy Changes in backwards compatibility policy

View File

@ -110,7 +110,6 @@ public class ByteValuesCreator extends CachedArrayCreator<ByteValues>
vals.values = new byte[maxDoc]; vals.values = new byte[maxDoc];
if (terms != null) { if (terms != null) {
final TermsEnum termsEnum = terms.iterator(); final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null; DocsEnum docs = null;
try { try {
@ -120,7 +119,7 @@ public class ByteValuesCreator extends CachedArrayCreator<ByteValues>
break; break;
} }
final byte termval = parser.parseByte(term); final byte termval = parser.parseByte(term);
docs = termsEnum.docs(delDocs, docs); docs = termsEnum.docs(null, docs);
while (true) { while (true) {
final int docID = docs.nextDoc(); final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) { if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -137,7 +136,7 @@ public class ByteValuesCreator extends CachedArrayCreator<ByteValues>
} catch (FieldCache.StopFillCacheException stop) {} } catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) { if( vals.valid == null ) {
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc ); vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
} }
} }
if( vals.valid == null && vals.numDocs < 1 ) { if( vals.valid == null && vals.numDocs < 1 ) {

View File

@ -100,25 +100,16 @@ public abstract class CachedArrayCreator<T extends CachedArray> extends EntryCre
/** /**
* Utility function to help check what bits are valid * Utility function to help check what bits are valid
*/ */
protected Bits checkMatchAllBits( Bits deleted, OpenBitSet valid, int numDocs, int maxDocs ) protected Bits checkMatchAllBits( OpenBitSet valid, int numDocs, int maxDocs )
{ {
if( numDocs != maxDocs ) { if( numDocs != maxDocs ) {
if( hasOption( OPTION_CACHE_BITS ) ) { if( hasOption( OPTION_CACHE_BITS ) ) {
if( deleted == null ) {
for( int i=0; i<maxDocs; i++ ) { for( int i=0; i<maxDocs; i++ ) {
if( !valid.get(i) ) { if( !valid.get(i) ) {
return valid; return valid;
} }
} }
} }
else {
for( int i=0; i<maxDocs; i++ ) {
if( !deleted.get(i) && !valid.get(i) ) {
return valid;
}
}
}
}
else { else {
return null; return null;
} }
@ -132,7 +123,6 @@ public abstract class CachedArrayCreator<T extends CachedArray> extends EntryCre
Terms terms = MultiFields.getTerms(reader, field); Terms terms = MultiFields.getTerms(reader, field);
if (terms != null) { if (terms != null) {
final TermsEnum termsEnum = terms.iterator(); final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = new OpenBitSet( reader.maxDoc() ); OpenBitSet validBits = new OpenBitSet( reader.maxDoc() );
DocsEnum docs = null; DocsEnum docs = null;
while(true) { while(true) {
@ -140,7 +130,7 @@ public abstract class CachedArrayCreator<T extends CachedArray> extends EntryCre
if (term == null) { if (term == null) {
break; break;
} }
docs = termsEnum.docs(delDocs, docs); docs = termsEnum.docs(null, docs);
while (true) { while (true) {
final int docID = docs.nextDoc(); final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) { if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -152,7 +142,7 @@ public abstract class CachedArrayCreator<T extends CachedArray> extends EntryCre
vals.numTerms++; vals.numTerms++;
} }
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, reader.maxDoc() ); vals.valid = checkMatchAllBits( validBits, vals.numDocs, reader.maxDoc() );
} }
if( vals.numDocs < 1 ) { if( vals.numDocs < 1 ) {
vals.valid = new Bits.MatchNoBits( reader.maxDoc() ); vals.valid = new Bits.MatchNoBits( reader.maxDoc() );

View File

@ -130,7 +130,6 @@ public class DocTermsIndexCreator extends EntryCreatorWithOptions<DocTermsIndex>
if (terms != null) { if (terms != null) {
final TermsEnum termsEnum = terms.iterator(); final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
DocsEnum docs = null; DocsEnum docs = null;
while(true) { while(true) {
@ -149,7 +148,7 @@ public class DocTermsIndexCreator extends EntryCreatorWithOptions<DocTermsIndex>
termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1+termOrd, 1)); termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1+termOrd, 1));
} }
termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term)); termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term));
docs = termsEnum.docs(delDocs, docs); docs = termsEnum.docs(null, docs);
while (true) { while (true) {
final int docID = docs.nextDoc(); final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) { if (docID == DocIdSetIterator.NO_MORE_DOCS) {

View File

@ -120,7 +120,6 @@ public class DoubleValuesCreator extends CachedArrayCreator<DoubleValues>
vals.values = null; vals.values = null;
if (terms != null) { if (terms != null) {
final TermsEnum termsEnum = terms.iterator(); final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null; DocsEnum docs = null;
try { try {
@ -130,7 +129,7 @@ public class DoubleValuesCreator extends CachedArrayCreator<DoubleValues>
break; break;
} }
final double termval = parser.parseDouble(term); final double termval = parser.parseDouble(term);
docs = termsEnum.docs(delDocs, docs); docs = termsEnum.docs(null, docs);
while (true) { while (true) {
final int docID = docs.nextDoc(); final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) { if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -150,7 +149,7 @@ public class DoubleValuesCreator extends CachedArrayCreator<DoubleValues>
} catch (FieldCache.StopFillCacheException stop) {} } catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) { if( vals.valid == null ) {
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc ); vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
} }
} }

View File

@ -121,7 +121,6 @@ public class FloatValuesCreator extends CachedArrayCreator<FloatValues>
vals.values = null; vals.values = null;
if (terms != null) { if (terms != null) {
final TermsEnum termsEnum = terms.iterator(); final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null; DocsEnum docs = null;
try { try {
@ -131,7 +130,7 @@ public class FloatValuesCreator extends CachedArrayCreator<FloatValues>
break; break;
} }
final float termval = parser.parseFloat(term); final float termval = parser.parseFloat(term);
docs = termsEnum.docs(delDocs, docs); docs = termsEnum.docs(null, docs);
while (true) { while (true) {
final int docID = docs.nextDoc(); final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) { if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -151,7 +150,7 @@ public class FloatValuesCreator extends CachedArrayCreator<FloatValues>
} catch (FieldCache.StopFillCacheException stop) {} } catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) { if( vals.valid == null ) {
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc ); vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
} }
} }

View File

@ -121,7 +121,6 @@ public class IntValuesCreator extends CachedArrayCreator<IntValues>
vals.values = null; vals.values = null;
if (terms != null) { if (terms != null) {
final TermsEnum termsEnum = terms.iterator(); final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null; DocsEnum docs = null;
try { try {
@ -131,7 +130,7 @@ public class IntValuesCreator extends CachedArrayCreator<IntValues>
break; break;
} }
final int termval = parser.parseInt(term); final int termval = parser.parseInt(term);
docs = termsEnum.docs(delDocs, docs); docs = termsEnum.docs(null, docs);
while (true) { while (true) {
final int docID = docs.nextDoc(); final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) { if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -151,7 +150,7 @@ public class IntValuesCreator extends CachedArrayCreator<IntValues>
} catch (FieldCache.StopFillCacheException stop) {} } catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) { if( vals.valid == null ) {
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc ); vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
} }
} }

View File

@ -121,7 +121,6 @@ public class LongValuesCreator extends CachedArrayCreator<LongValues>
vals.values = null; vals.values = null;
if (terms != null) { if (terms != null) {
final TermsEnum termsEnum = terms.iterator(); final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null; DocsEnum docs = null;
try { try {
@ -131,7 +130,7 @@ public class LongValuesCreator extends CachedArrayCreator<LongValues>
break; break;
} }
final long termval = parser.parseLong(term); final long termval = parser.parseLong(term);
docs = termsEnum.docs(delDocs, docs); docs = termsEnum.docs(null, docs);
while (true) { while (true) {
final int docID = docs.nextDoc(); final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) { if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -151,7 +150,7 @@ public class LongValuesCreator extends CachedArrayCreator<LongValues>
} catch (FieldCache.StopFillCacheException stop) {} } catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) { if( vals.valid == null ) {
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc ); vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
} }
} }

View File

@ -111,7 +111,6 @@ public class ShortValuesCreator extends CachedArrayCreator<ShortValues>
vals.values = new short[maxDoc]; vals.values = new short[maxDoc];
if (terms != null) { if (terms != null) {
final TermsEnum termsEnum = terms.iterator(); final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null; DocsEnum docs = null;
try { try {
@ -121,7 +120,7 @@ public class ShortValuesCreator extends CachedArrayCreator<ShortValues>
break; break;
} }
final Short termval = parser.parseShort(term); final Short termval = parser.parseShort(term);
docs = termsEnum.docs(delDocs, docs); docs = termsEnum.docs(null, docs);
while (true) { while (true) {
final int docID = docs.nextDoc(); final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) { if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -138,7 +137,7 @@ public class ShortValuesCreator extends CachedArrayCreator<ShortValues>
} catch (FieldCache.StopFillCacheException stop) {} } catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) { if( vals.valid == null ) {
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc ); vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
} }
} }
if( vals.valid == null && vals.numDocs < 1 ) { if( vals.valid == null && vals.numDocs < 1 ) {