Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-2803: ignore deleted docs when populating field cache 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1042888 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2010-12-07 01:46:20 +00:00
parent 7aa9193885
commit 5b2e0f786b
9 changed files with 25 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lucene/CHANGES.txt
View File

@ -339,6 +339,12 @@ Bug fixes
* LUCENE-2633: PackedInts Packed32 and Packed64 did not support internal
structures larger than 256MB (Toke Eskildsen via Mike McCandless)
* LUCENE-2803: The FieldCache can miss values if an entry for a reader
with more document deletions is requested before a reader with fewer
deletions, provided they share some segments. (yonik)
======================= Lucene 3.x (not yet released) =======================
Changes in backwards compatibility policy

5
lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java vendored
View File

@ -110,7 +110,6 @@ public class ByteValuesCreator extends CachedArrayCreator<ByteValues>
vals.values = new byte[maxDoc];
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null;
try {
@ -120,7 +119,7 @@ public class ByteValuesCreator extends CachedArrayCreator<ByteValues>
break;
}
final byte termval = parser.parseByte(term);
docs = termsEnum.docs(delDocs, docs);
docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -137,7 +136,7 @@ public class ByteValuesCreator extends CachedArrayCreator<ByteValues>
} catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) {
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc );
vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
}
}
if( vals.valid == null && vals.numDocs < 1 ) {

22
lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java vendored
View File

@ -100,22 +100,13 @@ public abstract class CachedArrayCreator<T extends CachedArray> extends EntryCre
/**
* Utility function to help check what bits are valid
*/
protected Bits checkMatchAllBits( Bits deleted, OpenBitSet valid, int numDocs, int maxDocs )
protected Bits checkMatchAllBits( OpenBitSet valid, int numDocs, int maxDocs )
{
if( numDocs != maxDocs ) {
if( hasOption( OPTION_CACHE_BITS ) ) {
if( deleted == null ) {
for( int i=0; i<maxDocs; i++ ) {
if( !valid.get(i) ) {
return valid;
}
}
}
else {
for( int i=0; i<maxDocs; i++ ) {
if( !deleted.get(i) && !valid.get(i) ) {
return valid;
}
for( int i=0; i<maxDocs; i++ ) {
if( !valid.get(i) ) {
return valid;
}
}
}
@ -132,7 +123,6 @@ public abstract class CachedArrayCreator<T extends CachedArray> extends EntryCre
Terms terms = MultiFields.getTerms(reader, field);
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = new OpenBitSet( reader.maxDoc() );
DocsEnum docs = null;
while(true) {
@ -140,7 +130,7 @@ public abstract class CachedArrayCreator<T extends CachedArray> extends EntryCre
if (term == null) {
break;
}
docs = termsEnum.docs(delDocs, docs);
docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -152,7 +142,7 @@ public abstract class CachedArrayCreator<T extends CachedArray> extends EntryCre
vals.numTerms++;
}
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, reader.maxDoc() );
vals.valid = checkMatchAllBits( validBits, vals.numDocs, reader.maxDoc() );
}
if( vals.numDocs < 1 ) {
vals.valid = new Bits.MatchNoBits( reader.maxDoc() );

3
lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java vendored
View File

@ -130,7 +130,6 @@ public class DocTermsIndexCreator extends EntryCreatorWithOptions<DocTermsIndex>
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
DocsEnum docs = null;
while(true) {
@ -149,7 +148,7 @@ public class DocTermsIndexCreator extends EntryCreatorWithOptions<DocTermsIndex>
termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1+termOrd, 1));
}
termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term));
docs = termsEnum.docs(delDocs, docs);
docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {

5
lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java vendored
View File

@ -120,7 +120,6 @@ public class DoubleValuesCreator extends CachedArrayCreator<DoubleValues>
vals.values = null;
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null;
try {
@ -130,7 +129,7 @@ public class DoubleValuesCreator extends CachedArrayCreator<DoubleValues>
break;
}
final double termval = parser.parseDouble(term);
docs = termsEnum.docs(delDocs, docs);
docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -150,7 +149,7 @@ public class DoubleValuesCreator extends CachedArrayCreator<DoubleValues>
} catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) {
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc );
vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
}
}

5
lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java vendored
View File

@ -121,7 +121,6 @@ public class FloatValuesCreator extends CachedArrayCreator<FloatValues>
vals.values = null;
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null;
try {
@ -131,7 +130,7 @@ public class FloatValuesCreator extends CachedArrayCreator<FloatValues>
break;
}
final float termval = parser.parseFloat(term);
docs = termsEnum.docs(delDocs, docs);
docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -151,7 +150,7 @@ public class FloatValuesCreator extends CachedArrayCreator<FloatValues>
} catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) {
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc );
vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
}
}

5
lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java vendored
View File

@ -121,7 +121,6 @@ public class IntValuesCreator extends CachedArrayCreator<IntValues>
vals.values = null;
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null;
try {
@ -131,7 +130,7 @@ public class IntValuesCreator extends CachedArrayCreator<IntValues>
break;
}
final int termval = parser.parseInt(term);
docs = termsEnum.docs(delDocs, docs);
docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -151,7 +150,7 @@ public class IntValuesCreator extends CachedArrayCreator<IntValues>
} catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) {
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc );
vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
}
}

5
lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java vendored
View File

@ -121,7 +121,6 @@ public class LongValuesCreator extends CachedArrayCreator<LongValues>
vals.values = null;
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null;
try {
@ -131,7 +130,7 @@ public class LongValuesCreator extends CachedArrayCreator<LongValues>
break;
}
final long termval = parser.parseLong(term);
docs = termsEnum.docs(delDocs, docs);
docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -151,7 +150,7 @@ public class LongValuesCreator extends CachedArrayCreator<LongValues>
} catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) {
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc );
vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
}
}

5
lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java vendored
View File

@ -111,7 +111,6 @@ public class ShortValuesCreator extends CachedArrayCreator<ShortValues>
vals.values = new short[maxDoc];
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null;
try {
@ -121,7 +120,7 @@ public class ShortValuesCreator extends CachedArrayCreator<ShortValues>
break;
}
final Short termval = parser.parseShort(term);
docs = termsEnum.docs(delDocs, docs);
docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@ -138,7 +137,7 @@ public class ShortValuesCreator extends CachedArrayCreator<ShortValues>
} catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) {
vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc );
vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
}
}
if( vals.valid == null && vals.numDocs < 1 ) {