diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 38ae4a50bc2..0a378f30793 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -339,6 +339,12 @@ Bug fixes * LUCENE-2633: PackedInts Packed32 and Packed64 did not support internal structures larger than 256MB (Toke Eskildsen via Mike McCandless) +* LUCENE-2803: The FieldCache can miss values if an entry for a reader + with more document deletions is requested before a reader with fewer + deletions, provided they share some segments. (yonik) + + + ======================= Lucene 3.x (not yet released) ======================= Changes in backwards compatibility policy diff --git a/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java b/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java index 04b092ab7a5..d28494b5db8 100644 --- a/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java +++ b/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java @@ -110,7 +110,6 @@ public class ByteValuesCreator extends CachedArrayCreator vals.values = new byte[maxDoc]; if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - final Bits delDocs = MultiFields.getDeletedDocs(reader); OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; DocsEnum docs = null; try { @@ -120,7 +119,7 @@ public class ByteValuesCreator extends CachedArrayCreator break; } final byte termval = parser.parseByte(term); - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(null, docs); while (true) { final int docID = docs.nextDoc(); if (docID == DocIdSetIterator.NO_MORE_DOCS) { @@ -137,7 +136,7 @@ public class ByteValuesCreator extends CachedArrayCreator } catch (FieldCache.StopFillCacheException stop) {} if( vals.valid == null ) { - vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc ); + vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc ); } } if( vals.valid == null && vals.numDocs < 1 ) { diff --git a/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java b/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java index b65b61478ae..3129c755495 100644 --- a/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java +++ b/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java @@ -100,22 +100,13 @@ public abstract class CachedArrayCreator extends EntryCre /** * Utility function to help check what bits are valid */ - protected Bits checkMatchAllBits( Bits deleted, OpenBitSet valid, int numDocs, int maxDocs ) + protected Bits checkMatchAllBits( OpenBitSet valid, int numDocs, int maxDocs ) { if( numDocs != maxDocs ) { if( hasOption( OPTION_CACHE_BITS ) ) { - if( deleted == null ) { - for( int i=0; i extends EntryCre Terms terms = MultiFields.getTerms(reader, field); if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - final Bits delDocs = MultiFields.getDeletedDocs(reader); OpenBitSet validBits = new OpenBitSet( reader.maxDoc() ); DocsEnum docs = null; while(true) { @@ -140,7 +130,7 @@ public abstract class CachedArrayCreator extends EntryCre if (term == null) { break; } - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(null, docs); while (true) { final int docID = docs.nextDoc(); if (docID == DocIdSetIterator.NO_MORE_DOCS) { @@ -152,7 +142,7 @@ public abstract class CachedArrayCreator extends EntryCre vals.numTerms++; } - vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, reader.maxDoc() ); + vals.valid = checkMatchAllBits( validBits, vals.numDocs, reader.maxDoc() ); } if( vals.numDocs < 1 ) { vals.valid = new Bits.MatchNoBits( reader.maxDoc() ); diff --git a/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java b/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java index b204111353e..2f0bb06e79d 100644 --- a/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java +++ b/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java @@ -130,7 +130,6 @@ public class DocTermsIndexCreator extends EntryCreatorWithOptions if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - final Bits delDocs = MultiFields.getDeletedDocs(reader); DocsEnum docs = null; while(true) { @@ -149,7 +148,7 @@ public class DocTermsIndexCreator extends EntryCreatorWithOptions termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1+termOrd, 1)); } termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term)); - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(null, docs); while (true) { final int docID = docs.nextDoc(); if (docID == DocIdSetIterator.NO_MORE_DOCS) { diff --git a/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java b/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java index de5171a00b4..a72a322e0dc 100644 --- a/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java +++ b/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java @@ -120,7 +120,6 @@ public class DoubleValuesCreator extends CachedArrayCreator vals.values = null; if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - final Bits delDocs = MultiFields.getDeletedDocs(reader); OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; DocsEnum docs = null; try { @@ -130,7 +129,7 @@ public class DoubleValuesCreator extends CachedArrayCreator break; } final double termval = parser.parseDouble(term); - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(null, docs); while (true) { final int docID = docs.nextDoc(); if (docID == DocIdSetIterator.NO_MORE_DOCS) { @@ -150,7 +149,7 @@ public class DoubleValuesCreator extends CachedArrayCreator } catch (FieldCache.StopFillCacheException stop) {} if( vals.valid == null ) { - vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc ); + vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc ); } } diff --git a/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java b/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java index b111dfa5492..42191168361 100644 --- a/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java +++ b/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java @@ -121,7 +121,6 @@ public class FloatValuesCreator extends CachedArrayCreator vals.values = null; if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - final Bits delDocs = MultiFields.getDeletedDocs(reader); OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; DocsEnum docs = null; try { @@ -131,7 +130,7 @@ public class FloatValuesCreator extends CachedArrayCreator break; } final float termval = parser.parseFloat(term); - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(null, docs); while (true) { final int docID = docs.nextDoc(); if (docID == DocIdSetIterator.NO_MORE_DOCS) { @@ -151,7 +150,7 @@ public class FloatValuesCreator extends CachedArrayCreator } catch (FieldCache.StopFillCacheException stop) {} if( vals.valid == null ) { - vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc ); + vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc ); } } diff --git a/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java b/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java index a739dca4048..287fcb43113 100644 --- a/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java +++ b/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java @@ -121,7 +121,6 @@ public class IntValuesCreator extends CachedArrayCreator vals.values = null; if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - final Bits delDocs = MultiFields.getDeletedDocs(reader); OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; DocsEnum docs = null; try { @@ -131,7 +130,7 @@ public class IntValuesCreator extends CachedArrayCreator break; } final int termval = parser.parseInt(term); - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(null, docs); while (true) { final int docID = docs.nextDoc(); if (docID == DocIdSetIterator.NO_MORE_DOCS) { @@ -151,7 +150,7 @@ public class IntValuesCreator extends CachedArrayCreator } catch (FieldCache.StopFillCacheException stop) {} if( vals.valid == null ) { - vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc ); + vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc ); } } diff --git a/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java b/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java index 60174dab46c..f28eee7ce0d 100644 --- a/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java +++ b/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java @@ -121,7 +121,6 @@ public class LongValuesCreator extends CachedArrayCreator vals.values = null; if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - final Bits delDocs = MultiFields.getDeletedDocs(reader); OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; DocsEnum docs = null; try { @@ -131,7 +130,7 @@ public class LongValuesCreator extends CachedArrayCreator break; } final long termval = parser.parseLong(term); - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(null, docs); while (true) { final int docID = docs.nextDoc(); if (docID == DocIdSetIterator.NO_MORE_DOCS) { @@ -151,7 +150,7 @@ public class LongValuesCreator extends CachedArrayCreator } catch (FieldCache.StopFillCacheException stop) {} if( vals.valid == null ) { - vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc ); + vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc ); } } diff --git a/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java b/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java index af723846aae..603fcbbb1f8 100644 --- a/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java +++ b/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java @@ -111,7 +111,6 @@ public class ShortValuesCreator extends CachedArrayCreator vals.values = new short[maxDoc]; if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - final Bits delDocs = MultiFields.getDeletedDocs(reader); OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; DocsEnum docs = null; try { @@ -121,7 +120,7 @@ public class ShortValuesCreator extends CachedArrayCreator break; } final Short termval = parser.parseShort(term); - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(null, docs); while (true) { final int docID = docs.nextDoc(); if (docID == DocIdSetIterator.NO_MORE_DOCS) { @@ -138,7 +137,7 @@ public class ShortValuesCreator extends CachedArrayCreator } catch (FieldCache.StopFillCacheException stop) {} if( vals.valid == null ) { - vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc ); + vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc ); } } if( vals.valid == null && vals.numDocs < 1 ) {