diff --git a/CHANGES.txt b/CHANGES.txt index e90686a020c..04f62fc759f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,6 +12,10 @@ Changes in backwards compatibility policy FuzzyQuery. The class was never intended to be public. (Uwe Schindler, Mike McCandless) +* LUCENE-2135: Added FieldCache.purge(IndexReader) method to the + interface. Anyone implementing FieldCache externally will need to + fix their code to implement this, on upgrading. (Mike McCandless) + Changes in runtime behavior API Changes @@ -92,6 +96,10 @@ Optimizations MTQ. This also fixes a slowdown / memory issue added by LUCENE-504. (Uwe Schindler, Robert Muir, Mike McCandless) +* LUCENE-2135: On IndexReader.close, forcefully evict any entries from + the FieldCache rather than waiting for the WeakHashMap to release + the reference (Mike McCandless) + Build * LUCENE-2124: Moved the JDK-based collation support from contrib/collation diff --git a/src/java/org/apache/lucene/index/DirectoryReader.java b/src/java/org/apache/lucene/index/DirectoryReader.java index 69ca67dcd41..605dab24340 100644 --- a/src/java/org/apache/lucene/index/DirectoryReader.java +++ b/src/java/org/apache/lucene/index/DirectoryReader.java @@ -36,6 +36,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close /** * An IndexReader which reads indexes with multiple segments. @@ -854,6 +855,12 @@ class DirectoryReader extends IndexReader implements Cloneable { if (ioe == null) ioe = e; } } + + // NOTE: only needed in case someone had asked for + // FieldCache for top-level reader (which is generally + // not a good idea): + FieldCache.DEFAULT.purge(this); + // throw the first exception if (ioe != null) throw ioe; } diff --git a/src/java/org/apache/lucene/index/FilterIndexReader.java b/src/java/org/apache/lucene/index/FilterIndexReader.java index 7cd1089b1cc..561815be691 100644 --- a/src/java/org/apache/lucene/index/FilterIndexReader.java +++ b/src/java/org/apache/lucene/index/FilterIndexReader.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.store.Directory; +import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close import java.io.IOException; import java.util.Collection; @@ -241,7 +242,14 @@ public class FilterIndexReader extends IndexReader { protected void doCommit(Map commitUserData) throws IOException { in.commit(commitUserData); } @Override - protected void doClose() throws IOException { in.close(); } + protected void doClose() throws IOException { + in.close(); + + // NOTE: only needed in case someone had asked for + // FieldCache for top-level reader (which is generally + // not a good idea): + FieldCache.DEFAULT.purge(this); + } @Override @@ -272,4 +280,12 @@ public class FilterIndexReader extends IndexReader { public IndexReader[] getSequentialSubReaders() { return in.getSequentialSubReaders(); } + + /** If the subclass of FilteredIndexReader modifies the + * contents of the FieldCache, you must override this + * method to provide a different key */ + @Override + public Object getFieldCacheKey() { + return in.getFieldCacheKey(); + } } diff --git a/src/java/org/apache/lucene/index/MultiReader.java b/src/java/org/apache/lucene/index/MultiReader.java index f13678c9ccf..f6487bf5d3f 100644 --- a/src/java/org/apache/lucene/index/MultiReader.java +++ b/src/java/org/apache/lucene/index/MultiReader.java @@ -29,6 +29,7 @@ import org.apache.lucene.index.DirectoryReader.MultiTermDocs; import org.apache.lucene.index.DirectoryReader.MultiTermEnum; import org.apache.lucene.index.DirectoryReader.MultiTermPositions; import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close /** An IndexReader which reads multiple indexes, appending * their content. */ @@ -415,6 +416,11 @@ public class MultiReader extends IndexReader implements Cloneable { subReaders[i].close(); } } + + // NOTE: only needed in case someone had asked for + // FieldCache for top-level reader (which is generally + // not a good idea): + FieldCache.DEFAULT.purge(this); } @Override diff --git a/src/java/org/apache/lucene/index/ParallelReader.java b/src/java/org/apache/lucene/index/ParallelReader.java index 8727453b97e..b9e00e41eba 100644 --- a/src/java/org/apache/lucene/index/ParallelReader.java +++ b/src/java/org/apache/lucene/index/ParallelReader.java @@ -21,6 +21,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.document.FieldSelectorResult; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close import java.io.IOException; import java.util.*; @@ -470,6 +471,8 @@ public class ParallelReader extends IndexReader { readers.get(i).close(); } } + + FieldCache.DEFAULT.purge(this); } @Override diff --git a/src/java/org/apache/lucene/index/SegmentReader.java b/src/java/org/apache/lucene/index/SegmentReader.java index 17ac9c1ebf7..75da673dc8e 100644 --- a/src/java/org/apache/lucene/index/SegmentReader.java +++ b/src/java/org/apache/lucene/index/SegmentReader.java @@ -38,6 +38,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BitVector; import org.apache.lucene.util.CloseableThreadLocal; +import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close /** @version $Id */ /** @@ -93,16 +94,19 @@ public class SegmentReader extends IndexReader implements Cloneable { final int readBufferSize; final int termsIndexDivisor; + private final SegmentReader origInstance; + TermInfosReader tis; FieldsReader fieldsReaderOrig; TermVectorsReader termVectorsReaderOrig; CompoundFileReader cfsReader; CompoundFileReader storeCFSReader; - CoreReaders(Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException { + CoreReaders(SegmentReader origInstance, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException { segment = si.name; this.readBufferSize = readBufferSize; this.dir = dir; + this.origInstance = origInstance; boolean success = false; @@ -233,6 +237,11 @@ public class SegmentReader extends IndexReader implements Cloneable { if (storeCFSReader != null) { storeCFSReader.close(); } + + // Force FieldCache to evict our entries at this point + if (freqStream != null) { + FieldCache.DEFAULT.purge(origInstance); + } } } @@ -549,7 +558,7 @@ public class SegmentReader extends IndexReader implements Cloneable { boolean success = false; try { - instance.core = new CoreReaders(dir, si, readBufferSize, termInfosIndexDivisor); + instance.core = new CoreReaders(instance, dir, si, readBufferSize, termInfosIndexDivisor); if (doOpenStores) { instance.core.openDocStores(si); } diff --git a/src/java/org/apache/lucene/search/FieldCache.java b/src/java/org/apache/lucene/search/FieldCache.java index 630340b501a..4b2ece2b43c 100644 --- a/src/java/org/apache/lucene/search/FieldCache.java +++ b/src/java/org/apache/lucene/search/FieldCache.java @@ -593,6 +593,15 @@ public interface FieldCache { */ public abstract void purgeAllCaches(); + /** + * Expert: drops all cache entries associated with this + * reader. NOTE: this reader must precisely match the + * reader that the cache entry is keyed on. If you pass a + * top-level reader, it usually will have no effect as + * Lucene now caches at the segment reader level. + */ + public abstract void purge(IndexReader r); + /** * If non-null, FieldCacheImpl will warn whenever * entries are created that are not sane according to diff --git a/src/java/org/apache/lucene/search/FieldCacheImpl.java b/src/java/org/apache/lucene/search/FieldCacheImpl.java index 56b92d776bf..e2e3c6a6346 100644 --- a/src/java/org/apache/lucene/search/FieldCacheImpl.java +++ b/src/java/org/apache/lucene/search/FieldCacheImpl.java @@ -61,6 +61,12 @@ class FieldCacheImpl implements FieldCache { public void purgeAllCaches() { init(); } + + public void purge(IndexReader r) { + for(Cache c : caches.values()) { + c.purge(r); + } + } public CacheEntry[] getCacheEntries() { List result = new ArrayList(17); @@ -144,6 +150,14 @@ class FieldCacheImpl implements FieldCache { protected abstract Object createValue(IndexReader reader, Entry key) throws IOException; + /** Remove this reader from the cache, if present. */ + public void purge(IndexReader r) { + Object readerKey = r.getFieldCacheKey(); + synchronized(readerCache) { + readerCache.remove(readerKey); + } + } + public Object get(IndexReader reader, Entry key) throws IOException { Map innerCache; Object value;