mirror of https://github.com/apache/lucene.git
LUCENE-2135: forcefully evict IndexReader from FieldCache when it's closed
git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@889866 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
2c675012fe
commit
53ce286a25
|
@ -12,6 +12,10 @@ Changes in backwards compatibility policy
|
||||||
FuzzyQuery. The class was never intended to be public.
|
FuzzyQuery. The class was never intended to be public.
|
||||||
(Uwe Schindler, Mike McCandless)
|
(Uwe Schindler, Mike McCandless)
|
||||||
|
|
||||||
|
* LUCENE-2135: Added FieldCache.purge(IndexReader) method to the
|
||||||
|
interface. Anyone implementing FieldCache externally will need to
|
||||||
|
fix their code to implement this, on upgrading. (Mike McCandless)
|
||||||
|
|
||||||
Changes in runtime behavior
|
Changes in runtime behavior
|
||||||
|
|
||||||
API Changes
|
API Changes
|
||||||
|
@ -92,6 +96,10 @@ Optimizations
|
||||||
MTQ. This also fixes a slowdown / memory issue added by LUCENE-504.
|
MTQ. This also fixes a slowdown / memory issue added by LUCENE-504.
|
||||||
(Uwe Schindler, Robert Muir, Mike McCandless)
|
(Uwe Schindler, Robert Muir, Mike McCandless)
|
||||||
|
|
||||||
|
* LUCENE-2135: On IndexReader.close, forcefully evict any entries from
|
||||||
|
the FieldCache rather than waiting for the WeakHashMap to release
|
||||||
|
the reference (Mike McCandless)
|
||||||
|
|
||||||
Build
|
Build
|
||||||
|
|
||||||
* LUCENE-2124: Moved the JDK-based collation support from contrib/collation
|
* LUCENE-2124: Moved the JDK-based collation support from contrib/collation
|
||||||
|
|
|
@ -36,6 +36,7 @@ import org.apache.lucene.store.AlreadyClosedException;
|
||||||
import org.apache.lucene.store.Directory;
|
import org.apache.lucene.store.Directory;
|
||||||
import org.apache.lucene.store.Lock;
|
import org.apache.lucene.store.Lock;
|
||||||
import org.apache.lucene.store.LockObtainFailedException;
|
import org.apache.lucene.store.LockObtainFailedException;
|
||||||
|
import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An IndexReader which reads indexes with multiple segments.
|
* An IndexReader which reads indexes with multiple segments.
|
||||||
|
@ -854,6 +855,12 @@ class DirectoryReader extends IndexReader implements Cloneable {
|
||||||
if (ioe == null) ioe = e;
|
if (ioe == null) ioe = e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NOTE: only needed in case someone had asked for
|
||||||
|
// FieldCache for top-level reader (which is generally
|
||||||
|
// not a good idea):
|
||||||
|
FieldCache.DEFAULT.purge(this);
|
||||||
|
|
||||||
// throw the first exception
|
// throw the first exception
|
||||||
if (ioe != null) throw ioe;
|
if (ioe != null) throw ioe;
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,6 +20,7 @@ package org.apache.lucene.index;
|
||||||
import org.apache.lucene.document.Document;
|
import org.apache.lucene.document.Document;
|
||||||
import org.apache.lucene.document.FieldSelector;
|
import org.apache.lucene.document.FieldSelector;
|
||||||
import org.apache.lucene.store.Directory;
|
import org.apache.lucene.store.Directory;
|
||||||
|
import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
|
@ -241,7 +242,14 @@ public class FilterIndexReader extends IndexReader {
|
||||||
protected void doCommit(Map<String,String> commitUserData) throws IOException { in.commit(commitUserData); }
|
protected void doCommit(Map<String,String> commitUserData) throws IOException { in.commit(commitUserData); }
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void doClose() throws IOException { in.close(); }
|
protected void doClose() throws IOException {
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
// NOTE: only needed in case someone had asked for
|
||||||
|
// FieldCache for top-level reader (which is generally
|
||||||
|
// not a good idea):
|
||||||
|
FieldCache.DEFAULT.purge(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -272,4 +280,12 @@ public class FilterIndexReader extends IndexReader {
|
||||||
public IndexReader[] getSequentialSubReaders() {
|
public IndexReader[] getSequentialSubReaders() {
|
||||||
return in.getSequentialSubReaders();
|
return in.getSequentialSubReaders();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** If the subclass of FilteredIndexReader modifies the
|
||||||
|
* contents of the FieldCache, you must override this
|
||||||
|
* method to provide a different key */
|
||||||
|
@Override
|
||||||
|
public Object getFieldCacheKey() {
|
||||||
|
return in.getFieldCacheKey();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,6 +29,7 @@ import org.apache.lucene.index.DirectoryReader.MultiTermDocs;
|
||||||
import org.apache.lucene.index.DirectoryReader.MultiTermEnum;
|
import org.apache.lucene.index.DirectoryReader.MultiTermEnum;
|
||||||
import org.apache.lucene.index.DirectoryReader.MultiTermPositions;
|
import org.apache.lucene.index.DirectoryReader.MultiTermPositions;
|
||||||
import org.apache.lucene.search.Similarity;
|
import org.apache.lucene.search.Similarity;
|
||||||
|
import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
|
||||||
|
|
||||||
/** An IndexReader which reads multiple indexes, appending
|
/** An IndexReader which reads multiple indexes, appending
|
||||||
* their content. */
|
* their content. */
|
||||||
|
@ -415,6 +416,11 @@ public class MultiReader extends IndexReader implements Cloneable {
|
||||||
subReaders[i].close();
|
subReaders[i].close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NOTE: only needed in case someone had asked for
|
||||||
|
// FieldCache for top-level reader (which is generally
|
||||||
|
// not a good idea):
|
||||||
|
FieldCache.DEFAULT.purge(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -21,6 +21,7 @@ import org.apache.lucene.document.Document;
|
||||||
import org.apache.lucene.document.FieldSelector;
|
import org.apache.lucene.document.FieldSelector;
|
||||||
import org.apache.lucene.document.FieldSelectorResult;
|
import org.apache.lucene.document.FieldSelectorResult;
|
||||||
import org.apache.lucene.document.Fieldable;
|
import org.apache.lucene.document.Fieldable;
|
||||||
|
import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
@ -470,6 +471,8 @@ public class ParallelReader extends IndexReader {
|
||||||
readers.get(i).close();
|
readers.get(i).close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
FieldCache.DEFAULT.purge(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -38,6 +38,7 @@ import org.apache.lucene.store.IndexInput;
|
||||||
import org.apache.lucene.store.IndexOutput;
|
import org.apache.lucene.store.IndexOutput;
|
||||||
import org.apache.lucene.util.BitVector;
|
import org.apache.lucene.util.BitVector;
|
||||||
import org.apache.lucene.util.CloseableThreadLocal;
|
import org.apache.lucene.util.CloseableThreadLocal;
|
||||||
|
import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
|
||||||
|
|
||||||
/** @version $Id */
|
/** @version $Id */
|
||||||
/**
|
/**
|
||||||
|
@ -93,16 +94,19 @@ public class SegmentReader extends IndexReader implements Cloneable {
|
||||||
final int readBufferSize;
|
final int readBufferSize;
|
||||||
final int termsIndexDivisor;
|
final int termsIndexDivisor;
|
||||||
|
|
||||||
|
private final SegmentReader origInstance;
|
||||||
|
|
||||||
TermInfosReader tis;
|
TermInfosReader tis;
|
||||||
FieldsReader fieldsReaderOrig;
|
FieldsReader fieldsReaderOrig;
|
||||||
TermVectorsReader termVectorsReaderOrig;
|
TermVectorsReader termVectorsReaderOrig;
|
||||||
CompoundFileReader cfsReader;
|
CompoundFileReader cfsReader;
|
||||||
CompoundFileReader storeCFSReader;
|
CompoundFileReader storeCFSReader;
|
||||||
|
|
||||||
CoreReaders(Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException {
|
CoreReaders(SegmentReader origInstance, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException {
|
||||||
segment = si.name;
|
segment = si.name;
|
||||||
this.readBufferSize = readBufferSize;
|
this.readBufferSize = readBufferSize;
|
||||||
this.dir = dir;
|
this.dir = dir;
|
||||||
|
this.origInstance = origInstance;
|
||||||
|
|
||||||
boolean success = false;
|
boolean success = false;
|
||||||
|
|
||||||
|
@ -233,6 +237,11 @@ public class SegmentReader extends IndexReader implements Cloneable {
|
||||||
if (storeCFSReader != null) {
|
if (storeCFSReader != null) {
|
||||||
storeCFSReader.close();
|
storeCFSReader.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Force FieldCache to evict our entries at this point
|
||||||
|
if (freqStream != null) {
|
||||||
|
FieldCache.DEFAULT.purge(origInstance);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -549,7 +558,7 @@ public class SegmentReader extends IndexReader implements Cloneable {
|
||||||
boolean success = false;
|
boolean success = false;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
instance.core = new CoreReaders(dir, si, readBufferSize, termInfosIndexDivisor);
|
instance.core = new CoreReaders(instance, dir, si, readBufferSize, termInfosIndexDivisor);
|
||||||
if (doOpenStores) {
|
if (doOpenStores) {
|
||||||
instance.core.openDocStores(si);
|
instance.core.openDocStores(si);
|
||||||
}
|
}
|
||||||
|
|
|
@ -593,6 +593,15 @@ public interface FieldCache {
|
||||||
*/
|
*/
|
||||||
public abstract void purgeAllCaches();
|
public abstract void purgeAllCaches();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Expert: drops all cache entries associated with this
|
||||||
|
* reader. NOTE: this reader must precisely match the
|
||||||
|
* reader that the cache entry is keyed on. If you pass a
|
||||||
|
* top-level reader, it usually will have no effect as
|
||||||
|
* Lucene now caches at the segment reader level.
|
||||||
|
*/
|
||||||
|
public abstract void purge(IndexReader r);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* If non-null, FieldCacheImpl will warn whenever
|
* If non-null, FieldCacheImpl will warn whenever
|
||||||
* entries are created that are not sane according to
|
* entries are created that are not sane according to
|
||||||
|
|
|
@ -61,6 +61,12 @@ class FieldCacheImpl implements FieldCache {
|
||||||
public void purgeAllCaches() {
|
public void purgeAllCaches() {
|
||||||
init();
|
init();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void purge(IndexReader r) {
|
||||||
|
for(Cache c : caches.values()) {
|
||||||
|
c.purge(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public CacheEntry[] getCacheEntries() {
|
public CacheEntry[] getCacheEntries() {
|
||||||
List<CacheEntry> result = new ArrayList<CacheEntry>(17);
|
List<CacheEntry> result = new ArrayList<CacheEntry>(17);
|
||||||
|
@ -144,6 +150,14 @@ class FieldCacheImpl implements FieldCache {
|
||||||
protected abstract Object createValue(IndexReader reader, Entry key)
|
protected abstract Object createValue(IndexReader reader, Entry key)
|
||||||
throws IOException;
|
throws IOException;
|
||||||
|
|
||||||
|
/** Remove this reader from the cache, if present. */
|
||||||
|
public void purge(IndexReader r) {
|
||||||
|
Object readerKey = r.getFieldCacheKey();
|
||||||
|
synchronized(readerCache) {
|
||||||
|
readerCache.remove(readerKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public Object get(IndexReader reader, Entry key) throws IOException {
|
public Object get(IndexReader reader, Entry key) throws IOException {
|
||||||
Map<Entry,Object> innerCache;
|
Map<Entry,Object> innerCache;
|
||||||
Object value;
|
Object value;
|
||||||
|
|
Loading…
Reference in New Issue