Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3646: throw exception for fieldcache on non-atomic reader 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1232444 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2012-01-17 15:34:43 +00:00
parent 37e03cbfa5
commit dea4b911da
16 changed files with 77 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/CHANGES.txt
View File

@ -231,6 +231,10 @@ Changes in backwards compatibility policy
FieldInfo.IndexOption: DOCS_AND_POSITIONS_AND_OFFSETS. (Robert
Muir, Mike McCandless)
* LUCENE-3646: FieldCacheImpl now throws UOE on non-atomic IndexReaders. If
you really want a top-level fieldcache, use SlowMultiReaderWrapper.
(Robert Muir)
Changes in Runtime Behavior
* LUCENE-2846: omitNorms now behaves like omitTermFrequencyAndPositions, if you

54
lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
View File

@ -30,7 +30,6 @@ import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.OrdTermState;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.TermState;
@ -163,11 +162,28 @@ class FieldCacheImpl implements FieldCache {
}
};
private void initReader(IndexReader reader) {
if (reader instanceof SegmentReader) {
((SegmentReader) reader).addCoreClosedListener(purgeCore);
} else if (reader.getSequentialSubReaders() != null) {
throw new UnsupportedOperationException("Please use SlowMultiReaderWrapper, if you really need a top level FieldCache");
} else {
// we have a slow reader of some sort, try to register a purge event
// rather than relying on gc:
Object key = reader.getCoreCacheKey();
if (key instanceof IndexReader) {
((IndexReader)key).addReaderClosedListener(new IndexReader.ReaderClosedListener() {
@Override
public void onClose(IndexReader reader) {
FieldCache.DEFAULT.purge(reader);
}
});
}
}
}
/** Expert: Internal cache. */
abstract static class Cache {
Cache() {
this.wrapper = null;
}
Cache(FieldCacheImpl wrapper) {
this.wrapper = wrapper;
@ -198,11 +214,7 @@ class FieldCacheImpl implements FieldCache {
// First time this reader is using FieldCache
innerCache = new HashMap<Entry,Object>();
readerCache.put(readerKey, innerCache);
if (reader instanceof SegmentReader) {
((SegmentReader) reader).addCoreClosedListener(wrapper.purgeCore);
} else {
reader.addReaderClosedListener(wrapper.purgeReader);
}
wrapper.initReader(reader);
}
if (innerCache.get(key) == null) {
innerCache.put(key, value);
@ -223,11 +235,7 @@ class FieldCacheImpl implements FieldCache {
// First time this reader is using FieldCache
innerCache = new HashMap<Entry,Object>();
readerCache.put(readerKey, innerCache);
if (reader instanceof SegmentReader) {
((SegmentReader) reader).addCoreClosedListener(wrapper.purgeCore);
} else {
reader.addReaderClosedListener(wrapper.purgeReader);
}
wrapper.initReader(reader);
value = null;
} else {
value = innerCache.get(key);
@ -339,7 +347,7 @@ class FieldCacheImpl implements FieldCache {
}
final int maxDoc = reader.maxDoc();
final byte[] retArray = new byte[maxDoc];
Terms terms = MultiFields.getTerms(reader, field);
Terms terms = reader.terms(field);
FixedBitSet docsWithField = null;
if (terms != null) {
if (setDocsWithField) {
@ -412,7 +420,7 @@ class FieldCacheImpl implements FieldCache {
}
final int maxDoc = reader.maxDoc();
final short[] retArray = new short[maxDoc];
Terms terms = MultiFields.getTerms(reader, field);
Terms terms = reader.terms(field);
FixedBitSet docsWithField = null;
if (terms != null) {
if (setDocsWithField) {
@ -511,7 +519,7 @@ class FieldCacheImpl implements FieldCache {
final int maxDoc = reader.maxDoc();
int[] retArray = null;
Terms terms = MultiFields.getTerms(reader, field);
Terms terms = reader.terms(field);
FixedBitSet docsWithField = null;
if (terms != null) {
if (setDocsWithField) {
@ -583,7 +591,7 @@ class FieldCacheImpl implements FieldCache {
throws IOException {
final String field = entryKey.field;
FixedBitSet res = null;
Terms terms = MultiFields.getTerms(reader, field);
Terms terms = reader.terms(field);
final int maxDoc = reader.maxDoc();
if (terms != null) {
final int termsDocCount = terms.getDocCount();
@ -661,7 +669,7 @@ class FieldCacheImpl implements FieldCache {
final int maxDoc = reader.maxDoc();
float[] retArray = null;
Terms terms = MultiFields.getTerms(reader, field);
Terms terms = reader.terms(field);
FixedBitSet docsWithField = null;
if (terms != null) {
if (setDocsWithField) {
@ -749,7 +757,7 @@ class FieldCacheImpl implements FieldCache {
final int maxDoc = reader.maxDoc();
long[] retArray = null;
Terms terms = MultiFields.getTerms(reader, field);
Terms terms = reader.terms(field);
FixedBitSet docsWithField = null;
if (terms != null) {
if (setDocsWithField) {
@ -838,7 +846,7 @@ class FieldCacheImpl implements FieldCache {
final int maxDoc = reader.maxDoc();
double[] retArray = null;
Terms terms = MultiFields.getTerms(reader, field);
Terms terms = reader.terms(field);
FixedBitSet docsWithField = null;
if (terms != null) {
if (setDocsWithField) {
@ -1086,7 +1094,7 @@ class FieldCacheImpl implements FieldCache {
protected Object createValue(IndexReader reader, Entry entryKey, boolean setDocsWithField /* ignored */)
throws IOException {
Terms terms = MultiFields.getTerms(reader, entryKey.field);
Terms terms = reader.terms(entryKey.field);
final boolean fasterButMoreRAM = ((Boolean) entryKey.custom).booleanValue();
@ -1231,7 +1239,7 @@ class FieldCacheImpl implements FieldCache {
protected Object createValue(IndexReader reader, Entry entryKey, boolean setDocsWithField /* ignored */)
throws IOException {
Terms terms = MultiFields.getTerms(reader, entryKey.field);
Terms terms = reader.terms(entryKey.field);
final boolean fasterButMoreRAM = ((Boolean) entryKey.custom).booleanValue();

4
lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
View File

@ -636,12 +636,12 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
assertEquals("wrong number of hits", 34, hits.length);
// check decoding into field cache
int[] fci = FieldCache.DEFAULT.getInts(searcher.getIndexReader(), "trieInt", false);
int[] fci = FieldCache.DEFAULT.getInts(new SlowMultiReaderWrapper(searcher.getIndexReader()), "trieInt", false);
for (int val : fci) {
assertTrue("value in id bounds", val >= 0 && val < 35);
}
long[] fcl = FieldCache.DEFAULT.getLongs(searcher.getIndexReader(), "trieLong", false);
long[] fcl = FieldCache.DEFAULT.getLongs(new SlowMultiReaderWrapper(searcher.getIndexReader()), "trieLong", false);
for (long val : fcl) {
assertTrue("value in id bounds", val >= 0L && val < 35L);
}

4
lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java
View File

@ -168,7 +168,7 @@ public class TestDocTermOrds extends LuceneTestCase {
if (VERBOSE) {
System.out.println("TEST: top reader");
}
verify(r, idToOrds, termsArray, null);
verify(new SlowMultiReaderWrapper(r), idToOrds, termsArray, null);
FieldCache.DEFAULT.purge(r);
@ -285,7 +285,7 @@ public class TestDocTermOrds extends LuceneTestCase {
if (VERBOSE) {
System.out.println("TEST: top reader");
}
verify(r, idToOrdsPrefix, termsArray, prefixRef);
verify(new SlowMultiReaderWrapper(r), idToOrdsPrefix, termsArray, prefixRef);
}
FieldCache.DEFAULT.purge(r);

2
lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
View File

@ -1675,7 +1675,7 @@ public class TestIndexWriter extends LuceneTestCase {
w.close();
assertEquals(1, reader.docFreq(new Term("content", bigTerm)));
FieldCache.DocTermsIndex dti = FieldCache.DEFAULT.getTermsIndex(reader, "content", random.nextBoolean());
FieldCache.DocTermsIndex dti = FieldCache.DEFAULT.getTermsIndex(new SlowMultiReaderWrapper(reader), "content", random.nextBoolean());
assertEquals(5, dti.numOrd()); // +1 for null ord
assertEquals(4, dti.size());
assertEquals(bigTermBytesRef, dti.lookup(3, new BytesRef()));

2
lucene/src/test/org/apache/lucene/index/TestTermsEnum.java
View File

@ -232,7 +232,7 @@ public class TestTermsEnum extends LuceneTestCase {
w.close();
// NOTE: intentional insanity!!
final int[] docIDToID = FieldCache.DEFAULT.getInts(r, "id", false);
final int[] docIDToID = FieldCache.DEFAULT.getInts(new SlowMultiReaderWrapper(r), "id", false);
for(int iter=0;iter<10*RANDOM_MULTIPLIER;iter++) {

8
lucene/src/test/org/apache/lucene/search/TestFieldCache.java
View File

@ -98,7 +98,8 @@ public class TestFieldCache extends LuceneTestCase {
}
writer.addDocument(doc);
}
reader = writer.getReader();
IndexReader r = writer.getReader();
reader = new SlowMultiReaderWrapper(r);
writer.close();
}
@ -293,8 +294,9 @@ public class TestFieldCache extends LuceneTestCase {
Directory dir = newDirectory();
IndexWriter writer= new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(500));
IndexReader r = IndexReader.open(writer, true);
FieldCache.DEFAULT.getTerms(r, "foobar");
FieldCache.DEFAULT.getTermsIndex(r, "foobar");
SlowMultiReaderWrapper reader = new SlowMultiReaderWrapper(r);
FieldCache.DEFAULT.getTerms(reader, "foobar");
FieldCache.DEFAULT.getTermsIndex(reader, "foobar");
writer.close();
r.close();
dir.close();

25
lucene/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.search.FieldCache;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
import org.apache.lucene.util.FieldCacheSanityChecker.InsanityType;
@ -87,12 +88,12 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase {
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
cache.getDoubles(readerA, "theDouble", false);
cache.getDoubles(readerA, "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false);
cache.getDoubles(readerB, "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false);
cache.getDoubles(new SlowMultiReaderWrapper(readerA), "theDouble", false);
cache.getDoubles(new SlowMultiReaderWrapper(readerA), "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false);
cache.getDoubles(new SlowMultiReaderWrapper(readerB), "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false);
cache.getInts(readerX, "theInt", false);
cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER, false);
cache.getInts(new SlowMultiReaderWrapper(readerX), "theInt", false);
cache.getInts(new SlowMultiReaderWrapper(readerX), "theInt", FieldCache.DEFAULT_INT_PARSER, false);
// // //
@ -110,9 +111,9 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase {
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER, false);
cache.getTerms(readerX, "theInt");
cache.getBytes(readerX, "theByte", false);
cache.getInts(new SlowMultiReaderWrapper(readerX), "theInt", FieldCache.DEFAULT_INT_PARSER, false);
cache.getTerms(new SlowMultiReaderWrapper(readerX), "theInt");
cache.getBytes(new SlowMultiReaderWrapper(readerX), "theByte", false);
// // //
@ -134,11 +135,11 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase {
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
cache.getTerms(readerA, "theString");
cache.getTerms(readerB, "theString");
cache.getTerms(readerX, "theString");
cache.getTerms(new SlowMultiReaderWrapper(readerA), "theString");
cache.getTerms(new SlowMultiReaderWrapper(readerB), "theString");
cache.getTerms(new SlowMultiReaderWrapper(readerX), "theString");
cache.getBytes(readerX, "theByte", false);
cache.getBytes(new SlowMultiReaderWrapper(readerX), "theByte", false);
// // //

3
modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
View File

@ -51,6 +51,7 @@ import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.FieldCache.DocTermsIndex;
@ -332,7 +333,7 @@ public class TestPerfTasksLogic extends BenchmarkTestCase {
Benchmark benchmark = execBenchmark(algLines);
IndexReader r = IndexReader.open(benchmark.getRunData().getDirectory());
DocTermsIndex idx = FieldCache.DEFAULT.getTermsIndex(r, "country");
DocTermsIndex idx = FieldCache.DEFAULT.getTermsIndex(new SlowMultiReaderWrapper(r), "country");
final int maxDoc = r.maxDoc();
assertEquals(1000, maxDoc);
BytesRef br = new BytesRef();

2
modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
View File

@ -276,7 +276,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
w.close();
// NOTE: intentional but temporary field cache insanity!
final int[] docIdToFieldId = FieldCache.DEFAULT.getInts(r, "id", false);
final int[] docIdToFieldId = FieldCache.DEFAULT.getInts(new SlowMultiReaderWrapper(r), "id", false);
final int[] fieldIdToDocID = new int[numDocs];
for (int i = 0; i < docIdToFieldId.length; i++) {
int fieldId = docIdToFieldId[i];

4
modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
View File

@ -766,7 +766,7 @@ public class TestGrouping extends LuceneTestCase {
w.close();
// NOTE: intentional but temporary field cache insanity!
final int[] docIDToID = FieldCache.DEFAULT.getInts(r, "id", false);
final int[] docIDToID = FieldCache.DEFAULT.getInts(new SlowMultiReaderWrapper(r), "id", false);
IndexReader rBlocks = null;
Directory dirBlocks = null;
@ -799,7 +799,7 @@ public class TestGrouping extends LuceneTestCase {
dirBlocks = newDirectory();
rBlocks = getDocBlockReader(dirBlocks, groupDocs);
final Filter lastDocInBlock = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("groupend", "x"))));
final int[] docIDToIDBlocks = FieldCache.DEFAULT.getInts(rBlocks, "id", false);
final int[] docIDToIDBlocks = FieldCache.DEFAULT.getInts(new SlowMultiReaderWrapper(rBlocks), "id", false);
final IndexSearcher sBlocks = newSearcher(rBlocks);
final ShardState shardsBlocks = new ShardState(sBlocks);

4
modules/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.queries.function.valuesource;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
@ -61,11 +62,12 @@ public class OrdFieldSource extends ValueSource {
}
// TODO: this is trappy? perhaps this query instead should make you pass a slow reader yourself?
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final int off = readerContext.docBase;
final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader;
final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(topReader, field);
final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(new SlowMultiReaderWrapper(topReader), field);
return new IntDocValues(this) {
protected String toTerm(String readableValue) {
return readableValue;

4
modules/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.queries.function.valuesource;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
@ -61,12 +62,13 @@ public class ReverseOrdFieldSource extends ValueSource {
return "rord("+field+')';
}
// TODO: this is trappy? perhaps this query instead should make you pass a slow reader yourself?
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader;
final int off = readerContext.docBase;
final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(topReader, field);
final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(new SlowMultiReaderWrapper(topReader), field);
final int end = sindex.numOrd();
return new IntDocValues(this) {

6
solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
View File

@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
@ -246,12 +247,13 @@ class SimpleStats {
return res;
}
// why does this use a top-level field cache?
public NamedList<?> getFieldCacheStats(String fieldName, String[] facet ) {
SchemaField sf = searcher.getSchema().getField(fieldName);
FieldCache.DocTermsIndex si;
try {
si = FieldCache.DEFAULT.getTermsIndex(searcher.getIndexReader(), fieldName);
si = FieldCache.DEFAULT.getTermsIndex(new SlowMultiReaderWrapper(searcher.getIndexReader()), fieldName);
}
catch (IOException e) {
throw new RuntimeException( "failed to open field cache for: "+fieldName, e );
@ -273,7 +275,7 @@ class SimpleStats {
+ "[" + facetFieldType + "]");
}
try {
facetTermsIndex = FieldCache.DEFAULT.getTermsIndex(searcher.getIndexReader(), facetField);
facetTermsIndex = FieldCache.DEFAULT.getTermsIndex(new SlowMultiReaderWrapper(searcher.getIndexReader()), facetField);
}
catch (IOException e) {
throw new RuntimeException( "failed to open field cache for: "

2
solr/core/src/java/org/apache/solr/request/SimpleFacets.java
View File

@ -406,7 +406,7 @@ public class SimpleFacets {
FieldType ft = searcher.getSchema().getFieldType(fieldName);
NamedList<Integer> res = new NamedList<Integer>();
FieldCache.DocTermsIndex si = FieldCache.DEFAULT.getTermsIndex(searcher.getIndexReader(), fieldName);
FieldCache.DocTermsIndex si = FieldCache.DEFAULT.getTermsIndex(new SlowMultiReaderWrapper(searcher.getIndexReader()), fieldName);
final BytesRef prefixRef;
if (prefix == null) {

3
solr/core/src/java/org/apache/solr/request/UnInvertedField.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.request;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.TermQuery;
@ -484,7 +485,7 @@ public class UnInvertedField extends DocTermOrds {
for (String f : facet) {
SchemaField facet_sf = searcher.getSchema().getField(f);
try {
si = FieldCache.DEFAULT.getTermsIndex(searcher.getIndexReader(), f);
si = FieldCache.DEFAULT.getTermsIndex(new SlowMultiReaderWrapper(searcher.getIndexReader()), f);
}
catch (IOException e) {
throw new RuntimeException("failed to open field cache for: " + f, e);