Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-1749: Addition of FieldCacheSanityChecker utility, and hooks to use it in all existing Lucene Tests. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@803676 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris M. Hostetter 2009-08-12 19:31:38 +00:00
parent ab2afad0d1
commit 2558c06120
18 changed files with 1870 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CHANGES.txt
View File

@ -669,6 +669,16 @@ New features
based on the maximum payload seen for a document.
Slight refactoring of Similarity and other payload queries (Grant Ingersoll)
36. LUCENE-1749: Addition of FieldCacheSanityChecker utility, and
hooks to use it in all existing Lucene Tests. This class can
be used by any application to inspect the FieldCache and provide
diagnostic information about the possibility of inconsistent
FieldCache usage. Namely: FieldCache entries for the same field
with different datatypes or parsers; and FieldCache entries for
the same field in both a reader, and one of it's (descendant) sub
readers.
(Chris Hostetter, Mark Miller)
Optimizations
1. LUCENE-1427: Fixed QueryWrapperFilter to not waste time computing

28
contrib/remote/src/test/org/apache/lucene/search/TestRemoteSort.java
View File

@ -244,11 +244,18 @@ public class TestRemoteSort extends LuceneTestCase implements Serializable {
assertMatches (multi, queryX, sort, "CAIEG");
sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource(), true));
assertMatches (multi, queryY, sort, "HJDBF");
assertSaneFieldCaches(getName() + " ComparatorSource");
FieldCache.DEFAULT.purgeAllCaches();
SortComparator custom = SampleComparable.getComparator();
sort.setSort (new SortField ("custom", custom));
assertMatches (multi, queryX, sort, "CAIEG");
sort.setSort (new SortField ("custom", custom, true));
assertMatches (multi, queryY, sort, "HJDBF");
assertSaneFieldCaches(getName() + " Comparator");
FieldCache.DEFAULT.purgeAllCaches();
}
// test that the relevancy scores are the same even if
@ -343,12 +350,6 @@ public class TestRemoteSort extends LuceneTestCase implements Serializable {
sort.setSort("string", true);
assertMatches(multi, queryA, sort, "CBEFGHIAJD");
sort.setSort(new SortField[] { new SortField ("string", Locale.US) });
assertMatches(multi, queryA, sort, "DJAIHGFEBC");
sort.setSort(new SortField[] { new SortField ("string", Locale.US, true) });
assertMatches(multi, queryA, sort, "CBEFGHIAJD");
sort.setSort(new String[] {"int","float"});
assertMatches(multi, queryA, sort, "IDHFGJEABC");
@ -369,6 +370,21 @@ public class TestRemoteSort extends LuceneTestCase implements Serializable {
sort.setSort("string", true);
assertMatches(multi, queryF, sort, "IJZ");
// up to this point, all of the searches should have "sane"
// FieldCache behavior, and should have reused hte cache in several cases
assertSaneFieldCaches(getName() + " Basics");
// next we'll check an alternate Locale for string, so purge first
FieldCache.DEFAULT.purgeAllCaches();
sort.setSort(new SortField[] { new SortField ("string", Locale.US) });
assertMatches(multi, queryA, sort, "DJAIHGFEBC");
sort.setSort(new SortField[] { new SortField ("string", Locale.US, true)});
assertMatches(multi, queryA, sort, "CBEFGHIAJD");
assertSaneFieldCaches(getName() + " Locale.US");
FieldCache.DEFAULT.purgeAllCaches();
}
// make sure the documents returned by the search match the expected list

138
src/java/org/apache/lucene/search/FieldCache.java
View File

@ -19,12 +19,15 @@ package org.apache.lucene.search;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.document.NumericField; // for javadocs
import org.apache.lucene.analysis.NumericTokenStream; // for javadocs
import java.io.IOException;
import java.io.Serializable;
import java.text.DecimalFormat;
/**
* Expert: Maintains caches of term values.
*
@ -32,9 +35,14 @@ import java.io.Serializable;
*
* @since lucene 1.4
* @version $Id$
* @see org.apache.lucene.util.FieldCacheSanityChecker
*/
public interface FieldCache {
public static final class CreationPlaceholder {
Object value;
}
/** Indicator for StringIndex values in the cache. */
// NOTE: the value assigned to this constant must not be
// the same as any of those in SortField!!
@ -146,6 +154,9 @@ public interface FieldCache {
protected Object readResolve() {
return DEFAULT_BYTE_PARSER;
}
public String toString() {
return FieldCache.class.getName()+".DEFAULT_BYTE_PARSER";
}
};
/** The default parser for short values, which are encoded by {@link Short#toString(short)} */
@ -156,6 +167,9 @@ public interface FieldCache {
protected Object readResolve() {
return DEFAULT_SHORT_PARSER;
}
public String toString() {
return FieldCache.class.getName()+".DEFAULT_SHORT_PARSER";
}
};
/** The default parser for int values, which are encoded by {@link Integer#toString(int)} */
@ -166,6 +180,9 @@ public interface FieldCache {
protected Object readResolve() {
return DEFAULT_INT_PARSER;
}
public String toString() {
return FieldCache.class.getName()+".DEFAULT_INT_PARSER";
}
};
/** The default parser for float values, which are encoded by {@link Float#toString(float)} */
@ -176,6 +193,9 @@ public interface FieldCache {
protected Object readResolve() {
return DEFAULT_FLOAT_PARSER;
}
public String toString() {
return FieldCache.class.getName()+".DEFAULT_FLOAT_PARSER";
}
};
/** The default parser for long values, which are encoded by {@link Long#toString(long)} */
@ -186,6 +206,9 @@ public interface FieldCache {
protected Object readResolve() {
return DEFAULT_LONG_PARSER;
}
public String toString() {
return FieldCache.class.getName()+".DEFAULT_LONG_PARSER";
}
};
/** The default parser for double values, which are encoded by {@link Double#toString(double)} */
@ -196,6 +219,9 @@ public interface FieldCache {
protected Object readResolve() {
return DEFAULT_DOUBLE_PARSER;
}
public String toString() {
return FieldCache.class.getName()+".DEFAULT_DOUBLE_PARSER";
}
};
/**
@ -212,6 +238,9 @@ public interface FieldCache {
protected Object readResolve() {
return NUMERIC_UTILS_INT_PARSER;
}
public String toString() {
return FieldCache.class.getName()+".NUMERIC_UTILS_INT_PARSER";
}
};
/**
@ -228,6 +257,9 @@ public interface FieldCache {
protected Object readResolve() {
return NUMERIC_UTILS_FLOAT_PARSER;
}
public String toString() {
return FieldCache.class.getName()+".NUMERIC_UTILS_FLOAT_PARSER";
}
};
/**
@ -244,6 +276,9 @@ public interface FieldCache {
protected Object readResolve() {
return NUMERIC_UTILS_LONG_PARSER;
}
public String toString() {
return FieldCache.class.getName()+".NUMERIC_UTILS_LONG_PARSER";
}
};
/**
@ -260,6 +295,9 @@ public interface FieldCache {
protected Object readResolve() {
return NUMERIC_UTILS_DOUBLE_PARSER;
}
public String toString() {
return FieldCache.class.getName()+".NUMERIC_UTILS_DOUBLE_PARSER";
}
};
/** Checks the internal cache for an appropriate entry, and if none is
@ -478,4 +516,104 @@ public interface FieldCache {
public Comparable[] getCustom (IndexReader reader, String field, SortComparator comparator)
throws IOException;
/**
* EXPERT: A unique Identifier/Description for each item in the FieldCache.
* Can be useful for logging/debugging.
* <p>
* <b>EXPERIMENTAL API:</b> This API is considered extremely advanced
* and experimental. It may be removed or altered w/o warning in future
* releases
* of Lucene.
* </p>
*/
public static abstract class CacheEntry {
public abstract Object getReaderKey();
public abstract String getFieldName();
public abstract Class getCacheType();
public abstract Object getCustom();
public abstract Object getValue();
private String size = null;
protected final void setEstimatedSize(String size) {
this.size = size;
}
/**
* @see #estimateSize(RamUsageEstimator)
*/
public void estimateSize() {
estimateSize(new RamUsageEstimator(false)); // doesn't check for interned
}
/**
* Computes (and stores) the estimated size of the cache Value
* @see #getEstimatedSize
*/
public void estimateSize(RamUsageEstimator ramCalc) {
long size = ramCalc.estimateRamUsage(getValue());
setEstimatedSize(RamUsageEstimator.humanReadableUnits
(size, new DecimalFormat("0.#")));
}
/**
* The most recently estimated size of the value, null unless
* estimateSize has been called.
*/
public final String getEstimatedSize() {
return size;
}
public String toString() {
StringBuffer b = new StringBuffer();
b.append("'").append(getReaderKey()).append("'=>");
b.append("'").append(getFieldName()).append("',");
b.append(getCacheType()).append(",").append(getCustom());
b.append("=>").append(getValue().getClass().getName()).append("#");
b.append(System.identityHashCode(getValue()));
String s = getEstimatedSize();
if(null != s) {
b.append(" (size =~ ").append(s).append(')');
}
return b.toString();
}
}
/**
* EXPERT: Generates an array of CacheEntry objects representing all items
* currently in the FieldCache.
* <p>
* NOTE: These CacheEntry objects maintain a strong refrence to the
* Cached Values. Maintaining refrences to a CacheEntry the IndexReader
* associated with it has garbage collected will prevent the Value itself
* from being garbage collected when the Cache drops the WeakRefrence.
* </p>
* <p>
* <b>EXPERIMENTAL API:</b> This API is considered extremely advanced
* and experimental. It may be removed or altered w/o warning in future
* releases
* of Lucene.
* </p>
*/
public abstract CacheEntry[] getCacheEntries();
/**
* <p>
* EXPERT: Instructs the FieldCache to forcibly expunge all entries
* from the underlying caches. This is intended only to be used for
* test methods as a way to ensure a known base state of the Cache
* (with out needing to rely on GC to free WeakReferences).
* It should not be relied on for "Cache maintenance" in general
* application code.
* </p>
* <p>
* <b>EXPERIMENTAL API:</b> This API is considered extremely advanced
* and experimental. It may be removed or altered w/o warning in future
* releases
* of Lucene.
* </p>
*/
public abstract void purgeAllCaches();
}

299
src/java/org/apache/lucene/search/FieldCacheImpl.java
View File

@ -17,18 +17,22 @@ package org.apache.lucene.search;
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.WeakHashMap;
import org.apache.lucene.document.NumericField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.TermEnum;
import org.apache.lucene.util.StringHelper;
import java.io.IOException;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.WeakHashMap;
/**
* Expert: The default cache implementation, storing all values in memory.
* A WeakHashMap is used for storage.
@ -41,6 +45,116 @@ import java.util.WeakHashMap;
// TODO: change interface to FieldCache in 3.0 when removed
class FieldCacheImpl implements ExtendedFieldCache {
private Map caches;
FieldCacheImpl() {
init();
}
private synchronized void init() {
caches = new HashMap(7);
caches.put(Byte.TYPE, new ByteCache(this));
caches.put(Short.TYPE, new ShortCache(this));
caches.put(Integer.TYPE, new IntCache(this));
caches.put(Float.TYPE, new FloatCache(this));
caches.put(Long.TYPE, new LongCache(this));
caches.put(Double.TYPE, new DoubleCache(this));
caches.put(String.class, new StringCache(this));
caches.put(StringIndex.class, new StringIndexCache(this));
caches.put(Comparable.class, new CustomCache(this));
caches.put(Object.class, new AutoCache(this));
}
public void purgeAllCaches() {
init();
}
public CacheEntry[] getCacheEntries() {
List result = new ArrayList(17);
Iterator outerKeys = caches.keySet().iterator();
while (outerKeys.hasNext()) {
Class cacheType = (Class)outerKeys.next();
Cache cache = (Cache)caches.get(cacheType);
Iterator innerKeys = cache.readerCache.keySet().iterator();
while (innerKeys.hasNext()) {
// we've now materialized a hard ref
Object readerKey = innerKeys.next();
// innerKeys was backed by WeakHashMap, sanity check
// that it wasn't GCed before we made hard ref
if (null != readerKey && cache.readerCache.containsKey(readerKey)) {
Map innerCache = ((Map)cache.readerCache.get(readerKey));
Iterator keys = innerCache.keySet().iterator();
while (keys.hasNext()) {
Entry entry = (Entry) keys.next();
result.add(new CacheEntryImpl(readerKey, entry.field,
cacheType, entry.type,
entry.custom, entry.locale,
innerCache.get(entry)));
}
}
}
}
return (CacheEntry[]) result.toArray(new CacheEntry[result.size()]);
}
private static final class CacheEntryImpl extends CacheEntry {
/**
* @deprecated Only needed because of Entry (ab)use by
* FieldSortedHitQueue, remove when FieldSortedHitQueue
* is removed
*/
private final int sortFieldType;
/**
* @deprecated Only needed because of Entry (ab)use by
* FieldSortedHitQueue, remove when FieldSortedHitQueue
* is removed
*/
private final Locale locale;
private final Object readerKey;
private final String fieldName;
private final Class cacheType;
private final Object custom;
private final Object value;
CacheEntryImpl(Object readerKey, String fieldName,
Class cacheType, int sortFieldType,
Object custom, Locale locale,
Object value) {
this.readerKey = readerKey;
this.fieldName = fieldName;
this.cacheType = cacheType;
this.sortFieldType = sortFieldType;
this.custom = custom;
this.locale = locale;
this.value = value;
// :HACK: for testing.
// if (null != locale || SortField.CUSTOM != sortFieldType) {
// throw new RuntimeException("Locale/sortFieldType: " + this);
// }
}
public Object getReaderKey() { return readerKey; }
public String getFieldName() { return fieldName; }
public Class getCacheType() { return cacheType; }
public Object getCustom() { return custom; }
public Object getValue() { return value; }
/**
* Adds warning to super.toString if Local or sortFieldType were specified
* @deprecated Only needed because of Entry (ab)use by
* FieldSortedHitQueue, remove when FieldSortedHitQueue
* is removed
*/
public String toString() {
String r = super.toString();
if (null != locale) {
r = r + "...!!!Locale:" + locale + "???";
}
if (SortField.CUSTOM != sortFieldType) {
r = r + "...!!!SortType:" + sortFieldType + "???";
}
return r;
}
}
/**
* Hack: When thrown from a Parser (NUMERIC_UTILS_* ones), this stops
* processing terms and returns the current FieldCache
@ -51,16 +165,25 @@ class FieldCacheImpl implements ExtendedFieldCache {
/** Expert: Internal cache. */
abstract static class Cache {
private final Map readerCache = new WeakHashMap();
Cache() {
this.wrapper = null;
}
protected abstract Object createValue(IndexReader reader, Object key)
Cache(FieldCache wrapper) {
this.wrapper = wrapper;
}
final FieldCache wrapper;
final Map readerCache = new WeakHashMap();
protected abstract Object createValue(IndexReader reader, Entry key)
throws IOException;
public Object get(IndexReader reader, Object key) throws IOException {
public Object get(IndexReader reader, Entry key) throws IOException {
Map innerCache;
Object value;
final Object readerKey = reader.getFieldCacheKey();
synchronized (readerCache) {
innerCache = (Map) readerCache.get(readerKey);
if (innerCache == null) {
@ -91,18 +214,25 @@ class FieldCacheImpl implements ExtendedFieldCache {
}
}
static final class CreationPlaceholder {
Object value;
}
/** Expert: Every composite-key in the internal cache is of this type. */
static class Entry {
final String field; // which Fieldable
/**
* @deprecated Only (ab)used by FieldSortedHitQueue,
* remove when FieldSortedHitQueue is removed
*/
final int type; // which SortField type
final Object custom; // which custom comparator or parser
/**
* @deprecated Only (ab)used by FieldSortedHitQueue,
* remove when FieldSortedHitQueue is removed
*/
final Locale locale; // the locale we're sorting (if string)
/** Creates one of these objects. */
/**
* @deprecated Only (ab)used by FieldSortedHitQueue,
* remove when FieldSortedHitQueue is removed
*/
Entry (String field, int type, Locale locale) {
this.field = StringHelper.intern(field);
this.type = type;
@ -118,7 +248,10 @@ class FieldCacheImpl implements ExtendedFieldCache {
this.locale = null;
}
/** Creates one of these objects for a custom type with parser, needed by FieldSortedHitQueue. */
/**
* @deprecated Only (ab)used by FieldSortedHitQueue,
* remove when FieldSortedHitQueue is removed
*/
Entry (String field, int type, Parser parser) {
this.field = StringHelper.intern(field);
this.type = type;
@ -157,18 +290,20 @@ class FieldCacheImpl implements ExtendedFieldCache {
// inherit javadocs
public byte[] getBytes(IndexReader reader, String field, ByteParser parser)
throws IOException {
return (byte[]) bytesCache.get(reader, new Entry(field, parser));
return (byte[]) ((Cache)caches.get(Byte.TYPE)).get(reader, new Entry(field, parser));
}
Cache bytesCache = new Cache() {
protected Object createValue(IndexReader reader, Object entryKey)
static final class ByteCache extends Cache {
ByteCache(FieldCache wrapper) {
super(wrapper);
}
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
Entry entry = (Entry) entryKey;
String field = entry.field;
ByteParser parser = (ByteParser) entry.custom;
if (parser == null) {
return getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER);
return wrapper.getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER);
}
final byte[] retArray = new byte[reader.maxDoc()];
TermDocs termDocs = reader.termDocs();
@ -200,18 +335,21 @@ class FieldCacheImpl implements ExtendedFieldCache {
// inherit javadocs
public short[] getShorts(IndexReader reader, String field, ShortParser parser)
throws IOException {
return (short[]) shortsCache.get(reader, new Entry(field, parser));
return (short[]) ((Cache)caches.get(Short.TYPE)).get(reader, new Entry(field, parser));
}
Cache shortsCache = new Cache() {
static final class ShortCache extends Cache {
ShortCache(FieldCache wrapper) {
super(wrapper);
}
protected Object createValue(IndexReader reader, Object entryKey)
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
Entry entry = (Entry) entryKey;
String field = entry.field;
ShortParser parser = (ShortParser) entry.custom;
if (parser == null) {
return getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER);
return wrapper.getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER);
}
final short[] retArray = new short[reader.maxDoc()];
TermDocs termDocs = reader.termDocs();
@ -243,21 +381,24 @@ class FieldCacheImpl implements ExtendedFieldCache {
// inherit javadocs
public int[] getInts(IndexReader reader, String field, IntParser parser)
throws IOException {
return (int[]) intsCache.get(reader, new Entry(field, parser));
return (int[]) ((Cache)caches.get(Integer.TYPE)).get(reader, new Entry(field, parser));
}
Cache intsCache = new Cache() {
static final class IntCache extends Cache {
IntCache(FieldCache wrapper) {
super(wrapper);
}
protected Object createValue(IndexReader reader, Object entryKey)
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
Entry entry = (Entry) entryKey;
String field = entry.field;
IntParser parser = (IntParser) entry.custom;
if (parser == null) {
try {
return getInts(reader, field, DEFAULT_INT_PARSER);
return wrapper.getInts(reader, field, DEFAULT_INT_PARSER);
} catch (NumberFormatException ne) {
return getInts(reader, field, NUMERIC_UTILS_INT_PARSER);
return wrapper.getInts(reader, field, NUMERIC_UTILS_INT_PARSER);
}
}
int[] retArray = null;
@ -295,24 +436,28 @@ class FieldCacheImpl implements ExtendedFieldCache {
// inherit javadocs
public float[] getFloats(IndexReader reader, String field, FloatParser parser)
throws IOException {
return (float[]) floatsCache.get(reader, new Entry(field, parser));
throws IOException {
return (float[]) ((Cache)caches.get(Float.TYPE)).get(reader, new Entry(field, parser));
}
Cache floatsCache = new Cache() {
static final class FloatCache extends Cache {
FloatCache(FieldCache wrapper) {
super(wrapper);
}
protected Object createValue(IndexReader reader, Object entryKey)
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
Entry entry = (Entry) entryKey;
String field = entry.field;
FloatParser parser = (FloatParser) entry.custom;
if (parser == null) {
try {
return getFloats(reader, field, DEFAULT_FLOAT_PARSER);
return wrapper.getFloats(reader, field, DEFAULT_FLOAT_PARSER);
} catch (NumberFormatException ne) {
return getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER);
return wrapper.getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER);
}
}
}
float[] retArray = null;
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms (new Term (field));
@ -347,27 +492,30 @@ class FieldCacheImpl implements ExtendedFieldCache {
// inherit javadocs
public long[] getLongs(IndexReader reader, String field, FieldCache.LongParser parser)
throws IOException {
return (long[]) longsCache.get(reader, new Entry(field, parser));
return (long[]) ((Cache)caches.get(Long.TYPE)).get(reader, new Entry(field, parser));
}
/** @deprecated Will be removed in 3.0, this is for binary compatibility only */
public long[] getLongs(IndexReader reader, String field, ExtendedFieldCache.LongParser parser)
throws IOException {
return (long[]) longsCache.get(reader, new Entry(field, parser));
return (long[]) ((Cache)caches.get(Long.TYPE)).get(reader, new Entry(field, parser));
}
Cache longsCache = new Cache() {
static final class LongCache extends Cache {
LongCache(FieldCache wrapper) {
super(wrapper);
}
protected Object createValue(IndexReader reader, Object entryKey)
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
Entry entry = (Entry) entryKey;
String field = entry.field;
FieldCache.LongParser parser = (FieldCache.LongParser) entry.custom;
if (parser == null) {
try {
return getLongs(reader, field, DEFAULT_LONG_PARSER);
return wrapper.getLongs(reader, field, DEFAULT_LONG_PARSER);
} catch (NumberFormatException ne) {
return getLongs(reader, field, NUMERIC_UTILS_LONG_PARSER);
return wrapper.getLongs(reader, field, NUMERIC_UTILS_LONG_PARSER);
}
}
long[] retArray = null;
@ -405,27 +553,30 @@ class FieldCacheImpl implements ExtendedFieldCache {
// inherit javadocs
public double[] getDoubles(IndexReader reader, String field, FieldCache.DoubleParser parser)
throws IOException {
return (double[]) doublesCache.get(reader, new Entry(field, parser));
return (double[]) ((Cache)caches.get(Double.TYPE)).get(reader, new Entry(field, parser));
}
/** @deprecated Will be removed in 3.0, this is for binary compatibility only */
public double[] getDoubles(IndexReader reader, String field, ExtendedFieldCache.DoubleParser parser)
throws IOException {
return (double[]) doublesCache.get(reader, new Entry(field, parser));
return (double[]) ((Cache)caches.get(Double.TYPE)).get(reader, new Entry(field, parser));
}
Cache doublesCache = new Cache() {
static final class DoubleCache extends Cache {
DoubleCache(FieldCache wrapper) {
super(wrapper);
}
protected Object createValue(IndexReader reader, Object entryKey)
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
Entry entry = (Entry) entryKey;
String field = entry.field;
FieldCache.DoubleParser parser = (FieldCache.DoubleParser) entry.custom;
if (parser == null) {
try {
return getDoubles(reader, field, DEFAULT_DOUBLE_PARSER);
return wrapper.getDoubles(reader, field, DEFAULT_DOUBLE_PARSER);
} catch (NumberFormatException ne) {
return getDoubles(reader, field, NUMERIC_UTILS_DOUBLE_PARSER);
return wrapper.getDoubles(reader, field, NUMERIC_UTILS_DOUBLE_PARSER);
}
}
double[] retArray = null;
@ -457,14 +608,17 @@ class FieldCacheImpl implements ExtendedFieldCache {
// inherit javadocs
public String[] getStrings(IndexReader reader, String field)
throws IOException {
return (String[]) stringsCache.get(reader, field);
return (String[]) ((Cache)caches.get(String.class)).get(reader, new Entry(field, (Parser)null));
}
Cache stringsCache = new Cache() {
static final class StringCache extends Cache {
StringCache(FieldCache wrapper) {
super(wrapper);
}
protected Object createValue(IndexReader reader, Object fieldKey)
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
String field = StringHelper.intern((String) fieldKey);
String field = StringHelper.intern((String) entryKey.field);
final String[] retArray = new String[reader.maxDoc()];
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms (new Term (field));
@ -489,14 +643,17 @@ class FieldCacheImpl implements ExtendedFieldCache {
// inherit javadocs
public StringIndex getStringIndex(IndexReader reader, String field)
throws IOException {
return (StringIndex) stringsIndexCache.get(reader, field);
return (StringIndex) ((Cache)caches.get(StringIndex.class)).get(reader, new Entry(field, (Parser)null));
}
Cache stringsIndexCache = new Cache() {
static final class StringIndexCache extends Cache {
StringIndexCache(FieldCache wrapper) {
super(wrapper);
}
protected Object createValue(IndexReader reader, Object fieldKey)
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
String field = StringHelper.intern((String) fieldKey);
String field = StringHelper.intern((String) entryKey.field);
final int[] retArray = new int[reader.maxDoc()];
String[] mterms = new String[reader.maxDoc()+1];
TermDocs termDocs = reader.termDocs();
@ -563,7 +720,7 @@ class FieldCacheImpl implements ExtendedFieldCache {
// inherit javadocs
public Object getAuto(IndexReader reader, String field) throws IOException {
return autoCache.get(reader, field);
return ((Cache)caches.get(Object.class)).get(reader, new Entry(field, (Parser)null));
}
/**
@ -571,11 +728,14 @@ class FieldCacheImpl implements ExtendedFieldCache {
* Especially, guessing does <b>not</b> work with the new
* {@link NumericField} type.
*/
Cache autoCache = new Cache() {
static final class AutoCache extends Cache {
AutoCache(FieldCache wrapper) {
super(wrapper);
}
protected Object createValue(IndexReader reader, Object fieldKey)
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
String field = StringHelper.intern((String) fieldKey);
String field = StringHelper.intern((String) entryKey.field);
TermEnum enumerator = reader.terms (new Term (field));
try {
Term term = enumerator.term();
@ -588,17 +748,17 @@ class FieldCacheImpl implements ExtendedFieldCache {
try {
Integer.parseInt (termtext);
ret = getInts (reader, field);
ret = wrapper.getInts (reader, field);
} catch (NumberFormatException nfe1) {
try {
Long.parseLong(termtext);
ret = getLongs (reader, field);
ret = wrapper.getLongs (reader, field);
} catch (NumberFormatException nfe2) {
try {
Float.parseFloat (termtext);
ret = getFloats (reader, field);
ret = wrapper.getFloats (reader, field);
} catch (NumberFormatException nfe3) {
ret = getStringIndex (reader, field);
ret = wrapper.getStringIndex (reader, field);
}
}
}
@ -615,13 +775,16 @@ class FieldCacheImpl implements ExtendedFieldCache {
/** @deprecated */
public Comparable[] getCustom(IndexReader reader, String field,
SortComparator comparator) throws IOException {
return (Comparable[]) customCache.get(reader, new Entry(field, comparator));
return (Comparable[]) ((Cache)caches.get(Comparable.class)).get(reader, new Entry(field, comparator));
}
/** @deprecated */
Cache customCache = new Cache() {
static final class CustomCache extends Cache {
CustomCache(FieldCache wrapper) {
super(wrapper);
}
protected Object createValue(IndexReader reader, Object entryKey)
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
Entry entry = (Entry) entryKey;
String field = entry.field;

2
src/java/org/apache/lucene/search/FieldSortedHitQueue.java
View File

@ -180,7 +180,7 @@ extends PriorityQueue {
* caches comparators instead of term values. */
static final FieldCacheImpl.Cache Comparators = new FieldCacheImpl.Cache() {
protected Object createValue(IndexReader reader, Object entryKey)
protected Object createValue(IndexReader reader, FieldCacheImpl.Entry entryKey)
throws IOException {
FieldCacheImpl.Entry entry = (FieldCacheImpl.Entry) entryKey;
String fieldname = entry.field;

74
src/java/org/apache/lucene/util/AverageGuessMemoryModel.java Normal file
View File

@ -0,0 +1,74 @@
package org.apache.lucene.util;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.IdentityHashMap;
import java.util.Map;
/**
* An average, best guess, MemoryModel that should work okay on most systems.
*
*/
public class AverageGuessMemoryModel extends MemoryModel {
// best guess primitive sizes
private final Map sizes = new IdentityHashMap() {
{
put(boolean.class, new Integer(1));
put(byte.class, new Integer(1));
put(char.class, new Integer(2));
put(short.class, new Integer(2));
put(int.class, new Integer(4));
put(float.class, new Integer(4));
put(double.class, new Integer(8));
put(long.class, new Integer(8));
}
};
/*
* (non-Javadoc)
*
* @see org.apache.lucene.util.MemoryModel#getArraySize()
*/
public int getArraySize() {
return 16;
}
/*
* (non-Javadoc)
*
* @see org.apache.lucene.util.MemoryModel#getClassSize()
*/
public int getClassSize() {
return 8;
}
/* (non-Javadoc)
* @see org.apache.lucene.util.MemoryModel#getPrimitiveSize(java.lang.Class)
*/
public int getPrimitiveSize(Class clazz) {
return ((Integer) sizes.get(clazz)).intValue();
}
/* (non-Javadoc)
* @see org.apache.lucene.util.MemoryModel#getReferenceSize()
*/
public int getReferenceSize() {
return 4;
}
}

436
src/java/org/apache/lucene/util/FieldCacheSanityChecker.java Normal file
View File

@ -0,0 +1,436 @@
package org.apache.lucene.util;
/**
* Copyright 2009 The Apache Software Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldCache.CacheEntry;
/**
* Provides methods for sanity checking that entries in the FieldCache
* are not wasteful or inconsistent.
* </p>
* <p>
* Lucene 2.9 Introduced numerous enhancements into how the FieldCache
* is used by the low levels of Lucene searching (for Sorting and
* ValueSourceQueries) to improve both the speed for Sorting, as well
* as reopening of IndexReaders. But these changes have shifted the
* usage of FieldCache from "top level" IndexReaders (frequently a
* MultiReader or DirectoryReader) down to the leaf level SegmentReaders.
* As a result, existing applications that directly access the FieldCache
* may find RAM usage increase significantly when upgrading to 2.9 or
* Later. This class provides an API for these applications (or their
* Unit tests) to check at run time if the FieldCache contains "insane"
* usages of the FieldCache.
* </p>
* <p>
* <b>EXPERIMENTAL API:</b> This API is considered extremely advanced and
* experimental. It may be removed or altered w/o warning in future releases
* of Lucene.
* </p>
* @see FieldCache
* @see FieldCacheSanityChecker.Insanity
* @see FieldCacheSanityChecker.InsanityType
*/
public final class FieldCacheSanityChecker {
private RamUsageEstimator ramCalc = null;
public FieldCacheSanityChecker() {
/* NOOP */
}
/**
* If set, will be used to estimate size for all CacheEntry objects
* dealt with.
*/
public void setRamUsageEstimator(RamUsageEstimator r) {
ramCalc = r;
}
/**
* Quick and dirty convenience method
* @see #check
*/
public static Insanity[] checkSanity(FieldCache cache) {
return checkSanity(cache.getCacheEntries());
}
/**
* Quick and dirty convenience method that instantiates an instance with
* "good defaults" and uses it to test the CacheEntry[]
* @see #check
*/
public static Insanity[] checkSanity(CacheEntry[] cacheEntries) {
FieldCacheSanityChecker sanityChecker = new FieldCacheSanityChecker();
// doesn't check for interned
sanityChecker.setRamUsageEstimator(new RamUsageEstimator(false));
return sanityChecker.check(cacheEntries);
}
/**
* Tests a CacheEntry[] for indication of "insane" cache usage.
* <p>
* <B>NOTE:</b>FieldCache CreationPlaceholder objects are ignored.
* (:TODO: is this a bad idea? are we masking a real problem?)
* </p>
*/
public Insanity[] check(CacheEntry[] cacheEntries) {
if (null == cacheEntries || 0 == cacheEntries.length)
return new Insanity[0];
if (null != ramCalc) {
for (int i = 0; i < cacheEntries.length; i++) {
cacheEntries[i].estimateSize(ramCalc);
}
}
// the indirect mapping lets MapOfSet dedup identical valIds for us
//
// maps the (valId) identityhashCode of cache values to
// sets of CacheEntry instances
final MapOfSets valIdToItems = new MapOfSets(new HashMap(17));
// maps ReaderField keys to Sets of ValueIds
final MapOfSets readerFieldToValIds = new MapOfSets(new HashMap(17));
//
// any keys that we know result in more then one valId
final Set valMismatchKeys = new HashSet();
// iterate over all the cacheEntries to get the mappings we'll need
for (int i = 0; i < cacheEntries.length; i++) {
final CacheEntry item = cacheEntries[i];
final Object val = item.getValue();
if (val instanceof FieldCache.CreationPlaceholder)
continue;
final ReaderField rf = new ReaderField(item.getReaderKey(),
item.getFieldName());
final Integer valId = new Integer(System.identityHashCode(val));
// indirect mapping, so the MapOfSet will dedup identical valIds for us
valIdToItems.put(valId, item);
if (1 < readerFieldToValIds.put(rf, valId)) {
valMismatchKeys.add(rf);
}
}
final List insanity = new ArrayList(valMismatchKeys.size() * 3);
insanity.addAll(checkValueMismatch(valIdToItems,
readerFieldToValIds,
valMismatchKeys));
insanity.addAll(checkSubreaders(valIdToItems,
readerFieldToValIds));
return (Insanity[]) insanity.toArray(new Insanity[insanity.size()]);
}
/**
* Internal helper method used by check that iterates over
* valMismatchKeys and generates a Collection of Insanity
* instances accordingly. The MapOfSets are used to populate
* the Insantiy objects.
* @see InsanityType#VALUEMISMATCH
*/
private Collection checkValueMismatch(MapOfSets valIdToItems,
MapOfSets readerFieldToValIds,
Set valMismatchKeys) {
final List insanity = new ArrayList(valMismatchKeys.size() * 3);
if (! valMismatchKeys.isEmpty() ) {
// we have multiple values for some ReaderFields
final Map rfMap = readerFieldToValIds.getMap();
final Map valMap = valIdToItems.getMap();
final Iterator mismatchIter = valMismatchKeys.iterator();
while (mismatchIter.hasNext()) {
final ReaderField rf = (ReaderField)mismatchIter.next();
final List badEntries = new ArrayList(valMismatchKeys.size() * 2);
final Iterator valIter = ((Set)rfMap.get(rf)).iterator();
while (valIter.hasNext()) {
Iterator entriesIter = ((Set)valMap.get(valIter.next())).iterator();
while (entriesIter.hasNext()) {
badEntries.add(entriesIter.next());
}
}
CacheEntry[] badness = new CacheEntry[badEntries.size()];
badness = (CacheEntry[]) badEntries.toArray(badness);
insanity.add(new Insanity(InsanityType.VALUEMISMATCH,
"Multiple distinct value objects for " +
rf.toString(), badness));
}
}
return insanity;
}
/**
* Internal helper method used by check that iterates over
* the keys of readerFieldToValIds and generates a Collection
* of Insanity instances whenever two (or more) ReaderField instances are
* found that have an ancestery relationships.
*
* @see InsanityType#SUBREADER
*/
private Collection checkSubreaders(MapOfSets valIdToItems,
MapOfSets readerFieldToValIds) {
final List insanity = new ArrayList(23);
Map badChildren = new HashMap(17);
MapOfSets badKids = new MapOfSets(badChildren); // wrapper
Map viToItemSets = valIdToItems.getMap();
Map rfToValIdSets = readerFieldToValIds.getMap();
Set seen = new HashSet(17);
Set readerFields = rfToValIdSets.keySet();
Iterator rfIter = readerFields.iterator();
while (rfIter.hasNext()) {
ReaderField rf = (ReaderField) rfIter.next();
if (seen.contains(rf)) continue;
List kids = getAllDecendentReaderKeys(rf.readerKey);
for (int i = 0; i < kids.size(); i++) {
ReaderField kid = new ReaderField(kids.get(i), rf.fieldName);
if (badChildren.containsKey(kid)) {
// we've already process this kid as RF and found other problems
// track those problems as our own
badKids.put(rf, kid);
badKids.putAll(rf, (Collection)badChildren.get(kid));
badChildren.remove(kid);
} else if (rfToValIdSets.containsKey(kid)) {
// we have cache entries for the kid
badKids.put(rf, kid);
}
seen.add(kid);
}
seen.add(rf);
}
// every mapping in badKids represents an Insanity
Iterator parentsIter = badChildren.keySet().iterator();
while (parentsIter.hasNext()) {
ReaderField parent = (ReaderField) parentsIter.next();
Set kids = (Set) badChildren.get(parent);
List badEntries = new ArrayList(kids.size() * 2);
// put parent entr(ies) in first
{
Iterator valIter =((Set)rfToValIdSets.get(parent)).iterator();
while (valIter.hasNext()) {
badEntries.addAll((Set)viToItemSets.get(valIter.next()));
}
}
// now the entries for the descendants
Iterator kidsIter = kids.iterator();
while (kidsIter.hasNext()) {
ReaderField kid = (ReaderField) kidsIter.next();
Iterator valIter =((Set)rfToValIdSets.get(kid)).iterator();
while (valIter.hasNext()) {
badEntries.addAll((Set)viToItemSets.get(valIter.next()));
}
}
CacheEntry[] badness = new CacheEntry[badEntries.size()];
badness = (CacheEntry[]) badEntries.toArray(badness);
insanity.add(new Insanity(InsanityType.SUBREADER,
"Found caches for decendents of " +
parent.toString(),
badness));
}
return insanity;
}
/**
* Checks if the seed is an IndexReader, and if so will walk
* the hierarchy of subReaders building up a list of the objects
* returned by obj.getFieldCacheKey()
*/
private List getAllDecendentReaderKeys(Object seed) {
List all = new ArrayList(17); // will grow as we iter
all.add(seed);
for (int i = 0; i < all.size(); i++) {
Object obj = all.get(i);
if (obj instanceof IndexReader) {
IndexReader[] subs = ((IndexReader)obj).getSequentialSubReaders();
for (int j = 0; (null != subs) && (j < subs.length); j++) {
all.add(subs[j].getFieldCacheKey());
}
}
}
// need to skip the first, because it was the seed
return all.subList(1, all.size());
}
/**
* Simple pair object for using "readerKey + fieldName" a Map key
*/
private final static class ReaderField {
public final Object readerKey;
public final String fieldName;
public ReaderField(Object readerKey, String fieldName) {
this.readerKey = readerKey;
this.fieldName = fieldName;
}
public int hashCode() {
return System.identityHashCode(readerKey) * fieldName.hashCode();
}
public boolean equals(Object that) {
if (! (that instanceof ReaderField)) return false;
ReaderField other = (ReaderField) that;
return (this.readerKey == other.readerKey &&
this.fieldName.equals(other.fieldName));
}
public String toString() {
return readerKey.toString() + "+" + fieldName;
}
}
/**
* Simple container for a collection of related CacheEntry objects that
* in conjunction with eachother represent some "insane" usage of the
* FieldCache.
*/
public final static class Insanity {
private final InsanityType type;
private final String msg;
private final CacheEntry[] entries;
public Insanity(InsanityType type, String msg, CacheEntry[] entries) {
if (null == type) {
throw new IllegalArgumentException
("Insanity requires non-null InsanityType");
}
if (null == entries || 0 == entries.length) {
throw new IllegalArgumentException
("Insanity requires non-null/non-empty CacheEntry[]");
}
this.type = type;
this.msg = msg;
this.entries = entries;
}
/**
* Type of insane behavior this object represents
*/
public InsanityType getType() { return type; }
/**
* Description of hte insane behavior
*/
public String getMsg() { return msg; }
/**
* CacheEntry objects which suggest a problem
*/
public CacheEntry[] getCacheEntries() { return entries; }
/**
* Multi-Line representation of this Insanity object, starting with
* the Type and Msg, followed by each CacheEntry.toString() on it's
* own line prefaced by a tab character
*/
public String toString() {
StringBuffer buf = new StringBuffer();
buf.append(getType()).append(": ");
String m = getMsg();
if (null != m) buf.append(m);
buf.append('\n');
CacheEntry[] ce = getCacheEntries();
for (int i = 0; i < ce.length; i++) {
buf.append('\t').append(ce[i].toString()).append('\n');
}
return buf.toString();
}
}
/**
* An Enumaration of the differnet types of "insane" behavior that
* may be detected in a FieldCache.
*
* @see InsanityType#SUBREADER
* @see InsanityType#VALUEMISMATCH
* @see InsanityType#EXPECTED
*/
public final static class InsanityType {
private final String label;
private InsanityType(final String label) {
this.label = label;
}
public String toString() { return label; }
/**
* Indicates an overlap in cache usage on a given field
* in sub/super readers.
*/
public final static InsanityType SUBREADER
= new InsanityType("SUBREADER");
/**
* <p>
* Indicates entries have the same reader+fieldname but
* different cached values. This can happen if different datatypes,
* or parsers are used -- and while it's not necessarily a bug
* it's typically an indication of a possible problem.
* </p>
* <p>
* <bPNOTE:</b> Only the reader, fieldname, and cached value are actually
* tested -- if two cache entries have different parsers or datatypes but
* the cached values are the same Object (== not just equal()) this method
* does not consider that a red flag. This allows for subtle variations
* in the way a Parser is specified (null vs DEFAULT_LONG_PARSER, etc...)
* </p>
*/
public final static InsanityType VALUEMISMATCH
= new InsanityType("VALUEMISMATCH");
/**
* Indicates an expected bit of "insanity". This may be useful for
* clients that wish to preserve/log information about insane usage
* but indicate that it was expected.
*/
public final static InsanityType EXPECTED
= new InsanityType("EXPECTED");
}
}

81
src/java/org/apache/lucene/util/MapOfSets.java Normal file
View File

@ -0,0 +1,81 @@
package org.apache.lucene.util;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.Set;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
/**
* Helper class for keeping Listss of Objects associated with keys. <b>WARNING: THIS CLASS IS NOT THREAD SAFE</b>
*/
public class MapOfSets {
private final Map theMap;
/**
* @param m the backing store for this object
*/
public MapOfSets(Map m) {
theMap = m;
}
/**
* @return direct access to the map backing this object.
*/
public Map getMap() {
return theMap;
}
/**
* Adds val to the Set associated with key in the Map. If key is not
* already in the map, a new Set will first be created.
* @return the size of the Set associated with key once val is added to it.
*/
public int put(Object key, Object val) {
final Set theSet;
if (theMap.containsKey(key)) {
theSet = (Set)theMap.get(key);
} else {
theSet = new HashSet(23);
theMap.put(key, theSet);
}
theSet.add(val);
return theSet.size();
}
/**
* Adds multiple vals to the Set associated with key in the Map.
* If key is not
* already in the map, a new Set will first be created.
* @return the size of the Set associated with key once val is added to it.
*/
public int putAll(Object key, Collection vals) {
final Set theSet;
if (theMap.containsKey(key)) {
theSet = (Set)theMap.get(key);
} else {
theSet = new HashSet(23);
theMap.put(key, theSet);
}
theSet.addAll(vals);
return theSet.size();
}
}

48
src/java/org/apache/lucene/util/MemoryModel.java Normal file
View File

@ -0,0 +1,48 @@
package org.apache.lucene.util;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
/**
* Returns primitive memory sizes for estimating RAM usage.
*
*/
public abstract class MemoryModel {
/**
* @return size of array beyond contents
*/
public abstract int getArraySize();
/**
* @return Class size overhead
*/
public abstract int getClassSize();
/**
* @param clazz a primitive Class - bool, byte, char, short, long, float,
* short, double, int
* @return the size in bytes of given primitive Class
*/
public abstract int getPrimitiveSize(Class clazz);
/**
* @return size of reference
*/
public abstract int getReferenceSize();
}

197
src/java/org/apache/lucene/util/RamUsageEstimator.java Normal file
View File

@ -0,0 +1,197 @@
package org.apache.lucene.util;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.lang.reflect.*;
import java.text.DecimalFormat;
import java.util.*;
/**
* Estimates the size of a given Object using a given MemoryModel for primitive
* size information.
*
* Resource Usage:
*
* Internally uses a Map to temporally hold a reference to every
* object seen.
*
* If checkIntered, all Strings checked will be interned, but those
* that were not already interned will be released for GC when the
* estimate is complete.
*/
public final class RamUsageEstimator {
private MemoryModel memoryModel;
private final Map seen;
private int refSize;
private int arraySize;
private int classSize;
private boolean checkInterned;
/**
* Constructs this object with an AverageGuessMemoryModel and
* checkInterned = true.
*/
public RamUsageEstimator() {
this(new AverageGuessMemoryModel());
}
/**
* @param checkInterned check if Strings are interned and don't add to size
* if they are. Defaults to true but if you know the objects you are checking
* won't likely contain many interned Strings, it will be faster to turn off
* intern checking.
*/
public RamUsageEstimator(boolean checkInterned) {
this(new AverageGuessMemoryModel(), checkInterned);
}
/**
* @param memoryModel MemoryModel to use for primitive object sizes.
*/
public RamUsageEstimator(MemoryModel memoryModel) {
this(memoryModel, true);
}
/**
* @param memoryModel MemoryModel to use for primitive object sizes.
* @param checkInterned check if Strings are interned and don't add to size
* if they are. Defaults to true but if you know the objects you are checking
* won't likely contain many interned Strings, it will be faster to turn off
* intern checking.
*/
public RamUsageEstimator(MemoryModel memoryModel, boolean checkInterned) {
this.memoryModel = memoryModel;
this.checkInterned = checkInterned;
// Use Map rather than Set so that we can use an IdentityHashMap - not
// seeing an IdentityHashSet
seen = new IdentityHashMap(64);
this.refSize = memoryModel.getReferenceSize();
this.arraySize = memoryModel.getArraySize();
this.classSize = memoryModel.getClassSize();
}
public long estimateRamUsage(Object obj) {
long size = size(obj);
seen.clear();
return size;
}
private long size(Object obj) {
if (obj == null) {
return 0;
}
// interned not part of this object
if (checkInterned && obj instanceof String
&& obj == ((String) obj).intern()) { // interned string will be eligible
// for GC on
// estimateRamUsage(Object) return
return 0;
}
// skip if we have seen before
if (seen.containsKey(obj)) {
return 0;
}
// add to seen
seen.put(obj, null);
Class clazz = obj.getClass();
if (clazz.isArray()) {
return sizeOfArray(obj);
}
long size = 0;
// walk type hierarchy
while (clazz != null) {
Field[] fields = clazz.getDeclaredFields();
for (int i = 0; i < fields.length; i++) {
if (Modifier.isStatic(fields[i].getModifiers())) {
continue;
}
if (fields[i].getType().isPrimitive()) {
size += memoryModel.getPrimitiveSize(fields[i].getType());
} else {
size += refSize;
fields[i].setAccessible(true);
try {
Object value = fields[i].get(obj);
if (value != null) {
size += size(value);
}
} catch (IllegalAccessException ex) {
// ignore for now?
}
}
}
clazz = clazz.getSuperclass();
}
size += classSize;
return size;
}
private long sizeOfArray(Object obj) {
int len = Array.getLength(obj);
if (len == 0) {
return 0;
}
long size = arraySize;
Class arrayElementClazz = obj.getClass().getComponentType();
if (arrayElementClazz.isPrimitive()) {
size += len * memoryModel.getPrimitiveSize(arrayElementClazz);
} else {
for (int i = 0; i < len; i++) {
size += refSize + size(Array.get(obj, i));
}
}
return size;
}
private static final long ONE_KB = 1024;
private static final long ONE_MB = ONE_KB * ONE_KB;
private static final long ONE_GB = ONE_KB * ONE_MB;
/**
* Return good default units based on byte size.
*/
public static String humanReadableUnits(long bytes, DecimalFormat df) {
String newSizeAndUnits;
if (bytes / ONE_GB > 0) {
newSizeAndUnits = String.valueOf(df.format((float) bytes / ONE_GB))
+ " GB";
} else if (bytes / ONE_MB > 0) {
newSizeAndUnits = String.valueOf(df.format((float) bytes / ONE_MB))
+ " MB";
} else if (bytes / ONE_KB > 0) {
newSizeAndUnits = String.valueOf(df.format((float) bytes / ONE_KB))
+ " KB";
} else {
newSizeAndUnits = String.valueOf(bytes) + " bytes";
}
return newSizeAndUnits;
}
}

168
src/test/org/apache/lucene/search/QueryUtils.java
View File

@ -146,74 +146,89 @@ public class QueryUtils {
{skip_op, skip_op, skip_op, next_op, next_op},
};
for (int k = 0; k < orders.length; k++) {
final int order[] = orders[k];
//System.out.print("Order:");for (int i = 0; i < order.length; i++) System.out.print(order[i]==skip_op ? " skip()":" next()"); System.out.println();
final int opidx[] = {0};
IndexReader[] readers = s.getIndexReader().getSequentialSubReaders();
final Weight w = q.weight(s);
final Scorer scorer = w.scorer(s.getIndexReader(), true, false);
if (scorer == null) {
continue;
}
for (int x = 0; x < readers.length; x++) {
IndexReader reader = readers[x];
// FUTURE: ensure scorer.doc()==-1
final int order[] = orders[k];
// System.out.print("Order:");for (int i = 0; i < order.length; i++)
// System.out.print(order[i]==skip_op ? " skip()":" next()");
// System.out.println();
final int opidx[] = { 0 };
final int[] sdoc = new int[] {-1};
final float maxDiff = 1e-5f;
s.search(q,new Collector() {
private int base = 0;
private Scorer sc;
public void setScorer(Scorer scorer) throws IOException {
this.sc = scorer;
final Weight w = q.weight(s);
final Scorer scorer = w.scorer(reader, true, false);
if (scorer == null) {
continue;
}
public void collect(int doc) throws IOException {
doc = doc + base;
float score = sc.score();
try {
int op = order[(opidx[0]++)%order.length];
//System.out.println(op==skip_op ? "skip("+(sdoc[0]+1)+")":"next()");
boolean more = op == skip_op ? scorer.advance(sdoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS
: scorer.nextDoc() != DocIdSetIterator.NO_MORE_DOCS;
sdoc[0] = scorer.docID();
float scorerScore = scorer.score();
float scorerScore2 = scorer.score();
float scoreDiff = Math.abs(score-scorerScore);
float scorerDiff = Math.abs(scorerScore2-scorerScore);
if (!more || doc != sdoc[0] || scoreDiff>maxDiff || scorerDiff>maxDiff) {
StringBuffer sbord = new StringBuffer();
for (int i = 0; i < order.length; i++)
sbord.append(order[i]==skip_op ? " skip()":" next()");
throw new RuntimeException("ERROR matching docs:"
+"\n\t"+(doc!=sdoc[0]?"--> ":"")+"doc="+sdoc[0]
+"\n\t"+(!more?"--> ":"")+"tscorer.more=" + more
+"\n\t"+(scoreDiff>maxDiff?"--> ":"")+"scorerScore="+scorerScore+" scoreDiff="+scoreDiff + " maxDiff="+maxDiff
+"\n\t"+(scorerDiff>maxDiff?"--> ":"")+"scorerScore2="+scorerScore2+" scorerDiff="+scorerDiff
+"\n\thitCollector.doc=" + doc + " score="+score
+"\n\t Scorer=" + scorer
+"\n\t Query=" + q + " "+q.getClass().getName()
+"\n\t Searcher=" + s
+"\n\t Order=" + sbord
+"\n\t Op=" + (op==skip_op ? " skip()":" next()")
);
}
} catch (IOException e) {
throw new RuntimeException(e);
// FUTURE: ensure scorer.doc()==-1
final int[] sdoc = new int[] { -1 };
final float maxDiff = 1e-5f;
s.search(q, new Collector() {
private int base = 0;
private Scorer sc;
public void setScorer(Scorer scorer) throws IOException {
this.sc = scorer;
}
}
public void setNextReader(IndexReader reader, int docBase) {
base = docBase;
}
public boolean acceptsDocsOutOfOrder() {
return true;
}
});
// make sure next call to scorer is false.
int op = order[(opidx[0]++)%order.length];
//System.out.println(op==skip_op ? "last: skip()":"last: next()");
boolean more = (op == skip_op ? scorer.advance(sdoc[0] + 1) : scorer
.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS;
Assert.assertFalse(more);
public void collect(int doc) throws IOException {
doc = doc + base;
float score = sc.score();
try {
int op = order[(opidx[0]++) % order.length];
// System.out.println(op==skip_op ?
// "skip("+(sdoc[0]+1)+")":"next()");
boolean more = op == skip_op ? scorer.advance(sdoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS
: scorer.nextDoc() != DocIdSetIterator.NO_MORE_DOCS;
sdoc[0] = scorer.docID();
float scorerScore = scorer.score();
float scorerScore2 = scorer.score();
float scoreDiff = Math.abs(score - scorerScore);
float scorerDiff = Math.abs(scorerScore2 - scorerScore);
if (!more || doc != sdoc[0] || scoreDiff > maxDiff
|| scorerDiff > maxDiff) {
StringBuffer sbord = new StringBuffer();
for (int i = 0; i < order.length; i++)
sbord.append(order[i] == skip_op ? " skip()" : " next()");
throw new RuntimeException("ERROR matching docs:" + "\n\t"
+ (doc != sdoc[0] ? "--> " : "") + "doc=" + sdoc[0]
+ "\n\t" + (!more ? "--> " : "") + "tscorer.more=" + more
+ "\n\t" + (scoreDiff > maxDiff ? "--> " : "")
+ "scorerScore=" + scorerScore + " scoreDiff=" + scoreDiff
+ " maxDiff=" + maxDiff + "\n\t"
+ (scorerDiff > maxDiff ? "--> " : "") + "scorerScore2="
+ scorerScore2 + " scorerDiff=" + scorerDiff
+ "\n\thitCollector.doc=" + doc + " score=" + score
+ "\n\t Scorer=" + scorer + "\n\t Query=" + q + " "
+ q.getClass().getName() + "\n\t Searcher=" + s
+ "\n\t Order=" + sbord + "\n\t Op="
+ (op == skip_op ? " skip()" : " next()"));
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public void setNextReader(IndexReader reader, int docBase) {
base = docBase;
}
public boolean acceptsDocsOutOfOrder() {
return true;
}
});
// make sure next call to scorer is false.
int op = order[(opidx[0]++) % order.length];
// System.out.println(op==skip_op ? "last: skip()":"last: next()");
boolean more = (op == skip_op ? scorer.advance(sdoc[0] + 1) : scorer
.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS;
Assert.assertFalse(more);
}
}
}
@ -223,19 +238,19 @@ public class QueryUtils {
final float maxDiff = 1e-5f;
final int lastDoc[] = {-1};
s.search(q,new Collector() {
private int base = 0;
private Scorer scorer;
private IndexReader reader;
public void setScorer(Scorer scorer) throws IOException {
this.scorer = scorer;
}
public void collect(int doc) throws IOException {
//System.out.println("doc="+doc);
doc = doc + base;
float score = scorer.score();
try {
for (int i=lastDoc[0]+1; i<=doc; i++) {
Weight w = q.weight(s);
Scorer scorer = w.scorer(s.getIndexReader(), true, false);
Scorer scorer = w.scorer(reader, true, false);
Assert.assertTrue("query collected "+doc+" but skipTo("+i+") says no more docs!",scorer.advance(i) != DocIdSetIterator.NO_MORE_DOCS);
Assert.assertEquals("query collected "+doc+" but skipTo("+i+") got to "+scorer.docID(),doc,scorer.docID());
float skipToScore = scorer.score();
@ -248,18 +263,27 @@ public class QueryUtils {
}
}
public void setNextReader(IndexReader reader, int docBase) {
base = docBase;
this.reader = reader;
lastDoc[0] = -1;
}
public boolean acceptsDocsOutOfOrder() {
return false;
}
});
Weight w = q.weight(s);
Scorer scorer = w.scorer(s.getIndexReader(), true, false);
if (scorer != null) {
boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
if (more)
Assert.assertFalse("query's last doc was "+lastDoc[0]+" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more);
IndexReader[] readers = s.getIndexReader().getSequentialSubReaders();
for(int i = 0; i < readers.length; i++) {
IndexReader reader = readers[i];
Weight w = q.weight(s);
Scorer scorer = w.scorer(reader, true, false);
if (scorer != null) {
boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
if (more && lastDoc[0] != -1)
Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more);
}
}
}
}

51
src/test/org/apache/lucene/search/TestSort.java
View File

@ -320,16 +320,26 @@ public class TestSort extends LuceneTestCase implements Serializable {
}
// test sorts where the type of field is specified and a custom field parser is used, that
// uses a simple char encoding. The sorted string contains a character beginning from 'A' that
// is mapped to a numeric value using some "funny" algorithm to be different for each data type.
/**
* test sorts where the type of field is specified and a custom field parser
* is used, that uses a simple char encoding. The sorted string contains a
* character beginning from 'A' that is mapped to a numeric value using some
* "funny" algorithm to be different for each data type.
*/
public void testCustomFieldParserSort() throws Exception {
// since tests explicilty uses different parsers on the same fieldname
// we explicitly check/purge the FieldCache between each assertMatch
FieldCache fc = FieldCache.DEFAULT;
sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.IntParser(){
public final int parseInt(final String val) {
return (val.charAt(0)-'A') * 123456;
}
}), SortField.FIELD_DOC });
assertMatches (full, queryA, sort, "JIHGFEDCBA");
assertSaneFieldCaches(getName() + " IntParser");
fc.purgeAllCaches();
sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.FloatParser(){
public final float parseFloat(final String val) {
@ -337,6 +347,8 @@ public class TestSort extends LuceneTestCase implements Serializable {
}
}), SortField.FIELD_DOC });
assertMatches (full, queryA, sort, "JIHGFEDCBA");
assertSaneFieldCaches(getName() + " FloatParser");
fc.purgeAllCaches();
sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.LongParser(){
public final long parseLong(final String val) {
@ -344,6 +356,8 @@ public class TestSort extends LuceneTestCase implements Serializable {
}
}), SortField.FIELD_DOC });
assertMatches (full, queryA, sort, "JIHGFEDCBA");
assertSaneFieldCaches(getName() + " LongParser");
fc.purgeAllCaches();
sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.DoubleParser(){
public final double parseDouble(final String val) {
@ -351,6 +365,8 @@ public class TestSort extends LuceneTestCase implements Serializable {
}
}), SortField.FIELD_DOC });
assertMatches (full, queryA, sort, "JIHGFEDCBA");
assertSaneFieldCaches(getName() + " DoubleParser");
fc.purgeAllCaches();
sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.ByteParser(){
public final byte parseByte(final String val) {
@ -358,6 +374,8 @@ public class TestSort extends LuceneTestCase implements Serializable {
}
}), SortField.FIELD_DOC });
assertMatches (full, queryA, sort, "JIHGFEDCBA");
assertSaneFieldCaches(getName() + " ByteParser");
fc.purgeAllCaches();
sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.ShortParser(){
public final short parseShort(final String val) {
@ -365,6 +383,8 @@ public class TestSort extends LuceneTestCase implements Serializable {
}
}), SortField.FIELD_DOC });
assertMatches (full, queryA, sort, "JIHGFEDCBA");
assertSaneFieldCaches(getName() + " ShortParser");
fc.purgeAllCaches();
}
// test sorts when there's nothing in the index
@ -930,12 +950,6 @@ public class TestSort extends LuceneTestCase implements Serializable {
sort.setSort("string", true);
assertMatches(multi, queryA, sort, "CBEFGHIAJD");
sort.setSort(new SortField[] { new SortField ("string", Locale.US) });
assertMatches(multi, queryA, sort, "DJAIHGFEBC");
sort.setSort(new SortField[] { new SortField ("string", Locale.US, true) });
assertMatches(multi, queryA, sort, "CBEFGHIAJD");
sort.setSort(new String[] {"int","float"});
assertMatches(multi, queryA, sort, "IDHFGJEABC");
@ -956,6 +970,25 @@ public class TestSort extends LuceneTestCase implements Serializable {
sort.setSort("string", true);
assertMatches(multi, queryF, sort, "IJZ");
// up to this point, all of the searches should have "sane"
// FieldCache behavior, and should have reused hte cache in several cases
assertSaneFieldCaches(getName() + " various");
// next we'll check Locale based (String[]) for 'string', so purge first
FieldCache.DEFAULT.purgeAllCaches();
sort.setSort(new SortField[] { new SortField ("string", Locale.US) });
assertMatches(multi, queryA, sort, "DJAIHGFEBC");
sort.setSort(new SortField[] { new SortField ("string", Locale.US, true) });
assertMatches(multi, queryA, sort, "CBEFGHIAJD");
sort.setSort(new SortField[] { new SortField ("string", Locale.UK) });
assertMatches(multi, queryA, sort, "DJAIHGFEBC");
assertSaneFieldCaches(getName() + " Locale.US + Locale.UK");
FieldCache.DEFAULT.purgeAllCaches();
}
// make sure the documents returned by the search match the expected list

44
src/test/org/apache/lucene/search/TestStressSort.java
View File

@ -26,6 +26,9 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.FieldCacheSanityChecker;
import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
import org.apache.lucene.util.FieldCacheSanityChecker.InsanityType;
import java.util.Random;
import java.util.Arrays;
@ -107,9 +110,13 @@ public class TestStressSort extends LuceneTestCase {
doc.add(doubleField);
doc2.add(doubleField);
// we use two diff string fields so our FieldCache usage
// is less suspicious to cache inspection
final Field stringField = new Field("string", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
doc.add(stringField);
// doc2 doesn't have stringField, so we get nulls
final Field stringFieldIdx = new Field("stringIdx", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
doc.add(stringFieldIdx);
// doc2 doesn't have stringField or stringFieldIdx, so we get nulls
for(int i=0;i<NUM_DOCS;i++) {
id.setValue(""+i);
@ -151,7 +158,9 @@ public class TestStressSort extends LuceneTestCase {
if (i % 197 == 0) {
writer.addDocument(doc2);
} else {
stringField.setValue(randomString(nextInt(20)));
String r = randomString(nextInt(20));
stringField.setValue(r);
stringFieldIdx.setValue(r);
writer.addDocument(doc);
}
}
@ -219,7 +228,7 @@ public class TestStressSort extends LuceneTestCase {
sort.setSort(new SortField[] {new SortField("string", SortField.STRING_VAL, reverse)});
sorts[sortCount++] = sort = new Sort();
sort.setSort(new SortField[] {new SortField("string", SortField.STRING, reverse)});
sort.setSort(new SortField[] {new SortField("stringIdx", SortField.STRING, reverse)});
//sorts[sortCount++] = sort = new Sort();
//sort.setSort(new SortField[] {new SortField("string", SortField.STRING_ORD, reverse)});
@ -309,6 +318,35 @@ public class TestStressSort extends LuceneTestCase {
}
}
// we explicitly test the old sort method and
// compare with the new, so we expect to see SUBREADER
// sanity checks fail.
Insanity[] insanity = FieldCacheSanityChecker.checkSanity
(FieldCache.DEFAULT);
try {
int ignored = 0;
for (int i = 0; i < insanity.length; i++) {
if (insanity[i].getType() == InsanityType.SUBREADER) {
insanity[i] = new Insanity(InsanityType.EXPECTED,
insanity[i].getMsg(),
insanity[i].getCacheEntries());
ignored++;
}
}
assertEquals("Not all insane field cache usage was expected",
ignored, insanity.length);
insanity = null;
} finally {
// report this in the event of any exception/failure
// if no failure, then insanity will be null
if (null != insanity) {
dumpArray(getTestLabel() + ": Insane FieldCache usage(s)", insanity, System.err);
}
}
// we've already checked FieldCache, purge so tearDown doesn't complain
purgeFieldCache(FieldCache.DEFAULT); // so
close();
}

59
src/test/org/apache/lucene/search/function/TestFieldScoreQuery.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.search.function;
import java.util.HashMap;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryUtils;
@ -170,19 +171,29 @@ public class TestFieldScoreQuery extends FunctionTestSetup {
FieldScoreQuery q = new FieldScoreQuery(field,tp);
ScoreDoc[] h = s.search(q, null, 1000).scoreDocs;
assertEquals("All docs should be matched!",N_DOCS,h.length);
try {
if (i==0) {
innerArray = q.valSrc.getValues(s.getIndexReader()).getInnerArray();
log(i+". compare: "+innerArray.getClass()+" to "+expectedArrayTypes.get(tp).getClass());
assertEquals("field values should be cached in the correct array type!", innerArray.getClass(),expectedArrayTypes.get(tp).getClass());
} else {
log(i+". compare: "+innerArray+" to "+q.valSrc.getValues(s.getIndexReader()).getInnerArray());
assertSame("field values should be cached and reused!", innerArray, q.valSrc.getValues(s.getIndexReader()).getInnerArray());
}
} catch (UnsupportedOperationException e) {
if (!warned) {
System.err.println("WARNING: "+testName()+" cannot fully test values of "+q);
warned = true;
IndexReader[] readers = s.getIndexReader().getSequentialSubReaders();
for (int j = 0; j < readers.length; j++) {
IndexReader reader = readers[j];
try {
if (i == 0) {
innerArray = q.valSrc.getValues(reader).getInnerArray();
log(i + ". compare: " + innerArray.getClass() + " to "
+ expectedArrayTypes.get(tp).getClass());
assertEquals(
"field values should be cached in the correct array type!",
innerArray.getClass(), expectedArrayTypes.get(tp).getClass());
} else {
log(i + ". compare: " + innerArray + " to "
+ q.valSrc.getValues(reader).getInnerArray());
assertSame("field values should be cached and reused!", innerArray,
q.valSrc.getValues(reader).getInnerArray());
}
} catch (UnsupportedOperationException e) {
if (!warned) {
System.err.println("WARNING: " + testName()
+ " cannot fully test values of " + q);
warned = true;
}
}
}
}
@ -192,13 +203,21 @@ public class TestFieldScoreQuery extends FunctionTestSetup {
FieldScoreQuery q = new FieldScoreQuery(field,tp);
ScoreDoc[] h = s.search(q, null, 1000).scoreDocs;
assertEquals("All docs should be matched!",N_DOCS,h.length);
try {
log("compare: "+innerArray+" to "+q.valSrc.getValues(s.getIndexReader()).getInnerArray());
assertNotSame("cached field values should not be reused if reader as changed!", innerArray, q.valSrc.getValues(s.getIndexReader()).getInnerArray());
} catch (UnsupportedOperationException e) {
if (!warned) {
System.err.println("WARNING: "+testName()+" cannot fully test values of "+q);
warned = true;
IndexReader[] readers = s.getIndexReader().getSequentialSubReaders();
for (int j = 0; j < readers.length; j++) {
IndexReader reader = readers[j];
try {
log("compare: " + innerArray + " to "
+ q.valSrc.getValues(reader).getInnerArray());
assertNotSame(
"cached field values should not be reused if reader as changed!",
innerArray, q.valSrc.getValues(reader).getInnerArray());
} catch (UnsupportedOperationException e) {
if (!warned) {
System.err.println("WARNING: " + testName()
+ " cannot fully test values of " + q);
warned = true;
}
}
}
}

62
src/test/org/apache/lucene/search/function/TestOrdValues.java
View File

@ -18,6 +18,7 @@ package org.apache.lucene.search.function;
*/
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryUtils;
@ -150,11 +151,16 @@ public class TestOrdValues extends FunctionTestSetup {
ScoreDoc[] h = s.search(q, null, 1000).scoreDocs;
try {
assertEquals("All docs should be matched!",N_DOCS,h.length);
if (i==0) {
innerArray = q.valSrc.getValues(s.getIndexReader()).getInnerArray();
} else {
log(i+". compare: "+innerArray+" to "+q.valSrc.getValues(s.getIndexReader()).getInnerArray());
assertSame("field values should be cached and reused!", innerArray, q.valSrc.getValues(s.getIndexReader()).getInnerArray());
IndexReader[] readers = s.getIndexReader().getSequentialSubReaders();
for(int j = 0; j < readers.length; j++) {
IndexReader reader = readers[j];
if (i==0) {
innerArray = q.valSrc.getValues(reader).getInnerArray();
} else {
log(i+". compare: "+innerArray+" to "+q.valSrc.getValues(reader).getInnerArray());
assertSame("field values should be cached and reused!", innerArray, q.valSrc.getValues(reader).getInnerArray());
}
}
} catch (UnsupportedOperationException e) {
if (!warned) {
@ -179,13 +185,22 @@ public class TestOrdValues extends FunctionTestSetup {
q = new ValueSourceQuery(vs);
h = s.search(q, null, 1000).scoreDocs;
assertEquals("All docs should be matched!",N_DOCS,h.length);
try {
log("compare (should differ): "+innerArray+" to "+q.valSrc.getValues(s.getIndexReader()).getInnerArray());
assertNotSame("different values shuold be loaded for a different field!", innerArray, q.valSrc.getValues(s.getIndexReader()).getInnerArray());
} catch (UnsupportedOperationException e) {
if (!warned) {
System.err.println("WARNING: "+testName()+" cannot fully test values of "+q);
warned = true;
IndexReader[] readers = s.getIndexReader().getSequentialSubReaders();
for (int j = 0; j < readers.length; j++) {
IndexReader reader = readers[j];
try {
log("compare (should differ): " + innerArray + " to "
+ q.valSrc.getValues(reader).getInnerArray());
assertNotSame(
"different values shuold be loaded for a different field!",
innerArray, q.valSrc.getValues(reader).getInnerArray());
} catch (UnsupportedOperationException e) {
if (!warned) {
System.err.println("WARNING: " + testName()
+ " cannot fully test values of " + q);
warned = true;
}
}
}
@ -199,13 +214,22 @@ public class TestOrdValues extends FunctionTestSetup {
q = new ValueSourceQuery(vs);
h = s.search(q, null, 1000).scoreDocs;
assertEquals("All docs should be matched!",N_DOCS,h.length);
try {
log("compare (should differ): "+innerArray+" to "+q.valSrc.getValues(s.getIndexReader()).getInnerArray());
assertNotSame("cached field values should not be reused if reader as changed!", innerArray, q.valSrc.getValues(s.getIndexReader()).getInnerArray());
} catch (UnsupportedOperationException e) {
if (!warned) {
System.err.println("WARNING: "+testName()+" cannot fully test values of "+q);
warned = true;
readers = s.getIndexReader().getSequentialSubReaders();
for (int j = 0; j < readers.length; j++) {
IndexReader reader = readers[j];
try {
log("compare (should differ): " + innerArray + " to "
+ q.valSrc.getValues(reader).getInnerArray());
assertNotSame(
"cached field values should not be reused if reader as changed!",
innerArray, q.valSrc.getValues(reader).getInnerArray());
} catch (UnsupportedOperationException e) {
if (!warned) {
System.err.println("WARNING: " + testName()
+ " cannot fully test values of " + q);
warned = true;
}
}
}
}

148
src/test/org/apache/lucene/util/LuceneTestCase.java
View File

@ -17,21 +17,36 @@ package org.apache.lucene.util;
* limitations under the License.
*/
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import junit.framework.TestCase;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Random;
/** Base class for all Lucene unit tests. Currently the
* only added functionality over JUnit's TestCase is
* asserting that no unhandled exceptions occurred in
* threads launched by ConcurrentMergeScheduler. If you
* override either <code>setUp()</code> or
* <code>tearDown()</code> in your unit test, make sure you
* call <code>super.setUp()</code> and
* <code>super.tearDown()</code>.
*/
import junit.framework.TestCase;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldCache.CacheEntry;
import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
/**
* Base class for all Lucene unit tests.
* <p>
* Currently the
* only added functionality over JUnit's TestCase is
* asserting that no unhandled exceptions occurred in
* threads launched by ConcurrentMergeScheduler and asserting sane
* FieldCache usage athe moment of tearDown.
* </p>
* <p>
* If you
* override either <code>setUp()</code> or
* <code>tearDown()</code> in your unit test, make sure you
* call <code>super.setUp()</code> and
* <code>super.tearDown()</code>
* </p>
* @see #assertSaneFieldCaches
*/
public abstract class LuceneTestCase extends TestCase {
public LuceneTestCase() {
@ -46,15 +61,114 @@ public abstract class LuceneTestCase extends TestCase {
ConcurrentMergeScheduler.setTestMode();
}
/**
* Forcible purges all cache entries from the FieldCache.
* <p>
* This method will be called by tearDown to clean up FieldCache.DEFAULT.
* If a (poorly written) test has some expectation that the FieldCache
* will persist across test methods (ie: a static IndexReader) this
* method can be overridden to do nothing.
* </p>
* @see FieldCache#purgeAllCaches()
*/
protected void purgeFieldCache(final FieldCache fc) {
fc.purgeAllCaches();
}
protected String getTestLabel() {
return getClass().getName() + "." + getName();
}
protected void tearDown() throws Exception {
if (ConcurrentMergeScheduler.anyUnhandledExceptions()) {
// Clear the failure so that we don't just keep
// failing subsequent test cases
ConcurrentMergeScheduler.clearUnhandledExceptions();
fail("ConcurrentMergeScheduler hit unhandled exceptions");
try {
// this isn't as useful as calling directly from the scope where the
// index readers are used, because they could be gc'ed just before
// tearDown is called.
// But it's better then nothing.
assertSaneFieldCaches(getTestLabel());
if (ConcurrentMergeScheduler.anyUnhandledExceptions()) {
// Clear the failure so that we don't just keep
// failing subsequent test cases
ConcurrentMergeScheduler.clearUnhandledExceptions();
fail("ConcurrentMergeScheduler hit unhandled exceptions");
}
} finally {
purgeFieldCache(FieldCache.DEFAULT);
}
}
/**
* Asserts that FieldCacheSanityChecker does not detect any
* problems with FieldCache.DEFAULT.
* <p>
* If any problems are found, they are logged to System.err
* (allong with the msg) when the Assertion is thrown.
* </p>
* <p>
* This method is called by tearDown after every test method,
* however IndexReaders scoped inside test methods may be garbage
* collected prior to this method being called, causing errors to
* be overlooked. Tests are encouraged to keep their IndexReaders
* scoped at the class level, or to explicitly call this method
* directly in the same scope as the IndexReader.
* </p>
* @see FieldCacheSanityChecker
*/
protected void assertSaneFieldCaches(final String msg) {
final CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries();
Insanity[] insanity = null;
try {
try {
insanity = FieldCacheSanityChecker.checkSanity(entries);
} catch (RuntimeException e) {
dumpArray(msg+ ": FieldCache", entries, System.err);
throw e;
}
assertEquals(msg + ": Insane FieldCache usage(s) found",
0, insanity.length);
insanity = null;
} finally {
// report this in the event of any exception/failure
// if no failure, then insanity will be null anyway
if (null != insanity) {
dumpArray(msg + ": Insane FieldCache usage(s)", insanity, System.err);
}
}
}
/**
* Convinience method for logging an iterator.
* @param label String logged before/after the items in the iterator
* @param iter Each next() is toString()ed and logged on it's own line. If iter is null this is logged differnetly then an empty iterator.
* @param stream Stream to log messages to.
*/
public static void dumpIterator(String label, Iterator iter,
PrintStream stream) {
stream.println("*** BEGIN "+label+" ***");
if (null == iter) {
stream.println(" ... NULL ...");
} else {
while (iter.hasNext()) {
stream.println(iter.next().toString());
}
}
stream.println("*** END "+label+" ***");
}
/**
* Convinience method for logging an array. Wraps the array in an iterator and delegates
* @see dumpIterator(String,Iterator,PrintStream)
*/
public static void dumpArray(String label, Object[] objs,
PrintStream stream) {
Iterator iter = (null == objs) ? null : Arrays.asList(objs).iterator();
dumpIterator(label, iter, stream);
}
/**
* Returns a {@link Random} instance for generating random numbers during the test.
* The random seed is logged during test execution and printed to System.out on any failure

181
src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java Normal file
View File

@ -0,0 +1,181 @@
package org.apache.lucene.util;
/**
* Copyright 2009 The Apache Software Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
import org.apache.lucene.util.FieldCacheSanityChecker.InsanityType;
import java.io.IOException;
public class TestFieldCacheSanityChecker extends LuceneTestCase {
protected IndexReader readerA;
protected IndexReader readerB;
protected IndexReader readerX;
private static final int NUM_DOCS = 1000;
protected void setUp() throws Exception {
super.setUp();
RAMDirectory dirA = new RAMDirectory();
RAMDirectory dirB = new RAMDirectory();
IndexWriter wA = new IndexWriter(dirA, new WhitespaceAnalyzer(), true,
IndexWriter.MaxFieldLength.LIMITED);
IndexWriter wB = new IndexWriter(dirB, new WhitespaceAnalyzer(), true,
IndexWriter.MaxFieldLength.LIMITED);
long theLong = Long.MAX_VALUE;
double theDouble = Double.MAX_VALUE;
byte theByte = Byte.MAX_VALUE;
short theShort = Short.MAX_VALUE;
int theInt = Integer.MAX_VALUE;
float theFloat = Float.MAX_VALUE;
for (int i = 0; i < NUM_DOCS; i++){
Document doc = new Document();
doc.add(new Field("theLong", String.valueOf(theLong--), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(new Field("theDouble", String.valueOf(theDouble--), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(new Field("theByte", String.valueOf(theByte--), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(new Field("theShort", String.valueOf(theShort--), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(new Field("theInt", String.valueOf(theInt--), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(new Field("theFloat", String.valueOf(theFloat--), Field.Store.NO, Field.Index.NOT_ANALYZED));
if (0 == i % 3) {
wA.addDocument(doc);
} else {
wB.addDocument(doc);
}
}
wA.close();
wB.close();
readerA = IndexReader.open(dirA);
readerB = IndexReader.open(dirB);
readerX = new MultiReader(new IndexReader[] { readerA, readerB });
}
public void tearDown() throws Exception {
super.tearDown();
readerA.close();
readerB.close();
readerX.close();
}
public void testSanity() throws IOException {
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
double [] doubles;
int [] ints;
doubles = cache.getDoubles(readerA, "theDouble");
doubles = cache.getDoubles(readerA, "theDouble",
FieldCache.DEFAULT_DOUBLE_PARSER);
doubles = cache.getDoubles(readerB, "theDouble",
FieldCache.DEFAULT_DOUBLE_PARSER);
ints = cache.getInts(readerX, "theInt");
ints = cache.getInts(readerX, "theInt",
FieldCache.DEFAULT_INT_PARSER);
// // //
Insanity[] insanity =
FieldCacheSanityChecker.checkSanity(cache.getCacheEntries());
if (0 < insanity.length)
dumpArray(getTestLabel() + " INSANITY", insanity, System.err);
assertEquals("shouldn't be any cache insanity", 0, insanity.length);
cache.purgeAllCaches();
}
public void testInsanity1() throws IOException {
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
int [] ints;
String [] strings;
byte [] bytes;
ints = cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER);
strings = cache.getStrings(readerX, "theInt");
// this one is ok
bytes = cache.getBytes(readerX, "theByte");
// // //
Insanity[] insanity =
FieldCacheSanityChecker.checkSanity(cache.getCacheEntries());
assertEquals("wrong number of cache errors", 1, insanity.length);
assertEquals("wrong type of cache error",
InsanityType.VALUEMISMATCH,
insanity[0].getType());
assertEquals("wrong number of entries in cache error", 2,
insanity[0].getCacheEntries().length);
// we expect bad things, don't let tearDown complain about them
cache.purgeAllCaches();
}
public void testInsanity2() throws IOException {
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
String [] strings;
byte [] bytes;
strings = cache.getStrings(readerA, "theString");
strings = cache.getStrings(readerB, "theString");
strings = cache.getStrings(readerX, "theString");
// this one is ok
bytes = cache.getBytes(readerX, "theByte");
// // //
Insanity[] insanity =
FieldCacheSanityChecker.checkSanity(cache.getCacheEntries());
assertEquals("wrong number of cache errors", 1, insanity.length);
assertEquals("wrong type of cache error",
InsanityType.SUBREADER,
insanity[0].getType());
assertEquals("wrong number of entries in cache error", 3,
insanity[0].getCacheEntries().length);
// we expect bad things, don't let tearDown complain about them
cache.purgeAllCaches();
}
public void testInsanity3() throws IOException {
// :TODO: subreader tree walking is really hairy ... add more crazy tests.
}
}

57
src/test/org/apache/lucene/util/TestRamUsageEstimator.java Normal file
View File

@ -0,0 +1,57 @@
package org.apache.lucene.util;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import junit.framework.TestCase;
public class TestRamUsageEstimator extends TestCase {
public void testBasic() {
String string = new String("test str");
RamUsageEstimator rue = new RamUsageEstimator();
long size = rue.estimateRamUsage(string);
System.out.println("size:" + size);
string = new String("test strin");
size = rue.estimateRamUsage(string);
System.out.println("size:" + size);
Holder holder = new Holder();
holder.holder = new Holder("string2", 5000L);
size = rue.estimateRamUsage(holder);
System.out.println("size:" + size);
String[] strings = new String[]{new String("test strin"), new String("hollow"), new String("catchmaster")};
size = rue.estimateRamUsage(strings);
System.out.println("size:" + size);
}
private static final class Holder {
long field1 = 5000L;
String name = "name";
Holder holder;
Holder() {
}
Holder(String name, long field1) {
this.name = name;
this.field1 = field1;
}
}
}