honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enable eager cache eviction based on readers, requires a nasty hack to Lucene, basically overriding two classes to hook into when a reader actually gets closed (and only enabled when elasticsearch is before lucene in the classpath, which we take for in the startup scripts)

This commit is contained in:
kimchy 2010-07-07 17:15:26 +03:00
parent a36b06c682
commit 97b5b41522
8 changed files with 1535 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/elasticsearch.bat
View File

@ -20,7 +20,7 @@ set JAVA_OPTS=^
-XX:+CMSParallelRemarkEnabled^
-XX:+HeapDumpOnOutOfMemoryError
set ES_CLASSPATH=$CLASSPATH;"%ES_HOME%/lib/*";"%ES_HOME%/lib/sigar/*"
set ES_CLASSPATH=$CLASSPATH;"%ES_HOME%/lib/elasticsearch-@ES_VERSION@.jar";"%ES_HOME%/lib/*";"%ES_HOME%/lib/sigar/*"
set ES_PARAMS=-Delasticsearch -Des-foreground=yes -Des.path.home="%ES_HOME%"
"%JAVA_HOME%\bin\java" %JAVA_OPTS% %ES_JAVA_OPTS% %ES_PARAMS% -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Bootstrap"

2
bin/elasticsearch.in.sh
View File

@ -1,4 +1,4 @@
CLASSPATH=$CLASSPATH:$ES_HOME/lib/*:$ES_HOME/lib/sigar/*
CLASSPATH=$CLASSPATH:$ES_HOME/lib/elasticsearch-@ES_VERSION@.jar:$ES_HOME/lib/*:$ES_HOME/lib/sigar/*
if [ "x$ES_MIN_MEM" = "x" ]; then
ES_MIN_MEM=256m

2
build.gradle
View File

@ -79,6 +79,8 @@ task explodedDist(dependsOn: [configurations.distLib], description: 'Builds a mi
include 'README.textile'
}
ant.replace(dir: explodedDistBinDir, token: "@ES_VERSION@", value: versionNumber)
ant.delete { fileset(dir: explodedDistLibDir, includes: "$archivesBaseName-*-javadoc.jar") }
ant.delete { fileset(dir: explodedDistLibDir, includes: "$archivesBaseName-*-sources.jar") }
ant.delete { fileset(dir: explodedDistLibDir, includes: "slf4j-*.jar") } // no need for slf4j

716
modules/elasticsearch/src/main/java/org/apache/lucene/search/FieldCache.java Normal file
View File

@ -0,0 +1,716 @@
package org.apache.lucene.search;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.lucene.document.NumericField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.RamUsageEstimator;
import java.io.IOException;
import java.io.PrintStream;
import java.io.Serializable;
import java.text.DecimalFormat;
/**
* Expert: Maintains caches of term values.
*
* <p>Created: May 19, 2004 11:13:14 AM
*
* @see org.apache.lucene.util.FieldCacheSanityChecker
* @since lucene 1.4
*/
// LUCENE MONITOR - Added the ability to listen to purge notifications, sadly no extension point in Lucene for this...
public interface FieldCache {
void setIndexReaderPurgedListener(IndexReaderPurgedListener listener);
public static final class CreationPlaceholder {
Object value;
}
/**
* Indicator for StringIndex values in the cache.
*/
// NOTE: the value assigned to this constant must not be
// the same as any of those in SortField!!
public static final int STRING_INDEX = -1;
/**
* Expert: Stores term text values and document ordering data.
*/
public static class StringIndex {
public int binarySearchLookup(String key) {
// this special case is the reason that Arrays.binarySearch() isn't useful.
if (key == null)
return 0;
int low = 1;
int high = lookup.length - 1;
while (low <= high) {
int mid = (low + high) >>> 1;
int cmp = lookup[mid].compareTo(key);
if (cmp < 0)
low = mid + 1;
else if (cmp > 0)
high = mid - 1;
else
return mid; // key found
}
return -(low + 1); // key not found.
}
/**
* All the term values, in natural order.
*/
public final String[] lookup;
/**
* For each document, an index into the lookup array.
*/
public final int[] order;
/**
* Creates one of these objects
*/
public StringIndex(int[] values, String[] lookup) {
this.order = values;
this.lookup = lookup;
}
}
/**
* Marker interface as super-interface to all parsers. It
* is used to specify a custom parser to {@link
* SortField#SortField(String, FieldCache.Parser)}.
*/
public interface Parser extends Serializable {
}
/**
* Interface to parse bytes from document fields.
*
* @see FieldCache#getBytes(IndexReader, String, FieldCache.ByteParser)
*/
public interface ByteParser extends Parser {
/**
* Return a single Byte representation of this field's value.
*/
public byte parseByte(String string);
}
/**
* Interface to parse shorts from document fields.
*
* @see FieldCache#getShorts(IndexReader, String, FieldCache.ShortParser)
*/
public interface ShortParser extends Parser {
/**
* Return a short representation of this field's value.
*/
public short parseShort(String string);
}
/**
* Interface to parse ints from document fields.
*
* @see FieldCache#getInts(IndexReader, String, FieldCache.IntParser)
*/
public interface IntParser extends Parser {
/**
* Return an integer representation of this field's value.
*/
public int parseInt(String string);
}
/**
* Interface to parse floats from document fields.
*
* @see FieldCache#getFloats(IndexReader, String, FieldCache.FloatParser)
*/
public interface FloatParser extends Parser {
/**
* Return an float representation of this field's value.
*/
public float parseFloat(String string);
}
/**
* Interface to parse long from document fields.
*
* @see FieldCache#getLongs(IndexReader, String, FieldCache.LongParser)
*/
public interface LongParser extends Parser {
/**
* Return an long representation of this field's value.
*/
public long parseLong(String string);
}
/**
* Interface to parse doubles from document fields.
*
* @see FieldCache#getDoubles(IndexReader, String, FieldCache.DoubleParser)
*/
public interface DoubleParser extends Parser {
/**
* Return an long representation of this field's value.
*/
public double parseDouble(String string);
}
/**
* Expert: The cache used internally by sorting and range query classes.
*/
public static FieldCache DEFAULT = new FieldCacheImpl();
/**
* The default parser for byte values, which are encoded by {@link Byte#toString(byte)}
*/
public static final ByteParser DEFAULT_BYTE_PARSER = new ByteParser() {
public byte parseByte(String value) {
return Byte.parseByte(value);
}
protected Object readResolve() {
return DEFAULT_BYTE_PARSER;
}
@Override
public String toString() {
return FieldCache.class.getName() + ".DEFAULT_BYTE_PARSER";
}
};
/**
* The default parser for short values, which are encoded by {@link Short#toString(short)}
*/
public static final ShortParser DEFAULT_SHORT_PARSER = new ShortParser() {
public short parseShort(String value) {
return Short.parseShort(value);
}
protected Object readResolve() {
return DEFAULT_SHORT_PARSER;
}
@Override
public String toString() {
return FieldCache.class.getName() + ".DEFAULT_SHORT_PARSER";
}
};
/**
* The default parser for int values, which are encoded by {@link Integer#toString(int)}
*/
public static final IntParser DEFAULT_INT_PARSER = new IntParser() {
public int parseInt(String value) {
return Integer.parseInt(value);
}
protected Object readResolve() {
return DEFAULT_INT_PARSER;
}
@Override
public String toString() {
return FieldCache.class.getName() + ".DEFAULT_INT_PARSER";
}
};
/**
* The default parser for float values, which are encoded by {@link Float#toString(float)}
*/
public static final FloatParser DEFAULT_FLOAT_PARSER = new FloatParser() {
public float parseFloat(String value) {
return Float.parseFloat(value);
}
protected Object readResolve() {
return DEFAULT_FLOAT_PARSER;
}
@Override
public String toString() {
return FieldCache.class.getName() + ".DEFAULT_FLOAT_PARSER";
}
};
/**
* The default parser for long values, which are encoded by {@link Long#toString(long)}
*/
public static final LongParser DEFAULT_LONG_PARSER = new LongParser() {
public long parseLong(String value) {
return Long.parseLong(value);
}
protected Object readResolve() {
return DEFAULT_LONG_PARSER;
}
@Override
public String toString() {
return FieldCache.class.getName() + ".DEFAULT_LONG_PARSER";
}
};
/**
* The default parser for double values, which are encoded by {@link Double#toString(double)}
*/
public static final DoubleParser DEFAULT_DOUBLE_PARSER = new DoubleParser() {
public double parseDouble(String value) {
return Double.parseDouble(value);
}
protected Object readResolve() {
return DEFAULT_DOUBLE_PARSER;
}
@Override
public String toString() {
return FieldCache.class.getName() + ".DEFAULT_DOUBLE_PARSER";
}
};
/**
* A parser instance for int values encoded by {@link NumericUtils#intToPrefixCoded(int)}, e.g. when indexed
* via {@link NumericField}/{@link NumericTokenStream}.
*/
public static final IntParser NUMERIC_UTILS_INT_PARSER = new IntParser() {
public int parseInt(String val) {
final int shift = val.charAt(0) - NumericUtils.SHIFT_START_INT;
if (shift > 0 && shift <= 31)
throw new FieldCacheImpl.StopFillCacheException();
return NumericUtils.prefixCodedToInt(val);
}
protected Object readResolve() {
return NUMERIC_UTILS_INT_PARSER;
}
@Override
public String toString() {
return FieldCache.class.getName() + ".NUMERIC_UTILS_INT_PARSER";
}
};
/**
* A parser instance for float values encoded with {@link NumericUtils}, e.g. when indexed
* via {@link NumericField}/{@link NumericTokenStream}.
*/
public static final FloatParser NUMERIC_UTILS_FLOAT_PARSER = new FloatParser() {
public float parseFloat(String val) {
final int shift = val.charAt(0) - NumericUtils.SHIFT_START_INT;
if (shift > 0 && shift <= 31)
throw new FieldCacheImpl.StopFillCacheException();
return NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(val));
}
protected Object readResolve() {
return NUMERIC_UTILS_FLOAT_PARSER;
}
@Override
public String toString() {
return FieldCache.class.getName() + ".NUMERIC_UTILS_FLOAT_PARSER";
}
};
/**
* A parser instance for long values encoded by {@link NumericUtils#longToPrefixCoded(long)}, e.g. when indexed
* via {@link NumericField}/{@link NumericTokenStream}.
*/
public static final LongParser NUMERIC_UTILS_LONG_PARSER = new LongParser() {
public long parseLong(String val) {
final int shift = val.charAt(0) - NumericUtils.SHIFT_START_LONG;
if (shift > 0 && shift <= 63)
throw new FieldCacheImpl.StopFillCacheException();
return NumericUtils.prefixCodedToLong(val);
}
protected Object readResolve() {
return NUMERIC_UTILS_LONG_PARSER;
}
@Override
public String toString() {
return FieldCache.class.getName() + ".NUMERIC_UTILS_LONG_PARSER";
}
};
/**
* A parser instance for double values encoded with {@link NumericUtils}, e.g. when indexed
* via {@link NumericField}/{@link NumericTokenStream}.
*/
public static final DoubleParser NUMERIC_UTILS_DOUBLE_PARSER = new DoubleParser() {
public double parseDouble(String val) {
final int shift = val.charAt(0) - NumericUtils.SHIFT_START_LONG;
if (shift > 0 && shift <= 63)
throw new FieldCacheImpl.StopFillCacheException();
return NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(val));
}
protected Object readResolve() {
return NUMERIC_UTILS_DOUBLE_PARSER;
}
@Override
public String toString() {
return FieldCache.class.getName() + ".NUMERIC_UTILS_DOUBLE_PARSER";
}
};
/**
* Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as a single byte and returns an array
* of size <code>reader.maxDoc()</code> of the value each document
* has in the given field.
*
* @param reader Used to get field values.
* @param field Which field contains the single byte values.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public byte[] getBytes(IndexReader reader, String field)
throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is found,
* reads the terms in <code>field</code> as bytes and returns an array of
* size <code>reader.maxDoc()</code> of the value each document has in the
* given field.
*
* @param reader Used to get field values.
* @param field Which field contains the bytes.
* @param parser Computes byte for string values.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public byte[] getBytes(IndexReader reader, String field, ByteParser parser)
throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as shorts and returns an array
* of size <code>reader.maxDoc()</code> of the value each document
* has in the given field.
*
* @param reader Used to get field values.
* @param field Which field contains the shorts.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public short[] getShorts(IndexReader reader, String field)
throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is found,
* reads the terms in <code>field</code> as shorts and returns an array of
* size <code>reader.maxDoc()</code> of the value each document has in the
* given field.
*
* @param reader Used to get field values.
* @param field Which field contains the shorts.
* @param parser Computes short for string values.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public short[] getShorts(IndexReader reader, String field, ShortParser parser)
throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as integers and returns an array
* of size <code>reader.maxDoc()</code> of the value each document
* has in the given field.
*
* @param reader Used to get field values.
* @param field Which field contains the integers.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public int[] getInts(IndexReader reader, String field)
throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is found,
* reads the terms in <code>field</code> as integers and returns an array of
* size <code>reader.maxDoc()</code> of the value each document has in the
* given field.
*
* @param reader Used to get field values.
* @param field Which field contains the integers.
* @param parser Computes integer for string values.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public int[] getInts(IndexReader reader, String field, IntParser parser)
throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if
* none is found, reads the terms in <code>field</code> as floats and returns an array
* of size <code>reader.maxDoc()</code> of the value each document
* has in the given field.
*
* @param reader Used to get field values.
* @param field Which field contains the floats.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public float[] getFloats(IndexReader reader, String field)
throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if
* none is found, reads the terms in <code>field</code> as floats and returns an array
* of size <code>reader.maxDoc()</code> of the value each document
* has in the given field.
*
* @param reader Used to get field values.
* @param field Which field contains the floats.
* @param parser Computes float for string values.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public float[] getFloats(IndexReader reader, String field,
FloatParser parser) throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as longs and returns an array
* of size <code>reader.maxDoc()</code> of the value each document
* has in the given field.
*
* @param reader Used to get field values.
* @param field Which field contains the longs.
* @return The values in the given field for each document.
* @throws java.io.IOException If any error occurs.
*/
public long[] getLongs(IndexReader reader, String field)
throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is found,
* reads the terms in <code>field</code> as longs and returns an array of
* size <code>reader.maxDoc()</code> of the value each document has in the
* given field.
*
* @param reader Used to get field values.
* @param field Which field contains the longs.
* @param parser Computes integer for string values.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public long[] getLongs(IndexReader reader, String field, LongParser parser)
throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as integers and returns an array
* of size <code>reader.maxDoc()</code> of the value each document
* has in the given field.
*
* @param reader Used to get field values.
* @param field Which field contains the doubles.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public double[] getDoubles(IndexReader reader, String field)
throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is found,
* reads the terms in <code>field</code> as doubles and returns an array of
* size <code>reader.maxDoc()</code> of the value each document has in the
* given field.
*
* @param reader Used to get field values.
* @param field Which field contains the doubles.
* @param parser Computes integer for string values.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public double[] getDoubles(IndexReader reader, String field, DoubleParser parser)
throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none
* is found, reads the term values in <code>field</code> and returns an array
* of size <code>reader.maxDoc()</code> containing the value each document
* has in the given field.
*
* @param reader Used to get field values.
* @param field Which field contains the strings.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public String[] getStrings(IndexReader reader, String field)
throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none
* is found reads the term values in <code>field</code> and returns
* an array of them in natural order, along with an array telling
* which element in the term array each document uses.
*
* @param reader Used to get field values.
* @param field Which field contains the strings.
* @return Array of terms and index into the array for each document.
* @throws IOException If any error occurs.
*/
public StringIndex getStringIndex(IndexReader reader, String field)
throws IOException;
/**
* EXPERT: A unique Identifier/Description for each item in the FieldCache.
* Can be useful for logging/debugging.
* <p>
* <b>EXPERIMENTAL API:</b> This API is considered extremely advanced
* and experimental. It may be removed or altered w/o warning in future
* releases
* of Lucene.
* </p>
*/
public static abstract class CacheEntry {
public abstract Object getReaderKey();
public abstract String getFieldName();
public abstract Class getCacheType();
public abstract Object getCustom();
public abstract Object getValue();
private String size = null;
protected final void setEstimatedSize(String size) {
this.size = size;
}
/**
* @see #estimateSize(RamUsageEstimator)
*/
public void estimateSize() {
estimateSize(new RamUsageEstimator(false)); // doesn't check for interned
}
/**
* Computes (and stores) the estimated size of the cache Value
*
* @see #getEstimatedSize
*/
public void estimateSize(RamUsageEstimator ramCalc) {
long size = ramCalc.estimateRamUsage(getValue());
setEstimatedSize(RamUsageEstimator.humanReadableUnits
(size, new DecimalFormat("0.#")));
}
/**
* The most recently estimated size of the value, null unless
* estimateSize has been called.
*/
public final String getEstimatedSize() {
return size;
}
@Override
public String toString() {
StringBuilder b = new StringBuilder();
b.append("'").append(getReaderKey()).append("'=>");
b.append("'").append(getFieldName()).append("',");
b.append(getCacheType()).append(",").append(getCustom());
b.append("=>").append(getValue().getClass().getName()).append("#");
b.append(System.identityHashCode(getValue()));
String s = getEstimatedSize();
if (null != s) {
b.append(" (size =~ ").append(s).append(')');
}
return b.toString();
}
}
/**
* EXPERT: Generates an array of CacheEntry objects representing all items
* currently in the FieldCache.
* <p>
* NOTE: These CacheEntry objects maintain a strong reference to the
* Cached Values. Maintaining references to a CacheEntry the IndexReader
* associated with it has garbage collected will prevent the Value itself
* from being garbage collected when the Cache drops the WeakRefrence.
* </p>
* <p>
* <b>EXPERIMENTAL API:</b> This API is considered extremely advanced
* and experimental. It may be removed or altered w/o warning in future
* releases
* of Lucene.
* </p>
*/
public abstract CacheEntry[] getCacheEntries();
/**
* <p>
* EXPERT: Instructs the FieldCache to forcibly expunge all entries
* from the underlying caches. This is intended only to be used for
* test methods as a way to ensure a known base state of the Cache
* (with out needing to rely on GC to free WeakReferences).
* It should not be relied on for "Cache maintenance" in general
* application code.
* </p>
* <p>
* <b>EXPERIMENTAL API:</b> This API is considered extremely advanced
* and experimental. It may be removed or altered w/o warning in future
* releases
* of Lucene.
* </p>
*/
public abstract void purgeAllCaches();
/**
* Expert: drops all cache entries associated with this
* reader. NOTE: this reader must precisely match the
* reader that the cache entry is keyed on. If you pass a
* top-level reader, it usually will have no effect as
* Lucene now caches at the segment reader level.
*/
public abstract void purge(IndexReader r);
/**
* If non-null, FieldCacheImpl will warn whenever
* entries are created that are not sane according to
* {@link org.apache.lucene.util.FieldCacheSanityChecker}.
*/
public void setInfoStream(PrintStream stream);
/**
* counterpart of {@link #setInfoStream(PrintStream)}
*/
public PrintStream getInfoStream();
}

754
modules/elasticsearch/src/main/java/org/apache/lucene/search/FieldCacheImpl.java Normal file
View File

@ -0,0 +1,754 @@
package org.apache.lucene.search;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.TermEnum;
import org.apache.lucene.util.FieldCacheSanityChecker;
import org.apache.lucene.util.StringHelper;
import java.io.IOException;
import java.io.PrintStream;
import java.util.*;
/**
* Expert: The default cache implementation, storing all values in memory.
* A WeakHashMap is used for storage.
*
* <p>Created: May 19, 2004 4:40:36 PM
*
* @since lucene 1.4
*/
// LUCENE MONITOR - Added a listener for purge events
class FieldCacheImpl implements FieldCache {
static {
if (Boolean.getBoolean("es.fieldcache.debug")) {
System.out.println("Using elasticsearch fieldcache");
}
}
private IndexReaderPurgedListener listener;
@Override public void setIndexReaderPurgedListener(IndexReaderPurgedListener listener) {
this.listener = listener;
}
private Map<Class<?>, Cache> caches;
FieldCacheImpl() {
init();
}
private synchronized void init() {
caches = new HashMap<Class<?>, Cache>(7);
caches.put(Byte.TYPE, new ByteCache(this));
caches.put(Short.TYPE, new ShortCache(this));
caches.put(Integer.TYPE, new IntCache(this));
caches.put(Float.TYPE, new FloatCache(this));
caches.put(Long.TYPE, new LongCache(this));
caches.put(Double.TYPE, new DoubleCache(this));
caches.put(String.class, new StringCache(this));
caches.put(StringIndex.class, new StringIndexCache(this));
}
public void purgeAllCaches() {
init();
}
public void purge(IndexReader r) {
if (listener != null) {
listener.indexReaderPurged(r);
}
for (Cache c : caches.values()) {
c.purge(r);
}
}
public CacheEntry[] getCacheEntries() {
List<CacheEntry> result = new ArrayList<CacheEntry>(17);
for (final Class<?> cacheType : caches.keySet()) {
Cache cache = caches.get(cacheType);
for (final Object readerKey : cache.readerCache.keySet()) {
// we've now materialized a hard ref
// innerKeys was backed by WeakHashMap, sanity check
// that it wasn't GCed before we made hard ref
if (null != readerKey && cache.readerCache.containsKey(readerKey)) {
Map<Entry, Object> innerCache = cache.readerCache.get(readerKey);
for (final Map.Entry<Entry, Object> mapEntry : innerCache.entrySet()) {
Entry entry = mapEntry.getKey();
result.add(new CacheEntryImpl(readerKey, entry.field,
cacheType, entry.custom,
mapEntry.getValue()));
}
}
}
}
return result.toArray(new CacheEntry[result.size()]);
}
private static final class CacheEntryImpl extends CacheEntry {
private final Object readerKey;
private final String fieldName;
private final Class<?> cacheType;
private final Object custom;
private final Object value;
CacheEntryImpl(Object readerKey, String fieldName,
Class<?> cacheType,
Object custom,
Object value) {
this.readerKey = readerKey;
this.fieldName = fieldName;
this.cacheType = cacheType;
this.custom = custom;
this.value = value;
// :HACK: for testing.
// if (null != locale || SortField.CUSTOM != sortFieldType) {
// throw new RuntimeException("Locale/sortFieldType: " + this);
// }
}
@Override
public Object getReaderKey() {
return readerKey;
}
@Override
public String getFieldName() {
return fieldName;
}
@Override
public Class<?> getCacheType() {
return cacheType;
}
@Override
public Object getCustom() {
return custom;
}
@Override
public Object getValue() {
return value;
}
}
/**
* Hack: When thrown from a Parser (NUMERIC_UTILS_* ones), this stops
* processing terms and returns the current FieldCache
* array.
*/
static final class StopFillCacheException extends RuntimeException {
}
/**
* Expert: Internal cache.
*/
abstract static class Cache {
Cache() {
this.wrapper = null;
}
Cache(FieldCache wrapper) {
this.wrapper = wrapper;
}
final FieldCache wrapper;
final Map<Object, Map<Entry, Object>> readerCache = new WeakHashMap<Object, Map<Entry, Object>>();
protected abstract Object createValue(IndexReader reader, Entry key)
throws IOException;
/**
* Remove this reader from the cache, if present.
*/
public void purge(IndexReader r) {
Object readerKey = r.getFieldCacheKey();
synchronized (readerCache) {
readerCache.remove(readerKey);
}
}
public Object get(IndexReader reader, Entry key) throws IOException {
Map<Entry, Object> innerCache;
Object value;
final Object readerKey = reader.getFieldCacheKey();
synchronized (readerCache) {
innerCache = readerCache.get(readerKey);
if (innerCache == null) {
innerCache = new HashMap<Entry, Object>();
readerCache.put(readerKey, innerCache);
value = null;
} else {
value = innerCache.get(key);
}
if (value == null) {
value = new CreationPlaceholder();
innerCache.put(key, value);
}
}
if (value instanceof CreationPlaceholder) {
synchronized (value) {
CreationPlaceholder progress = (CreationPlaceholder) value;
if (progress.value == null) {
progress.value = createValue(reader, key);
synchronized (readerCache) {
innerCache.put(key, progress.value);
}
// Only check if key.custom (the parser) is
// non-null; else, we check twice for a single
// call to FieldCache.getXXX
if (key.custom != null && wrapper != null) {
final PrintStream infoStream = wrapper.getInfoStream();
if (infoStream != null) {
printNewInsanity(infoStream, progress.value);
}
}
}
return progress.value;
}
}
return value;
}
private void printNewInsanity(PrintStream infoStream, Object value) {
final FieldCacheSanityChecker.Insanity[] insanities = FieldCacheSanityChecker.checkSanity(wrapper);
for (int i = 0; i < insanities.length; i++) {
final FieldCacheSanityChecker.Insanity insanity = insanities[i];
final CacheEntry[] entries = insanity.getCacheEntries();
for (int j = 0; j < entries.length; j++) {
if (entries[j].getValue() == value) {
// OK this insanity involves our entry
infoStream.println("WARNING: new FieldCache insanity created\nDetails: " + insanity.toString());
infoStream.println("\nStack:\n");
new Throwable().printStackTrace(infoStream);
break;
}
}
}
}
}
/**
* Expert: Every composite-key in the internal cache is of this type.
*/
static class Entry {
final String field; // which Fieldable
final Object custom; // which custom comparator or parser
/**
* Creates one of these objects for a custom comparator/parser.
*/
Entry(String field, Object custom) {
this.field = StringHelper.intern(field);
this.custom = custom;
}
/**
* Two of these are equal iff they reference the same field and type.
*/
@Override
public boolean equals(Object o) {
if (o instanceof Entry) {
Entry other = (Entry) o;
if (other.field == field) {
if (other.custom == null) {
if (custom == null) return true;
} else if (other.custom.equals(custom)) {
return true;
}
}
}
return false;
}
/**
* Composes a hashcode based on the field and type.
*/
@Override
public int hashCode() {
return field.hashCode() ^ (custom == null ? 0 : custom.hashCode());
}
}
// inherit javadocs
public byte[] getBytes(IndexReader reader, String field) throws IOException {
return getBytes(reader, field, null);
}
// inherit javadocs
public byte[] getBytes(IndexReader reader, String field, ByteParser parser)
throws IOException {
return (byte[]) caches.get(Byte.TYPE).get(reader, new Entry(field, parser));
}
static final class ByteCache extends Cache {
ByteCache(FieldCache wrapper) {
super(wrapper);
}
@Override
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
Entry entry = entryKey;
String field = entry.field;
ByteParser parser = (ByteParser) entry.custom;
if (parser == null) {
return wrapper.getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER);
}
final byte[] retArray = new byte[reader.maxDoc()];
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms(new Term(field));
try {
do {
Term term = termEnum.term();
if (term == null || term.field() != field) break;
byte termval = parser.parseByte(term.text());
termDocs.seek(termEnum);
while (termDocs.next()) {
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
} catch (StopFillCacheException stop) {
} finally {
termDocs.close();
termEnum.close();
}
return retArray;
}
}
;
// inherit javadocs
public short[] getShorts(IndexReader reader, String field) throws IOException {
return getShorts(reader, field, null);
}
// inherit javadocs
public short[] getShorts(IndexReader reader, String field, ShortParser parser)
throws IOException {
return (short[]) caches.get(Short.TYPE).get(reader, new Entry(field, parser));
}
static final class ShortCache extends Cache {
ShortCache(FieldCache wrapper) {
super(wrapper);
}
@Override
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
Entry entry = entryKey;
String field = entry.field;
ShortParser parser = (ShortParser) entry.custom;
if (parser == null) {
return wrapper.getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER);
}
final short[] retArray = new short[reader.maxDoc()];
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms(new Term(field));
try {
do {
Term term = termEnum.term();
if (term == null || term.field() != field) break;
short termval = parser.parseShort(term.text());
termDocs.seek(termEnum);
while (termDocs.next()) {
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
} catch (StopFillCacheException stop) {
} finally {
termDocs.close();
termEnum.close();
}
return retArray;
}
}
;
// inherit javadocs
public int[] getInts(IndexReader reader, String field) throws IOException {
return getInts(reader, field, null);
}
// inherit javadocs
public int[] getInts(IndexReader reader, String field, IntParser parser)
throws IOException {
return (int[]) caches.get(Integer.TYPE).get(reader, new Entry(field, parser));
}
static final class IntCache extends Cache {
IntCache(FieldCache wrapper) {
super(wrapper);
}
@Override
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
Entry entry = entryKey;
String field = entry.field;
IntParser parser = (IntParser) entry.custom;
if (parser == null) {
try {
return wrapper.getInts(reader, field, DEFAULT_INT_PARSER);
} catch (NumberFormatException ne) {
return wrapper.getInts(reader, field, NUMERIC_UTILS_INT_PARSER);
}
}
int[] retArray = null;
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms(new Term(field));
try {
do {
Term term = termEnum.term();
if (term == null || term.field() != field) break;
int termval = parser.parseInt(term.text());
if (retArray == null) // late init
retArray = new int[reader.maxDoc()];
termDocs.seek(termEnum);
while (termDocs.next()) {
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
} catch (StopFillCacheException stop) {
} finally {
termDocs.close();
termEnum.close();
}
if (retArray == null) // no values
retArray = new int[reader.maxDoc()];
return retArray;
}
}
;
// inherit javadocs
public float[] getFloats(IndexReader reader, String field)
throws IOException {
return getFloats(reader, field, null);
}
// inherit javadocs
public float[] getFloats(IndexReader reader, String field, FloatParser parser)
throws IOException {
return (float[]) caches.get(Float.TYPE).get(reader, new Entry(field, parser));
}
static final class FloatCache extends Cache {
FloatCache(FieldCache wrapper) {
super(wrapper);
}
@Override
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
Entry entry = entryKey;
String field = entry.field;
FloatParser parser = (FloatParser) entry.custom;
if (parser == null) {
try {
return wrapper.getFloats(reader, field, DEFAULT_FLOAT_PARSER);
} catch (NumberFormatException ne) {
return wrapper.getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER);
}
}
float[] retArray = null;
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms(new Term(field));
try {
do {
Term term = termEnum.term();
if (term == null || term.field() != field) break;
float termval = parser.parseFloat(term.text());
if (retArray == null) // late init
retArray = new float[reader.maxDoc()];
termDocs.seek(termEnum);
while (termDocs.next()) {
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
} catch (StopFillCacheException stop) {
} finally {
termDocs.close();
termEnum.close();
}
if (retArray == null) // no values
retArray = new float[reader.maxDoc()];
return retArray;
}
}
;
public long[] getLongs(IndexReader reader, String field) throws IOException {
return getLongs(reader, field, null);
}
// inherit javadocs
public long[] getLongs(IndexReader reader, String field, FieldCache.LongParser parser)
throws IOException {
return (long[]) caches.get(Long.TYPE).get(reader, new Entry(field, parser));
}
static final class LongCache extends Cache {
LongCache(FieldCache wrapper) {
super(wrapper);
}
@Override
protected Object createValue(IndexReader reader, Entry entry)
throws IOException {
String field = entry.field;
FieldCache.LongParser parser = (FieldCache.LongParser) entry.custom;
if (parser == null) {
try {
return wrapper.getLongs(reader, field, DEFAULT_LONG_PARSER);
} catch (NumberFormatException ne) {
return wrapper.getLongs(reader, field, NUMERIC_UTILS_LONG_PARSER);
}
}
long[] retArray = null;
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms(new Term(field));
try {
do {
Term term = termEnum.term();
if (term == null || term.field() != field) break;
long termval = parser.parseLong(term.text());
if (retArray == null) // late init
retArray = new long[reader.maxDoc()];
termDocs.seek(termEnum);
while (termDocs.next()) {
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
} catch (StopFillCacheException stop) {
} finally {
termDocs.close();
termEnum.close();
}
if (retArray == null) // no values
retArray = new long[reader.maxDoc()];
return retArray;
}
}
;
// inherit javadocs
public double[] getDoubles(IndexReader reader, String field)
throws IOException {
return getDoubles(reader, field, null);
}
// inherit javadocs
public double[] getDoubles(IndexReader reader, String field, FieldCache.DoubleParser parser)
throws IOException {
return (double[]) caches.get(Double.TYPE).get(reader, new Entry(field, parser));
}
static final class DoubleCache extends Cache {
DoubleCache(FieldCache wrapper) {
super(wrapper);
}
@Override
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
Entry entry = entryKey;
String field = entry.field;
FieldCache.DoubleParser parser = (FieldCache.DoubleParser) entry.custom;
if (parser == null) {
try {
return wrapper.getDoubles(reader, field, DEFAULT_DOUBLE_PARSER);
} catch (NumberFormatException ne) {
return wrapper.getDoubles(reader, field, NUMERIC_UTILS_DOUBLE_PARSER);
}
}
double[] retArray = null;
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms(new Term(field));
try {
do {
Term term = termEnum.term();
if (term == null || term.field() != field) break;
double termval = parser.parseDouble(term.text());
if (retArray == null) // late init
retArray = new double[reader.maxDoc()];
termDocs.seek(termEnum);
while (termDocs.next()) {
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
} catch (StopFillCacheException stop) {
} finally {
termDocs.close();
termEnum.close();
}
if (retArray == null) // no values
retArray = new double[reader.maxDoc()];
return retArray;
}
}
;
// inherit javadocs
public String[] getStrings(IndexReader reader, String field)
throws IOException {
return (String[]) caches.get(String.class).get(reader, new Entry(field, (Parser) null));
}
static final class StringCache extends Cache {
StringCache(FieldCache wrapper) {
super(wrapper);
}
@Override
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
String field = StringHelper.intern(entryKey.field);
final String[] retArray = new String[reader.maxDoc()];
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms(new Term(field));
try {
do {
Term term = termEnum.term();
if (term == null || term.field() != field) break;
String termval = term.text();
termDocs.seek(termEnum);
while (termDocs.next()) {
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
} finally {
termDocs.close();
termEnum.close();
}
return retArray;
}
}
;
// inherit javadocs
public StringIndex getStringIndex(IndexReader reader, String field)
throws IOException {
return (StringIndex) caches.get(StringIndex.class).get(reader, new Entry(field, (Parser) null));
}
static final class StringIndexCache extends Cache {
StringIndexCache(FieldCache wrapper) {
super(wrapper);
}
@Override
protected Object createValue(IndexReader reader, Entry entryKey)
throws IOException {
String field = StringHelper.intern(entryKey.field);
final int[] retArray = new int[reader.maxDoc()];
String[] mterms = new String[reader.maxDoc() + 1];
TermDocs termDocs = reader.termDocs();
TermEnum termEnum = reader.terms(new Term(field));
int t = 0; // current term number
// an entry for documents that have no terms in this field
// should a document with no terms be at top or bottom?
// this puts them at the top - if it is changed, FieldDocSortedHitQueue
// needs to change as well.
mterms[t++] = null;
try {
do {
Term term = termEnum.term();
if (term == null || term.field() != field) break;
// store term text
mterms[t] = term.text();
termDocs.seek(termEnum);
while (termDocs.next()) {
retArray[termDocs.doc()] = t;
}
t++;
} while (termEnum.next());
} finally {
termDocs.close();
termEnum.close();
}
if (t == 0) {
// if there are no terms, make the term array
// have a single null entry
mterms = new String[1];
} else if (t < mterms.length) {
// if there are less terms than documents,
// trim off the dead array space
String[] terms = new String[t];
System.arraycopy(mterms, 0, terms, 0, t);
mterms = terms;
}
StringIndex value = new StringIndex(retArray, mterms);
return value;
}
}
;
private volatile PrintStream infoStream;
public void setInfoStream(PrintStream stream) {
infoStream = stream;
}
public PrintStream getInfoStream() {
return infoStream;
}
}

30
modules/elasticsearch/src/main/java/org/apache/lucene/search/IndexReaderPurgedListener.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lucene.search;
import org.apache.lucene.index.IndexReader;
/**
* @author kimchy (shay.banon)
*/
public interface IndexReaderPurgedListener {
void indexReaderPurged(IndexReader reader);
}

6
modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCacheModule.java vendored
View File

@ -22,10 +22,10 @@ package org.elasticsearch.index.cache.field.data;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Scopes;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.cache.field.data.weak.WeakFieldDataCache;
import org.elasticsearch.index.cache.field.data.soft.SoftFieldDataCache;
/**
* @author kimchy (Shay Banon)
* @author kimchy (shay.banon)
*/
public class FieldDataCacheModule extends AbstractModule {
@ -41,7 +41,7 @@ public class FieldDataCacheModule extends AbstractModule {
@Override protected void configure() {
bind(FieldDataCache.class)
.to(settings.getAsClass(FieldDataCacheSettings.FIELD_DATA_CACHE_TYPE, WeakFieldDataCache.class, "org.elasticsearch.index.cache.field.", "FieldDataCache"))
.to(settings.getAsClass(FieldDataCacheSettings.FIELD_DATA_CACHE_TYPE, SoftFieldDataCache.class, "org.elasticsearch.index.cache.field.", "FieldDataCache"))
.in(Scopes.SINGLETON);
}
}

28
modules/elasticsearch/src/main/java/org/elasticsearch/indices/InternalIndicesService.java
View File

@ -19,6 +19,9 @@
package org.elasticsearch.indices;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexReaderPurgedListener;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.routing.GroupShardsIterator;
@ -93,6 +96,16 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
this.injector = injector;
this.pluginsService = injector.getInstance(PluginsService.class);
try {
FieldCache.DEFAULT.getClass().getMethod("setIndexReaderPurgedListener", IndexReaderPurgedListener.class);
// LUCENE MONITOR - This is a hack to eagerly clean caches based on index reader
FieldCache.DEFAULT.setIndexReaderPurgedListener(new CacheReaderPurgeListener(this));
logger.trace("eager reader based cache eviction enabled");
} catch (NoSuchMethodException e) {
// no method
logger.debug("lucene default FieldCache is used, not enabling eager reader based cache eviction");
}
}
@Override protected void doStart() throws ElasticSearchException {
@ -249,4 +262,19 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
indicesLifecycle.afterIndexClosed(indexService.index(), delete);
}
private static class CacheReaderPurgeListener implements IndexReaderPurgedListener {
private final IndicesService indicesService;
private CacheReaderPurgeListener(IndicesService indicesService) {
this.indicesService = indicesService;
}
@Override public void indexReaderPurged(IndexReader reader) {
for (IndexService indexService : indicesService) {
indexService.cache().clear(reader);
}
}
}
}