enable eager cache eviction based on readers, requires a nasty hack to Lucene, basically overriding two classes to hook into when a reader actually gets closed (and only enabled when elasticsearch is before lucene in the classpath, which we take for in the startup scripts)

bin/elasticsearch.bat

@@ -20,7 +20,7 @@ set JAVA_OPTS=^
  -XX:+CMSParallelRemarkEnabled^
  -XX:+HeapDumpOnOutOfMemoryError
 
-set ES_CLASSPATH=$CLASSPATH;"%ES_HOME%/lib/*";"%ES_HOME%/lib/sigar/*"
+set ES_CLASSPATH=$CLASSPATH;"%ES_HOME%/lib/elasticsearch-@ES_VERSION@.jar";"%ES_HOME%/lib/*";"%ES_HOME%/lib/sigar/*"
 set ES_PARAMS=-Delasticsearch -Des-foreground=yes -Des.path.home="%ES_HOME%"
 
 "%JAVA_HOME%\bin\java" %JAVA_OPTS% %ES_JAVA_OPTS% %ES_PARAMS% -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Bootstrap"

bin/elasticsearch.in.sh

@@ -1,4 +1,4 @@
-CLASSPATH=$CLASSPATH:$ES_HOME/lib/*:$ES_HOME/lib/sigar/*
+CLASSPATH=$CLASSPATH:$ES_HOME/lib/elasticsearch-@ES_VERSION@.jar:$ES_HOME/lib/*:$ES_HOME/lib/sigar/*
 
 if [ "x$ES_MIN_MEM" = "x" ]; then
     ES_MIN_MEM=256m

build.gradle

@@ -79,6 +79,8 @@ task explodedDist(dependsOn: [configurations.distLib], description: 'Builds a mi
         include 'README.textile'
     }
 
+    ant.replace(dir: explodedDistBinDir, token: "@ES_VERSION@", value: versionNumber)
+
     ant.delete { fileset(dir: explodedDistLibDir, includes: "$archivesBaseName-*-javadoc.jar") }
     ant.delete { fileset(dir: explodedDistLibDir, includes: "$archivesBaseName-*-sources.jar") }
     ant.delete { fileset(dir: explodedDistLibDir, includes: "slf4j-*.jar") } // no need for slf4j

modules/elasticsearch/src/main/java/org/apache/lucene/search/FieldCache.java

@@ -0,0 +1,716 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.document.NumericField;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.RamUsageEstimator;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.Serializable;
+import java.text.DecimalFormat;
+
+/**
+ * Expert: Maintains caches of term values.
+ *
+ * <p>Created: May 19, 2004 11:13:14 AM
+ *
+ * @see org.apache.lucene.util.FieldCacheSanityChecker
+ * @since lucene 1.4
+ */
+// LUCENE MONITOR - Added the ability to listen to purge notifications, sadly no extension point in Lucene for this...
+public interface FieldCache {
+
+    void setIndexReaderPurgedListener(IndexReaderPurgedListener listener);
+
+    public static final class CreationPlaceholder {
+        Object value;
+    }
+
+    /**
+     * Indicator for StringIndex values in the cache.
+     */
+    // NOTE: the value assigned to this constant must not be
+    // the same as any of those in SortField!!
+    public static final int STRING_INDEX = -1;
+
+
+    /**
+     * Expert: Stores term text values and document ordering data.
+     */
+    public static class StringIndex {
+
+        public int binarySearchLookup(String key) {
+            // this special case is the reason that Arrays.binarySearch() isn't useful.
+            if (key == null)
+                return 0;
+
+            int low = 1;
+            int high = lookup.length - 1;
+
+            while (low <= high) {
+                int mid = (low + high) >>> 1;
+                int cmp = lookup[mid].compareTo(key);
+
+                if (cmp < 0)
+                    low = mid + 1;
+                else if (cmp > 0)
+                    high = mid - 1;
+                else
+                    return mid; // key found
+            }
+            return -(low + 1);  // key not found.
+        }
+
+        /**
+         * All the term values, in natural order.
+         */
+        public final String[] lookup;
+
+        /**
+         * For each document, an index into the lookup array.
+         */
+        public final int[] order;
+
+        /**
+         * Creates one of these objects
+         */
+        public StringIndex(int[] values, String[] lookup) {
+            this.order = values;
+            this.lookup = lookup;
+        }
+    }
+
+    /**
+     * Marker interface as super-interface to all parsers. It
+     * is used to specify a custom parser to {@link
+     * SortField#SortField(String, FieldCache.Parser)}.
+     */
+    public interface Parser extends Serializable {
+    }
+
+    /**
+     * Interface to parse bytes from document fields.
+     *
+     * @see FieldCache#getBytes(IndexReader, String, FieldCache.ByteParser)
+     */
+    public interface ByteParser extends Parser {
+        /**
+         * Return a single Byte representation of this field's value.
+         */
+        public byte parseByte(String string);
+    }
+
+    /**
+     * Interface to parse shorts from document fields.
+     *
+     * @see FieldCache#getShorts(IndexReader, String, FieldCache.ShortParser)
+     */
+    public interface ShortParser extends Parser {
+        /**
+         * Return a short representation of this field's value.
+         */
+        public short parseShort(String string);
+    }
+
+    /**
+     * Interface to parse ints from document fields.
+     *
+     * @see FieldCache#getInts(IndexReader, String, FieldCache.IntParser)
+     */
+    public interface IntParser extends Parser {
+        /**
+         * Return an integer representation of this field's value.
+         */
+        public int parseInt(String string);
+    }
+
+    /**
+     * Interface to parse floats from document fields.
+     *
+     * @see FieldCache#getFloats(IndexReader, String, FieldCache.FloatParser)
+     */
+    public interface FloatParser extends Parser {
+        /**
+         * Return an float representation of this field's value.
+         */
+        public float parseFloat(String string);
+    }
+
+    /**
+     * Interface to parse long from document fields.
+     *
+     * @see FieldCache#getLongs(IndexReader, String, FieldCache.LongParser)
+     */
+    public interface LongParser extends Parser {
+        /**
+         * Return an long representation of this field's value.
+         */
+        public long parseLong(String string);
+    }
+
+    /**
+     * Interface to parse doubles from document fields.
+     *
+     * @see FieldCache#getDoubles(IndexReader, String, FieldCache.DoubleParser)
+     */
+    public interface DoubleParser extends Parser {
+        /**
+         * Return an long representation of this field's value.
+         */
+        public double parseDouble(String string);
+    }
+
+    /**
+     * Expert: The cache used internally by sorting and range query classes.
+     */
+    public static FieldCache DEFAULT = new FieldCacheImpl();
+
+    /**
+     * The default parser for byte values, which are encoded by {@link Byte#toString(byte)}
+     */
+    public static final ByteParser DEFAULT_BYTE_PARSER = new ByteParser() {
+        public byte parseByte(String value) {
+            return Byte.parseByte(value);
+        }
+
+        protected Object readResolve() {
+            return DEFAULT_BYTE_PARSER;
+        }
+
+        @Override
+        public String toString() {
+            return FieldCache.class.getName() + ".DEFAULT_BYTE_PARSER";
+        }
+    };
+
+    /**
+     * The default parser for short values, which are encoded by {@link Short#toString(short)}
+     */
+    public static final ShortParser DEFAULT_SHORT_PARSER = new ShortParser() {
+        public short parseShort(String value) {
+            return Short.parseShort(value);
+        }
+
+        protected Object readResolve() {
+            return DEFAULT_SHORT_PARSER;
+        }
+
+        @Override
+        public String toString() {
+            return FieldCache.class.getName() + ".DEFAULT_SHORT_PARSER";
+        }
+    };
+
+    /**
+     * The default parser for int values, which are encoded by {@link Integer#toString(int)}
+     */
+    public static final IntParser DEFAULT_INT_PARSER = new IntParser() {
+        public int parseInt(String value) {
+            return Integer.parseInt(value);
+        }
+
+        protected Object readResolve() {
+            return DEFAULT_INT_PARSER;
+        }
+
+        @Override
+        public String toString() {
+            return FieldCache.class.getName() + ".DEFAULT_INT_PARSER";
+        }
+    };
+
+    /**
+     * The default parser for float values, which are encoded by {@link Float#toString(float)}
+     */
+    public static final FloatParser DEFAULT_FLOAT_PARSER = new FloatParser() {
+        public float parseFloat(String value) {
+            return Float.parseFloat(value);
+        }
+
+        protected Object readResolve() {
+            return DEFAULT_FLOAT_PARSER;
+        }
+
+        @Override
+        public String toString() {
+            return FieldCache.class.getName() + ".DEFAULT_FLOAT_PARSER";
+        }
+    };
+
+    /**
+     * The default parser for long values, which are encoded by {@link Long#toString(long)}
+     */
+    public static final LongParser DEFAULT_LONG_PARSER = new LongParser() {
+        public long parseLong(String value) {
+            return Long.parseLong(value);
+        }
+
+        protected Object readResolve() {
+            return DEFAULT_LONG_PARSER;
+        }
+
+        @Override
+        public String toString() {
+            return FieldCache.class.getName() + ".DEFAULT_LONG_PARSER";
+        }
+    };
+
+    /**
+     * The default parser for double values, which are encoded by {@link Double#toString(double)}
+     */
+    public static final DoubleParser DEFAULT_DOUBLE_PARSER = new DoubleParser() {
+        public double parseDouble(String value) {
+            return Double.parseDouble(value);
+        }
+
+        protected Object readResolve() {
+            return DEFAULT_DOUBLE_PARSER;
+        }
+
+        @Override
+        public String toString() {
+            return FieldCache.class.getName() + ".DEFAULT_DOUBLE_PARSER";
+        }
+    };
+
+    /**
+     * A parser instance for int values encoded by {@link NumericUtils#intToPrefixCoded(int)}, e.g. when indexed
+     * via {@link NumericField}/{@link NumericTokenStream}.
+     */
+    public static final IntParser NUMERIC_UTILS_INT_PARSER = new IntParser() {
+        public int parseInt(String val) {
+            final int shift = val.charAt(0) - NumericUtils.SHIFT_START_INT;
+            if (shift > 0 && shift <= 31)
+                throw new FieldCacheImpl.StopFillCacheException();
+            return NumericUtils.prefixCodedToInt(val);
+        }
+
+        protected Object readResolve() {
+            return NUMERIC_UTILS_INT_PARSER;
+        }
+
+        @Override
+        public String toString() {
+            return FieldCache.class.getName() + ".NUMERIC_UTILS_INT_PARSER";
+        }
+    };
+
+    /**
+     * A parser instance for float values encoded with {@link NumericUtils}, e.g. when indexed
+     * via {@link NumericField}/{@link NumericTokenStream}.
+     */
+    public static final FloatParser NUMERIC_UTILS_FLOAT_PARSER = new FloatParser() {
+        public float parseFloat(String val) {
+            final int shift = val.charAt(0) - NumericUtils.SHIFT_START_INT;
+            if (shift > 0 && shift <= 31)
+                throw new FieldCacheImpl.StopFillCacheException();
+            return NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(val));
+        }
+
+        protected Object readResolve() {
+            return NUMERIC_UTILS_FLOAT_PARSER;
+        }
+
+        @Override
+        public String toString() {
+            return FieldCache.class.getName() + ".NUMERIC_UTILS_FLOAT_PARSER";
+        }
+    };
+
+    /**
+     * A parser instance for long values encoded by {@link NumericUtils#longToPrefixCoded(long)}, e.g. when indexed
+     * via {@link NumericField}/{@link NumericTokenStream}.
+     */
+    public static final LongParser NUMERIC_UTILS_LONG_PARSER = new LongParser() {
+        public long parseLong(String val) {
+            final int shift = val.charAt(0) - NumericUtils.SHIFT_START_LONG;
+            if (shift > 0 && shift <= 63)
+                throw new FieldCacheImpl.StopFillCacheException();
+            return NumericUtils.prefixCodedToLong(val);
+        }
+
+        protected Object readResolve() {
+            return NUMERIC_UTILS_LONG_PARSER;
+        }
+
+        @Override
+        public String toString() {
+            return FieldCache.class.getName() + ".NUMERIC_UTILS_LONG_PARSER";
+        }
+    };
+
+    /**
+     * A parser instance for double values encoded with {@link NumericUtils}, e.g. when indexed
+     * via {@link NumericField}/{@link NumericTokenStream}.
+     */
+    public static final DoubleParser NUMERIC_UTILS_DOUBLE_PARSER = new DoubleParser() {
+        public double parseDouble(String val) {
+            final int shift = val.charAt(0) - NumericUtils.SHIFT_START_LONG;
+            if (shift > 0 && shift <= 63)
+                throw new FieldCacheImpl.StopFillCacheException();
+            return NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(val));
+        }
+
+        protected Object readResolve() {
+            return NUMERIC_UTILS_DOUBLE_PARSER;
+        }
+
+        @Override
+        public String toString() {
+            return FieldCache.class.getName() + ".NUMERIC_UTILS_DOUBLE_PARSER";
+        }
+    };
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if none is
+     * found, reads the terms in <code>field</code> as a single byte and returns an array
+     * of size <code>reader.maxDoc()</code> of the value each document
+     * has in the given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the single byte values.
+     * @return The values in the given field for each document.
+     * @throws IOException If any error occurs.
+     */
+    public byte[] getBytes(IndexReader reader, String field)
+            throws IOException;
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if none is found,
+     * reads the terms in <code>field</code> as bytes and returns an array of
+     * size <code>reader.maxDoc()</code> of the value each document has in the
+     * given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the bytes.
+     * @param parser Computes byte for string values.
+     * @return The values in the given field for each document.
+     * @throws IOException If any error occurs.
+     */
+    public byte[] getBytes(IndexReader reader, String field, ByteParser parser)
+            throws IOException;
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if none is
+     * found, reads the terms in <code>field</code> as shorts and returns an array
+     * of size <code>reader.maxDoc()</code> of the value each document
+     * has in the given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the shorts.
+     * @return The values in the given field for each document.
+     * @throws IOException If any error occurs.
+     */
+    public short[] getShorts(IndexReader reader, String field)
+            throws IOException;
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if none is found,
+     * reads the terms in <code>field</code> as shorts and returns an array of
+     * size <code>reader.maxDoc()</code> of the value each document has in the
+     * given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the shorts.
+     * @param parser Computes short for string values.
+     * @return The values in the given field for each document.
+     * @throws IOException If any error occurs.
+     */
+    public short[] getShorts(IndexReader reader, String field, ShortParser parser)
+            throws IOException;
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if none is
+     * found, reads the terms in <code>field</code> as integers and returns an array
+     * of size <code>reader.maxDoc()</code> of the value each document
+     * has in the given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the integers.
+     * @return The values in the given field for each document.
+     * @throws IOException If any error occurs.
+     */
+    public int[] getInts(IndexReader reader, String field)
+            throws IOException;
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if none is found,
+     * reads the terms in <code>field</code> as integers and returns an array of
+     * size <code>reader.maxDoc()</code> of the value each document has in the
+     * given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the integers.
+     * @param parser Computes integer for string values.
+     * @return The values in the given field for each document.
+     * @throws IOException If any error occurs.
+     */
+    public int[] getInts(IndexReader reader, String field, IntParser parser)
+            throws IOException;
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if
+     * none is found, reads the terms in <code>field</code> as floats and returns an array
+     * of size <code>reader.maxDoc()</code> of the value each document
+     * has in the given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the floats.
+     * @return The values in the given field for each document.
+     * @throws IOException If any error occurs.
+     */
+    public float[] getFloats(IndexReader reader, String field)
+            throws IOException;
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if
+     * none is found, reads the terms in <code>field</code> as floats and returns an array
+     * of size <code>reader.maxDoc()</code> of the value each document
+     * has in the given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the floats.
+     * @param parser Computes float for string values.
+     * @return The values in the given field for each document.
+     * @throws IOException If any error occurs.
+     */
+    public float[] getFloats(IndexReader reader, String field,
+                             FloatParser parser) throws IOException;
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if none is
+     * found, reads the terms in <code>field</code> as longs and returns an array
+     * of size <code>reader.maxDoc()</code> of the value each document
+     * has in the given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the longs.
+     * @return The values in the given field for each document.
+     * @throws java.io.IOException If any error occurs.
+     */
+    public long[] getLongs(IndexReader reader, String field)
+            throws IOException;
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if none is found,
+     * reads the terms in <code>field</code> as longs and returns an array of
+     * size <code>reader.maxDoc()</code> of the value each document has in the
+     * given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the longs.
+     * @param parser Computes integer for string values.
+     * @return The values in the given field for each document.
+     * @throws IOException If any error occurs.
+     */
+    public long[] getLongs(IndexReader reader, String field, LongParser parser)
+            throws IOException;
+
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if none is
+     * found, reads the terms in <code>field</code> as integers and returns an array
+     * of size <code>reader.maxDoc()</code> of the value each document
+     * has in the given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the doubles.
+     * @return The values in the given field for each document.
+     * @throws IOException If any error occurs.
+     */
+    public double[] getDoubles(IndexReader reader, String field)
+            throws IOException;
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if none is found,
+     * reads the terms in <code>field</code> as doubles and returns an array of
+     * size <code>reader.maxDoc()</code> of the value each document has in the
+     * given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the doubles.
+     * @param parser Computes integer for string values.
+     * @return The values in the given field for each document.
+     * @throws IOException If any error occurs.
+     */
+    public double[] getDoubles(IndexReader reader, String field, DoubleParser parser)
+            throws IOException;
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if none
+     * is found, reads the term values in <code>field</code> and returns an array
+     * of size <code>reader.maxDoc()</code> containing the value each document
+     * has in the given field.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the strings.
+     * @return The values in the given field for each document.
+     * @throws IOException If any error occurs.
+     */
+    public String[] getStrings(IndexReader reader, String field)
+            throws IOException;
+
+    /**
+     * Checks the internal cache for an appropriate entry, and if none
+     * is found reads the term values in <code>field</code> and returns
+     * an array of them in natural order, along with an array telling
+     * which element in the term array each document uses.
+     *
+     * @param reader Used to get field values.
+     * @param field  Which field contains the strings.
+     * @return Array of terms and index into the array for each document.
+     * @throws IOException If any error occurs.
+     */
+    public StringIndex getStringIndex(IndexReader reader, String field)
+            throws IOException;
+
+    /**
+     * EXPERT: A unique Identifier/Description for each item in the FieldCache.
+     * Can be useful for logging/debugging.
+     * <p>
+     * <b>EXPERIMENTAL API:</b> This API is considered extremely advanced
+     * and experimental.  It may be removed or altered w/o warning in future
+     * releases
+     * of Lucene.
+     * </p>
+     */
+    public static abstract class CacheEntry {
+        public abstract Object getReaderKey();
+
+        public abstract String getFieldName();
+
+        public abstract Class getCacheType();
+
+        public abstract Object getCustom();
+
+        public abstract Object getValue();
+
+        private String size = null;
+
+        protected final void setEstimatedSize(String size) {
+            this.size = size;
+        }
+
+        /**
+         * @see #estimateSize(RamUsageEstimator)
+         */
+        public void estimateSize() {
+            estimateSize(new RamUsageEstimator(false)); // doesn't check for interned
+        }
+
+        /**
+         * Computes (and stores) the estimated size of the cache Value
+         *
+         * @see #getEstimatedSize
+         */
+        public void estimateSize(RamUsageEstimator ramCalc) {
+            long size = ramCalc.estimateRamUsage(getValue());
+            setEstimatedSize(RamUsageEstimator.humanReadableUnits
+                    (size, new DecimalFormat("0.#")));
+
+        }
+
+        /**
+         * The most recently estimated size of the value, null unless
+         * estimateSize has been called.
+         */
+        public final String getEstimatedSize() {
+            return size;
+        }
+
+
+        @Override
+        public String toString() {
+            StringBuilder b = new StringBuilder();
+            b.append("'").append(getReaderKey()).append("'=>");
+            b.append("'").append(getFieldName()).append("',");
+            b.append(getCacheType()).append(",").append(getCustom());
+            b.append("=>").append(getValue().getClass().getName()).append("#");
+            b.append(System.identityHashCode(getValue()));
+
+            String s = getEstimatedSize();
+            if (null != s) {
+                b.append(" (size =~ ").append(s).append(')');
+            }
+
+            return b.toString();
+        }
+
+    }
+
+    /**
+     * EXPERT: Generates an array of CacheEntry objects representing all items
+     * currently in the FieldCache.
+     * <p>
+     * NOTE: These CacheEntry objects maintain a strong reference to the
+     * Cached Values.  Maintaining references to a CacheEntry the IndexReader
+     * associated with it has garbage collected will prevent the Value itself
+     * from being garbage collected when the Cache drops the WeakRefrence.
+     * </p>
+     * <p>
+     * <b>EXPERIMENTAL API:</b> This API is considered extremely advanced
+     * and experimental.  It may be removed or altered w/o warning in future
+     * releases
+     * of Lucene.
+     * </p>
+     */
+    public abstract CacheEntry[] getCacheEntries();
+
+    /**
+     * <p>
+     * EXPERT: Instructs the FieldCache to forcibly expunge all entries
+     * from the underlying caches.  This is intended only to be used for
+     * test methods as a way to ensure a known base state of the Cache
+     * (with out needing to rely on GC to free WeakReferences).
+     * It should not be relied on for "Cache maintenance" in general
+     * application code.
+     * </p>
+     * <p>
+     * <b>EXPERIMENTAL API:</b> This API is considered extremely advanced
+     * and experimental.  It may be removed or altered w/o warning in future
+     * releases
+     * of Lucene.
+     * </p>
+     */
+    public abstract void purgeAllCaches();
+
+    /**
+     * Expert: drops all cache entries associated with this
+     * reader.  NOTE: this reader must precisely match the
+     * reader that the cache entry is keyed on. If you pass a
+     * top-level reader, it usually will have no effect as
+     * Lucene now caches at the segment reader level.
+     */
+    public abstract void purge(IndexReader r);
+
+    /**
+     * If non-null, FieldCacheImpl will warn whenever
+     * entries are created that are not sane according to
+     * {@link org.apache.lucene.util.FieldCacheSanityChecker}.
+     */
+    public void setInfoStream(PrintStream stream);
+
+    /**
+     * counterpart of {@link #setInfoStream(PrintStream)}
+     */
+    public PrintStream getInfoStream();
+}

modules/elasticsearch/src/main/java/org/apache/lucene/search/FieldCacheImpl.java

@@ -0,0 +1,754 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.util.FieldCacheSanityChecker;
+import org.apache.lucene.util.StringHelper;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.*;
+
+/**
+ * Expert: The default cache implementation, storing all values in memory.
+ * A WeakHashMap is used for storage.
+ *
+ * <p>Created: May 19, 2004 4:40:36 PM
+ *
+ * @since lucene 1.4
+ */
+// LUCENE MONITOR - Added a listener for purge events
+class FieldCacheImpl implements FieldCache {
+
+    static {
+        if (Boolean.getBoolean("es.fieldcache.debug")) {
+            System.out.println("Using elasticsearch fieldcache");
+        }
+    }
+
+    private IndexReaderPurgedListener listener;
+
+    @Override public void setIndexReaderPurgedListener(IndexReaderPurgedListener listener) {
+        this.listener = listener;
+    }
+
+    private Map<Class<?>, Cache> caches;
+
+    FieldCacheImpl() {
+        init();
+    }
+
+    private synchronized void init() {
+        caches = new HashMap<Class<?>, Cache>(7);
+        caches.put(Byte.TYPE, new ByteCache(this));
+        caches.put(Short.TYPE, new ShortCache(this));
+        caches.put(Integer.TYPE, new IntCache(this));
+        caches.put(Float.TYPE, new FloatCache(this));
+        caches.put(Long.TYPE, new LongCache(this));
+        caches.put(Double.TYPE, new DoubleCache(this));
+        caches.put(String.class, new StringCache(this));
+        caches.put(StringIndex.class, new StringIndexCache(this));
+    }
+
+    public void purgeAllCaches() {
+        init();
+    }
+
+    public void purge(IndexReader r) {
+        if (listener != null) {
+            listener.indexReaderPurged(r);
+        }
+        for (Cache c : caches.values()) {
+            c.purge(r);
+        }
+    }
+
+    public CacheEntry[] getCacheEntries() {
+        List<CacheEntry> result = new ArrayList<CacheEntry>(17);
+        for (final Class<?> cacheType : caches.keySet()) {
+            Cache cache = caches.get(cacheType);
+            for (final Object readerKey : cache.readerCache.keySet()) {
+                // we've now materialized a hard ref
+
+                // innerKeys was backed by WeakHashMap, sanity check
+                // that it wasn't GCed before we made hard ref
+                if (null != readerKey && cache.readerCache.containsKey(readerKey)) {
+                    Map<Entry, Object> innerCache = cache.readerCache.get(readerKey);
+                    for (final Map.Entry<Entry, Object> mapEntry : innerCache.entrySet()) {
+                        Entry entry = mapEntry.getKey();
+                        result.add(new CacheEntryImpl(readerKey, entry.field,
+                                cacheType, entry.custom,
+                                mapEntry.getValue()));
+                    }
+                }
+            }
+        }
+        return result.toArray(new CacheEntry[result.size()]);
+    }
+
+    private static final class CacheEntryImpl extends CacheEntry {
+        private final Object readerKey;
+        private final String fieldName;
+        private final Class<?> cacheType;
+        private final Object custom;
+        private final Object value;
+
+        CacheEntryImpl(Object readerKey, String fieldName,
+                       Class<?> cacheType,
+                       Object custom,
+                       Object value) {
+            this.readerKey = readerKey;
+            this.fieldName = fieldName;
+            this.cacheType = cacheType;
+            this.custom = custom;
+            this.value = value;
+
+            // :HACK: for testing.
+//         if (null != locale || SortField.CUSTOM != sortFieldType) {
+//           throw new RuntimeException("Locale/sortFieldType: " + this);
+//         }
+
+        }
+
+        @Override
+        public Object getReaderKey() {
+            return readerKey;
+        }
+
+        @Override
+        public String getFieldName() {
+            return fieldName;
+        }
+
+        @Override
+        public Class<?> getCacheType() {
+            return cacheType;
+        }
+
+        @Override
+        public Object getCustom() {
+            return custom;
+        }
+
+        @Override
+        public Object getValue() {
+            return value;
+        }
+    }
+
+    /**
+     * Hack: When thrown from a Parser (NUMERIC_UTILS_* ones), this stops
+     * processing terms and returns the current FieldCache
+     * array.
+     */
+    static final class StopFillCacheException extends RuntimeException {
+    }
+
+    /**
+     * Expert: Internal cache.
+     */
+    abstract static class Cache {
+        Cache() {
+            this.wrapper = null;
+        }
+
+        Cache(FieldCache wrapper) {
+            this.wrapper = wrapper;
+        }
+
+        final FieldCache wrapper;
+
+        final Map<Object, Map<Entry, Object>> readerCache = new WeakHashMap<Object, Map<Entry, Object>>();
+
+        protected abstract Object createValue(IndexReader reader, Entry key)
+                throws IOException;
+
+        /**
+         * Remove this reader from the cache, if present.
+         */
+        public void purge(IndexReader r) {
+            Object readerKey = r.getFieldCacheKey();
+            synchronized (readerCache) {
+                readerCache.remove(readerKey);
+            }
+        }
+
+        public Object get(IndexReader reader, Entry key) throws IOException {
+            Map<Entry, Object> innerCache;
+            Object value;
+            final Object readerKey = reader.getFieldCacheKey();
+            synchronized (readerCache) {
+                innerCache = readerCache.get(readerKey);
+                if (innerCache == null) {
+                    innerCache = new HashMap<Entry, Object>();
+                    readerCache.put(readerKey, innerCache);
+                    value = null;
+                } else {
+                    value = innerCache.get(key);
+                }
+                if (value == null) {
+                    value = new CreationPlaceholder();
+                    innerCache.put(key, value);
+                }
+            }
+            if (value instanceof CreationPlaceholder) {
+                synchronized (value) {
+                    CreationPlaceholder progress = (CreationPlaceholder) value;
+                    if (progress.value == null) {
+                        progress.value = createValue(reader, key);
+                        synchronized (readerCache) {
+                            innerCache.put(key, progress.value);
+                        }
+
+                        // Only check if key.custom (the parser) is
+                        // non-null; else, we check twice for a single
+                        // call to FieldCache.getXXX
+                        if (key.custom != null && wrapper != null) {
+                            final PrintStream infoStream = wrapper.getInfoStream();
+                            if (infoStream != null) {
+                                printNewInsanity(infoStream, progress.value);
+                            }
+                        }
+                    }
+                    return progress.value;
+                }
+            }
+            return value;
+        }
+
+        private void printNewInsanity(PrintStream infoStream, Object value) {
+            final FieldCacheSanityChecker.Insanity[] insanities = FieldCacheSanityChecker.checkSanity(wrapper);
+            for (int i = 0; i < insanities.length; i++) {
+                final FieldCacheSanityChecker.Insanity insanity = insanities[i];
+                final CacheEntry[] entries = insanity.getCacheEntries();
+                for (int j = 0; j < entries.length; j++) {
+                    if (entries[j].getValue() == value) {
+                        // OK this insanity involves our entry
+                        infoStream.println("WARNING: new FieldCache insanity created\nDetails: " + insanity.toString());
+                        infoStream.println("\nStack:\n");
+                        new Throwable().printStackTrace(infoStream);
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Expert: Every composite-key in the internal cache is of this type.
+     */
+    static class Entry {
+        final String field;        // which Fieldable
+        final Object custom;       // which custom comparator or parser
+
+        /**
+         * Creates one of these objects for a custom comparator/parser.
+         */
+        Entry(String field, Object custom) {
+            this.field = StringHelper.intern(field);
+            this.custom = custom;
+        }
+
+        /**
+         * Two of these are equal iff they reference the same field and type.
+         */
+        @Override
+        public boolean equals(Object o) {
+            if (o instanceof Entry) {
+                Entry other = (Entry) o;
+                if (other.field == field) {
+                    if (other.custom == null) {
+                        if (custom == null) return true;
+                    } else if (other.custom.equals(custom)) {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }
+
+        /**
+         * Composes a hashcode based on the field and type.
+         */
+        @Override
+        public int hashCode() {
+            return field.hashCode() ^ (custom == null ? 0 : custom.hashCode());
+        }
+    }
+
+    // inherit javadocs
+
+    public byte[] getBytes(IndexReader reader, String field) throws IOException {
+        return getBytes(reader, field, null);
+    }
+
+    // inherit javadocs
+
+    public byte[] getBytes(IndexReader reader, String field, ByteParser parser)
+            throws IOException {
+        return (byte[]) caches.get(Byte.TYPE).get(reader, new Entry(field, parser));
+    }
+
+    static final class ByteCache extends Cache {
+        ByteCache(FieldCache wrapper) {
+            super(wrapper);
+        }
+
+        @Override
+        protected Object createValue(IndexReader reader, Entry entryKey)
+                throws IOException {
+            Entry entry = entryKey;
+            String field = entry.field;
+            ByteParser parser = (ByteParser) entry.custom;
+            if (parser == null) {
+                return wrapper.getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER);
+            }
+            final byte[] retArray = new byte[reader.maxDoc()];
+            TermDocs termDocs = reader.termDocs();
+            TermEnum termEnum = reader.terms(new Term(field));
+            try {
+                do {
+                    Term term = termEnum.term();
+                    if (term == null || term.field() != field) break;
+                    byte termval = parser.parseByte(term.text());
+                    termDocs.seek(termEnum);
+                    while (termDocs.next()) {
+                        retArray[termDocs.doc()] = termval;
+                    }
+                } while (termEnum.next());
+            } catch (StopFillCacheException stop) {
+            } finally {
+                termDocs.close();
+                termEnum.close();
+            }
+            return retArray;
+        }
+    }
+
+    ;
+
+    // inherit javadocs
+
+    public short[] getShorts(IndexReader reader, String field) throws IOException {
+        return getShorts(reader, field, null);
+    }
+
+    // inherit javadocs
+
+    public short[] getShorts(IndexReader reader, String field, ShortParser parser)
+            throws IOException {
+        return (short[]) caches.get(Short.TYPE).get(reader, new Entry(field, parser));
+    }
+
+    static final class ShortCache extends Cache {
+        ShortCache(FieldCache wrapper) {
+            super(wrapper);
+        }
+
+        @Override
+        protected Object createValue(IndexReader reader, Entry entryKey)
+                throws IOException {
+            Entry entry = entryKey;
+            String field = entry.field;
+            ShortParser parser = (ShortParser) entry.custom;
+            if (parser == null) {
+                return wrapper.getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER);
+            }
+            final short[] retArray = new short[reader.maxDoc()];
+            TermDocs termDocs = reader.termDocs();
+            TermEnum termEnum = reader.terms(new Term(field));
+            try {
+                do {
+                    Term term = termEnum.term();
+                    if (term == null || term.field() != field) break;
+                    short termval = parser.parseShort(term.text());
+                    termDocs.seek(termEnum);
+                    while (termDocs.next()) {
+                        retArray[termDocs.doc()] = termval;
+                    }
+                } while (termEnum.next());
+            } catch (StopFillCacheException stop) {
+            } finally {
+                termDocs.close();
+                termEnum.close();
+            }
+            return retArray;
+        }
+    }
+
+    ;
+
+    // inherit javadocs
+
+    public int[] getInts(IndexReader reader, String field) throws IOException {
+        return getInts(reader, field, null);
+    }
+
+    // inherit javadocs
+
+    public int[] getInts(IndexReader reader, String field, IntParser parser)
+            throws IOException {
+        return (int[]) caches.get(Integer.TYPE).get(reader, new Entry(field, parser));
+    }
+
+    static final class IntCache extends Cache {
+        IntCache(FieldCache wrapper) {
+            super(wrapper);
+        }
+
+        @Override
+        protected Object createValue(IndexReader reader, Entry entryKey)
+                throws IOException {
+            Entry entry = entryKey;
+            String field = entry.field;
+            IntParser parser = (IntParser) entry.custom;
+            if (parser == null) {
+                try {
+                    return wrapper.getInts(reader, field, DEFAULT_INT_PARSER);
+                } catch (NumberFormatException ne) {
+                    return wrapper.getInts(reader, field, NUMERIC_UTILS_INT_PARSER);
+                }
+            }
+            int[] retArray = null;
+            TermDocs termDocs = reader.termDocs();
+            TermEnum termEnum = reader.terms(new Term(field));
+            try {
+                do {
+                    Term term = termEnum.term();
+                    if (term == null || term.field() != field) break;
+                    int termval = parser.parseInt(term.text());
+                    if (retArray == null) // late init
+                        retArray = new int[reader.maxDoc()];
+                    termDocs.seek(termEnum);
+                    while (termDocs.next()) {
+                        retArray[termDocs.doc()] = termval;
+                    }
+                } while (termEnum.next());
+            } catch (StopFillCacheException stop) {
+            } finally {
+                termDocs.close();
+                termEnum.close();
+            }
+            if (retArray == null) // no values
+                retArray = new int[reader.maxDoc()];
+            return retArray;
+        }
+    }
+
+    ;
+
+
+    // inherit javadocs
+
+    public float[] getFloats(IndexReader reader, String field)
+            throws IOException {
+        return getFloats(reader, field, null);
+    }
+
+    // inherit javadocs
+
+    public float[] getFloats(IndexReader reader, String field, FloatParser parser)
+            throws IOException {
+
+        return (float[]) caches.get(Float.TYPE).get(reader, new Entry(field, parser));
+    }
+
+    static final class FloatCache extends Cache {
+        FloatCache(FieldCache wrapper) {
+            super(wrapper);
+        }
+
+        @Override
+        protected Object createValue(IndexReader reader, Entry entryKey)
+                throws IOException {
+            Entry entry = entryKey;
+            String field = entry.field;
+            FloatParser parser = (FloatParser) entry.custom;
+            if (parser == null) {
+                try {
+                    return wrapper.getFloats(reader, field, DEFAULT_FLOAT_PARSER);
+                } catch (NumberFormatException ne) {
+                    return wrapper.getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER);
+                }
+            }
+            float[] retArray = null;
+            TermDocs termDocs = reader.termDocs();
+            TermEnum termEnum = reader.terms(new Term(field));
+            try {
+                do {
+                    Term term = termEnum.term();
+                    if (term == null || term.field() != field) break;
+                    float termval = parser.parseFloat(term.text());
+                    if (retArray == null) // late init
+                        retArray = new float[reader.maxDoc()];
+                    termDocs.seek(termEnum);
+                    while (termDocs.next()) {
+                        retArray[termDocs.doc()] = termval;
+                    }
+                } while (termEnum.next());
+            } catch (StopFillCacheException stop) {
+            } finally {
+                termDocs.close();
+                termEnum.close();
+            }
+            if (retArray == null) // no values
+                retArray = new float[reader.maxDoc()];
+            return retArray;
+        }
+    }
+
+    ;
+
+
+    public long[] getLongs(IndexReader reader, String field) throws IOException {
+        return getLongs(reader, field, null);
+    }
+
+    // inherit javadocs
+
+    public long[] getLongs(IndexReader reader, String field, FieldCache.LongParser parser)
+            throws IOException {
+        return (long[]) caches.get(Long.TYPE).get(reader, new Entry(field, parser));
+    }
+
+    static final class LongCache extends Cache {
+        LongCache(FieldCache wrapper) {
+            super(wrapper);
+        }
+
+        @Override
+        protected Object createValue(IndexReader reader, Entry entry)
+                throws IOException {
+            String field = entry.field;
+            FieldCache.LongParser parser = (FieldCache.LongParser) entry.custom;
+            if (parser == null) {
+                try {
+                    return wrapper.getLongs(reader, field, DEFAULT_LONG_PARSER);
+                } catch (NumberFormatException ne) {
+                    return wrapper.getLongs(reader, field, NUMERIC_UTILS_LONG_PARSER);
+                }
+            }
+            long[] retArray = null;
+            TermDocs termDocs = reader.termDocs();
+            TermEnum termEnum = reader.terms(new Term(field));
+            try {
+                do {
+                    Term term = termEnum.term();
+                    if (term == null || term.field() != field) break;
+                    long termval = parser.parseLong(term.text());
+                    if (retArray == null) // late init
+                        retArray = new long[reader.maxDoc()];
+                    termDocs.seek(termEnum);
+                    while (termDocs.next()) {
+                        retArray[termDocs.doc()] = termval;
+                    }
+                } while (termEnum.next());
+            } catch (StopFillCacheException stop) {
+            } finally {
+                termDocs.close();
+                termEnum.close();
+            }
+            if (retArray == null) // no values
+                retArray = new long[reader.maxDoc()];
+            return retArray;
+        }
+    }
+
+    ;
+
+    // inherit javadocs
+
+    public double[] getDoubles(IndexReader reader, String field)
+            throws IOException {
+        return getDoubles(reader, field, null);
+    }
+
+    // inherit javadocs
+
+    public double[] getDoubles(IndexReader reader, String field, FieldCache.DoubleParser parser)
+            throws IOException {
+        return (double[]) caches.get(Double.TYPE).get(reader, new Entry(field, parser));
+    }
+
+    static final class DoubleCache extends Cache {
+        DoubleCache(FieldCache wrapper) {
+            super(wrapper);
+        }
+
+        @Override
+        protected Object createValue(IndexReader reader, Entry entryKey)
+                throws IOException {
+            Entry entry = entryKey;
+            String field = entry.field;
+            FieldCache.DoubleParser parser = (FieldCache.DoubleParser) entry.custom;
+            if (parser == null) {
+                try {
+                    return wrapper.getDoubles(reader, field, DEFAULT_DOUBLE_PARSER);
+                } catch (NumberFormatException ne) {
+                    return wrapper.getDoubles(reader, field, NUMERIC_UTILS_DOUBLE_PARSER);
+                }
+            }
+            double[] retArray = null;
+            TermDocs termDocs = reader.termDocs();
+            TermEnum termEnum = reader.terms(new Term(field));
+            try {
+                do {
+                    Term term = termEnum.term();
+                    if (term == null || term.field() != field) break;
+                    double termval = parser.parseDouble(term.text());
+                    if (retArray == null) // late init
+                        retArray = new double[reader.maxDoc()];
+                    termDocs.seek(termEnum);
+                    while (termDocs.next()) {
+                        retArray[termDocs.doc()] = termval;
+                    }
+                } while (termEnum.next());
+            } catch (StopFillCacheException stop) {
+            } finally {
+                termDocs.close();
+                termEnum.close();
+            }
+            if (retArray == null) // no values
+                retArray = new double[reader.maxDoc()];
+            return retArray;
+        }
+    }
+
+    ;
+
+    // inherit javadocs
+
+    public String[] getStrings(IndexReader reader, String field)
+            throws IOException {
+        return (String[]) caches.get(String.class).get(reader, new Entry(field, (Parser) null));
+    }
+
+    static final class StringCache extends Cache {
+        StringCache(FieldCache wrapper) {
+            super(wrapper);
+        }
+
+        @Override
+        protected Object createValue(IndexReader reader, Entry entryKey)
+                throws IOException {
+            String field = StringHelper.intern(entryKey.field);
+            final String[] retArray = new String[reader.maxDoc()];
+            TermDocs termDocs = reader.termDocs();
+            TermEnum termEnum = reader.terms(new Term(field));
+            try {
+                do {
+                    Term term = termEnum.term();
+                    if (term == null || term.field() != field) break;
+                    String termval = term.text();
+                    termDocs.seek(termEnum);
+                    while (termDocs.next()) {
+                        retArray[termDocs.doc()] = termval;
+                    }
+                } while (termEnum.next());
+            } finally {
+                termDocs.close();
+                termEnum.close();
+            }
+            return retArray;
+        }
+    }
+
+    ;
+
+    // inherit javadocs
+
+    public StringIndex getStringIndex(IndexReader reader, String field)
+            throws IOException {
+        return (StringIndex) caches.get(StringIndex.class).get(reader, new Entry(field, (Parser) null));
+    }
+
+    static final class StringIndexCache extends Cache {
+        StringIndexCache(FieldCache wrapper) {
+            super(wrapper);
+        }
+
+        @Override
+        protected Object createValue(IndexReader reader, Entry entryKey)
+                throws IOException {
+            String field = StringHelper.intern(entryKey.field);
+            final int[] retArray = new int[reader.maxDoc()];
+            String[] mterms = new String[reader.maxDoc() + 1];
+            TermDocs termDocs = reader.termDocs();
+            TermEnum termEnum = reader.terms(new Term(field));
+            int t = 0;  // current term number
+
+            // an entry for documents that have no terms in this field
+            // should a document with no terms be at top or bottom?
+            // this puts them at the top - if it is changed, FieldDocSortedHitQueue
+            // needs to change as well.
+            mterms[t++] = null;
+
+            try {
+                do {
+                    Term term = termEnum.term();
+                    if (term == null || term.field() != field) break;
+
+                    // store term text
+                    mterms[t] = term.text();
+
+                    termDocs.seek(termEnum);
+                    while (termDocs.next()) {
+                        retArray[termDocs.doc()] = t;
+                    }
+
+                    t++;
+                } while (termEnum.next());
+            } finally {
+                termDocs.close();
+                termEnum.close();
+            }
+
+            if (t == 0) {
+                // if there are no terms, make the term array
+                // have a single null entry
+                mterms = new String[1];
+            } else if (t < mterms.length) {
+                // if there are less terms than documents,
+                // trim off the dead array space
+                String[] terms = new String[t];
+                System.arraycopy(mterms, 0, terms, 0, t);
+                mterms = terms;
+            }
+
+            StringIndex value = new StringIndex(retArray, mterms);
+            return value;
+        }
+    }
+
+    ;
+
+    private volatile PrintStream infoStream;
+
+    public void setInfoStream(PrintStream stream) {
+        infoStream = stream;
+    }
+
+    public PrintStream getInfoStream() {
+        return infoStream;
+    }
+}
+

modules/elasticsearch/src/main/java/org/apache/lucene/search/IndexReaderPurgedListener.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.search;
+
+import org.apache.lucene.index.IndexReader;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public interface IndexReaderPurgedListener {
+
+    void indexReaderPurged(IndexReader reader);
+}

modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCacheModule.java

@@ -22,10 +22,10 @@ package org.elasticsearch.index.cache.field.data;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Scopes;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.cache.field.data.weak.WeakFieldDataCache;
+import org.elasticsearch.index.cache.field.data.soft.SoftFieldDataCache;
 
 /**
- * @author kimchy (Shay Banon)
+ * @author kimchy (shay.banon)
  */
 public class FieldDataCacheModule extends AbstractModule {
 
@@ -41,7 +41,7 @@ public class FieldDataCacheModule extends AbstractModule {
 
     @Override protected void configure() {
         bind(FieldDataCache.class)
-                .to(settings.getAsClass(FieldDataCacheSettings.FIELD_DATA_CACHE_TYPE, WeakFieldDataCache.class, "org.elasticsearch.index.cache.field.", "FieldDataCache"))
+                .to(settings.getAsClass(FieldDataCacheSettings.FIELD_DATA_CACHE_TYPE, SoftFieldDataCache.class, "org.elasticsearch.index.cache.field.", "FieldDataCache"))
                 .in(Scopes.SINGLETON);
     }
 }

modules/elasticsearch/src/main/java/org/elasticsearch/indices/InternalIndicesService.java

@@ -19,6 +19,9 @@
 
 package org.elasticsearch.indices;
 
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.IndexReaderPurgedListener;
 import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
@@ -93,6 +96,16 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
         this.injector = injector;
 
         this.pluginsService = injector.getInstance(PluginsService.class);
+
+        try {
+            FieldCache.DEFAULT.getClass().getMethod("setIndexReaderPurgedListener", IndexReaderPurgedListener.class);
+            // LUCENE MONITOR - This is a hack to eagerly clean caches based on index reader
+            FieldCache.DEFAULT.setIndexReaderPurgedListener(new CacheReaderPurgeListener(this));
+            logger.trace("eager reader based cache eviction enabled");
+        } catch (NoSuchMethodException e) {
+            // no method
+            logger.debug("lucene default FieldCache is used, not enabling eager reader based cache eviction");
+        }
     }
 
     @Override protected void doStart() throws ElasticSearchException {
@@ -249,4 +262,19 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
 
         indicesLifecycle.afterIndexClosed(indexService.index(), delete);
     }
+
+    private static class CacheReaderPurgeListener implements IndexReaderPurgedListener {
+
+        private final IndicesService indicesService;
+
+        private CacheReaderPurgeListener(IndicesService indicesService) {
+            this.indicesService = indicesService;
+        }
+
+        @Override public void indexReaderPurged(IndexReader reader) {
+            for (IndexService indexService : indicesService) {
+                indexService.cache().clear(reader);
+            }
+        }
+    }
 }