SOLR-3985: Allow ExternalFileField caches to be reloaded on newSearcher/firstSearcher events

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1402463 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -49,6 +49,9 @@ New Features
   underlying analyzed form used for suggestions is separate from the returned
   text.  (Robert Muir)
 
+* SOLR-3985: ExternalFileField caches can be reloaded on firstSearcher/
+  newSearcher events using the ExternalFileFieldReloader (Alan Woodward)
+
 Optimizations
 ----------------------
 

solr/core/src/java/org/apache/solr/schema/ExternalFileField.java

@@ -16,18 +16,16 @@
  */
 package org.apache.solr.schema;
 
+import org.apache.lucene.index.StorableField;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.index.GeneralField;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.StorableField;
-import org.apache.solr.search.function.FileFloatSource;
-import org.apache.solr.search.QParser;
-import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.response.TextResponseWriter;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.function.FileFloatSource;
 
-import java.util.Map;
 import java.io.IOException;
+import java.util.Map;
 
 /** Get values from an external file instead of the index.
  *
@@ -55,7 +53,7 @@ import java.io.IOException;
  * <p/>The external file may be sorted or unsorted by the key field, but it will be substantially slower (untested) if it isn't sorted.
  * <p/>Fields of this type may currently only be used as a ValueSource in a FunctionQuery.
  *
- *
+ * @see ExternalFileFieldReloader
  */
 public class ExternalFileField extends FieldType {
   private FieldType ftype;
@@ -94,10 +92,26 @@ public class ExternalFileField extends FieldType {
 
   @Override
   public ValueSource getValueSource(SchemaField field, QParser parser) {
-    // default key field to unique key
-    SchemaField keyField = keyFieldName==null ? schema.getUniqueKeyField() : schema.getField(keyFieldName);
-    return new FileFloatSource(field, keyField, defVal, parser);
+    return getFileFloatSource(field, parser.getReq().getCore().getDataDir());
   }
 
+  /**
+   * Get a FileFloatSource for the given field, looking in datadir for the relevant file
+   * @param field the field to get a source for
+   * @param datadir the data directory in which to look for the external file
+   * @return a FileFloatSource
+   */
+  public FileFloatSource getFileFloatSource(SchemaField field, String datadir) {
+    // Because the float source uses a static cache, all source objects will
+    // refer to the same data.
+    return new FileFloatSource(field, getKeyField(), defVal, datadir);
+  }
+
+  // If no key field is defined, we use the unique key field
+  private SchemaField getKeyField() {
+    return keyFieldName == null ?
+        schema.getUniqueKeyField() :
+        schema.getField(keyFieldName);
+  }
 
 }

solr/core/src/java/org/apache/solr/schema/ExternalFileFieldReloader.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.schema;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.AbstractSolrEventListener;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.function.FileFloatSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An event listener to reload ExternalFileFields for new searchers.
+ *
+ * Opening a new IndexSearcher will invalidate the internal caches used by
+ * {@link ExternalFileField}.  By default, these caches are reloaded lazily
+ * by the first search that uses them.  For large external files, this can
+ * slow down searches unacceptably.
+ *
+ * To reload the caches when the searcher is first opened, set up event
+ * listeners in your solrconfig.xml:
+ *
+ * <pre>
+ *   &lt;listener event="newSearcher" class="org.apache.solr.schema.ExternalFileFieldReloader"/>
+ *   &lt;listener event="firstSearcher" class="org.apache.solr.schema.ExternalFileFieldReloader"/>
+ * </pre>
+ *
+ * The caches will be reloaded for all ExternalFileFields in your schema after
+ * each commit.
+ */
+public class ExternalFileFieldReloader extends AbstractSolrEventListener {
+
+  private IndexSchema schema;
+  private String datadir;
+  private List<FileFloatSource> fieldSources = new ArrayList<FileFloatSource>();
+
+  private static final Logger log = LoggerFactory.getLogger(ExternalFileFieldReloader.class);
+
+  public ExternalFileFieldReloader(SolrCore core) {
+    super(core);
+    schema = core.getSchema();
+    datadir = core.getDataDir();
+  }
+
+  @Override
+  public void init(NamedList args) {
+    for (SchemaField field : schema.getFields().values()) {
+      FieldType type = field.getType();
+      if (type instanceof ExternalFileField) {
+        ExternalFileField eff = (ExternalFileField) type;
+        fieldSources.add(eff.getFileFloatSource(field, datadir));
+        log.info("Adding ExternalFileFieldReloader listener for field {}", field.getName());
+      }
+    }
+  }
+
+  @Override
+  public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher) {
+    // We need to reload the caches for the new searcher
+    IndexReader reader = newSearcher.getIndexReader();
+    for (FileFloatSource fieldSource : fieldSources) {
+      fieldSource.refreshCache(reader);
+    }
+  }
+}
+

solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java

@@ -16,24 +16,7 @@
  */
 package org.apache.solr.search.function;
 
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.WeakHashMap;
-
-import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultiFields;
-import org.apache.lucene.index.IndexReaderContext;
-import org.apache.lucene.index.ReaderUtil;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.*;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.docvalues.FloatDocValues;
@@ -47,29 +30,45 @@ import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.schema.FieldType;
 import org.apache.solr.schema.SchemaField;
-import org.apache.solr.search.QParser;
 import org.apache.solr.update.processor.UpdateRequestProcessor;
 import org.apache.solr.util.VersionedFile;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.*;
+
 /**
  * Obtains float field values from an external file.
  *
+ * @see org.apache.solr.schema.ExternalFileField
+ * @see org.apache.solr.schema.ExternalFileFieldReloader
  */
 
 public class FileFloatSource extends ValueSource {
+
   private SchemaField field;
   private final SchemaField keyField;
   private final float defVal;
-
   private final String dataDir;
 
-  public FileFloatSource(SchemaField field, SchemaField keyField, float defVal, QParser parser) {
+  private static final Logger log = LoggerFactory.getLogger(FileFloatSource.class);
+
+  /**
+   * Creates a new FileFloatSource
+   * @param field the source's SchemaField
+   * @param keyField the field to use as a key
+   * @param defVal the default value to use if a field has no entry in the external file
+   * @param datadir the directory in which to look for the external file
+   */
+  public FileFloatSource(SchemaField field, SchemaField keyField, float defVal, String datadir) {
     this.field = field;
     this.keyField = keyField;
     this.defVal = defVal;
-    this.dataDir = parser.getReq().getCore().getDataDir();
+    this.dataDir = datadir;
   }
 
   @Override
@@ -117,11 +116,27 @@ public class FileFloatSource extends ValueSource {
             + ",defVal="+defVal+",dataDir="+dataDir+")";
 
   }
-  
+
+  /**
+   * Remove all cached entries.  Values are lazily loaded next time getValues() is
+   * called.
+   */
   public static void resetCache(){
     floatCache.resetCache();
   }
 
+  /**
+   * Refresh the cache for an IndexReader.  The new values are loaded in the background
+   * and then swapped in, so queries against the cache should not block while the reload
+   * is happening.
+   * @param reader the IndexReader whose cache needs refreshing
+   */
+  public void refreshCache(IndexReader reader) {
+    log.info("Refreshing FlaxFileFloatSource cache for field {}", this.field.getName());
+    floatCache.refresh(reader, new Entry(this));
+    log.info("FlaxFileFloatSource cache for field {} reloaded", this.field.getName());
+  }
+
   private final float[] getCachedFloats(IndexReader reader) {
     return (float[])floatCache.get(reader, new Entry(this));
   }
@@ -139,6 +154,18 @@ public class FileFloatSource extends ValueSource {
 
     protected abstract Object createValue(IndexReader reader, Object key);
 
+    public void refresh(IndexReader reader, Object key) {
+      Object refreshedValues = createValue(reader, key);
+      synchronized (readerCache) {
+        Map innerCache = (Map) readerCache.get(reader);
+        if (innerCache == null) {
+          innerCache = new HashMap();
+          readerCache.put(reader, innerCache);
+        }
+        innerCache.put(key, refreshedValues);
+      }
+    }
+
     public Object get(IndexReader reader, Object key) {
       Map innerCache;
       Object value;