SOLR-465: Add configurable DirectoryProvider so that alternate Directory implementations can be specified via solrconfig.xml

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@712014 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -77,6 +77,11 @@ New Features
     contention and synchronization overhead, to utilize multiple CPU cores more effectively.
     (Fuad Efendi, Noble Paul, yonik via shalin)
 
+14. SOLR-465: Add configurable DirectoryProvider so that alternate Directory
+    implementations can be specified via solrconfig.xml.  The default
+    DirectoryProvider will use NIOFSDirectory for better concurrency
+    on non Windows platforms.  (Mark Miller, TJ Laurenzo via yonik)
+
 Optimizations
 ----------------------
  1. SOLR-374: Use IndexReader.reopen to save resources by re-using parts of the

src/java/org/apache/solr/core/SolrCore.java

@@ -96,6 +96,7 @@ public final class SolrCore implements SolrInfoMBean {
   private final Map<String,UpdateRequestProcessorChain> updateProcessorChains;
   private final Map<String, SolrInfoMBean> infoRegistry;
   private IndexDeletionPolicyWrapper solrDelPolicy;
+  private DirectoryFactory directoryFactory;
 
   public long getStartTime() { return startTime; }
 
@@ -217,7 +218,11 @@ public final class SolrCore implements SolrInfoMBean {
     }
     return result;
   }
-
+  
+  public DirectoryFactory getDirectoryFactory() {
+    return directoryFactory;
+  }
+  
   public String getName() {
     return name;
   }
@@ -319,9 +324,28 @@ public final class SolrCore implements SolrInfoMBean {
   
   // gets a non-caching searcher
   public SolrIndexSearcher newSearcher(String name, boolean readOnly) throws IOException {
-    return new SolrIndexSearcher(this, schema, "main", IndexReader.open(FSDirectory.getDirectory(getIndexDir()), readOnly), true, false);
+    return new SolrIndexSearcher(this, schema, name, directoryFactory.open(getIndexDir()), false);
+    //return new SolrIndexSearcher(this, schema, "main", IndexReader.open(FSDirectory.getDirectory(getIndexDir()), readOnly), true, false);
   }
 
+  private void initDirectoryFactory() {
+    String xpath = "directoryFactory";
+    Node node = (Node) solrConfig.evaluate(xpath, XPathConstants.NODE);
+    DirectoryFactory dirFactory;
+    if (node != null) {
+      Map<String, DirectoryFactory> registry = new HashMap<String, DirectoryFactory>();
+      NamedListPluginLoader<DirectoryFactory> indexReaderFactoryLoader = new NamedListPluginLoader<DirectoryFactory>(
+          "[solrconfig.xml] " + xpath, registry);
+
+      dirFactory = indexReaderFactoryLoader.loadSingle(solrConfig
+          .getResourceLoader(), node);
+    } else {
+      dirFactory = new StandardDirectoryFactory();
+    }
+
+    // And set it
+    directoryFactory = dirFactory;
+  }
 
   // protect via synchronized(SolrCore.class)
   private static Set<String> dirs = new HashSet<String>();
@@ -351,6 +375,8 @@ public final class SolrCore implements SolrInfoMBean {
         SolrIndexWriter writer = new SolrIndexWriter("SolrCore.initIndex",getIndexDir(), true, schema, solrConfig.mainIndexConfig);
         writer.close();
       }
+      
+      initDirectoryFactory();
 
     } catch (IOException e) {
       throw new RuntimeException(e);
@@ -1011,22 +1037,22 @@ public final class SolrCore implements SolrInfoMBean {
 
     try {
       newestSearcher = getNewestSearcher(false);
+      String newIndexDir = getNewIndexDir();
       if (newestSearcher != null) {
         IndexReader currentReader = newestSearcher.get().getReader();
-        String newIndexDir = getNewIndexDir();
         if(new File(getIndexDir()).equals(new File(newIndexDir)))  {
           IndexReader newReader = currentReader.reopen();
 
           if(newReader == currentReader) {
             currentReader.incRef();
           }
-
+          
           tmp = new SolrIndexSearcher(this, schema, "main", newReader, true, true);
         } else  {
-          tmp = new SolrIndexSearcher(this, schema, "main", newIndexDir, true);
+          tmp = new SolrIndexSearcher(this, schema, "main", getDirectoryFactory().open(newIndexDir), true, true);
         }
       } else {
-        tmp = new SolrIndexSearcher(this, schema, "main", getNewIndexDir(), true);
+          tmp = new SolrIndexSearcher(this, schema, "main", getDirectoryFactory().open(newIndexDir), true, true);
       }
     } catch (Throwable th) {
       synchronized(searcherLock) {

src/java/org/apache/solr/schema/UUIDField.java

@@ -33,7 +33,7 @@ import org.apache.solr.request.XMLWriter;
  *
  * @see UUID#toString
  * @see UUID#randomUUID
- * @version $Id:$
+ * @version $Id$
  */
 public class UUIDField extends FieldType {
   private static final String NEW = "NEW";

src/java/org/apache/solr/search/SolrIndexSearcher.java

@@ -91,7 +91,10 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
   private final SolrCache[] cacheList;
   private static final SolrCache[] noCaches = new SolrCache[0];
 
-  /** Creates a searcher searching the index in the named directory. */
+  /** Creates a searcher searching the index in the named directory.
+   * 
+   * @deprecated use alternate constructor
+   */
   public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, String path, boolean enableCache) throws IOException {
     this(core, schema,name,IndexReader.open(path), true, enableCache);
   }
@@ -100,6 +103,11 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
   public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, Directory directory, boolean enableCache) throws IOException {
     this(core, schema,name,IndexReader.open(directory), true, enableCache);
   }
+  
+  /** Creates a searcher searching the index in the provided directory. */
+  public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, Directory directory, boolean readOnly, boolean enableCache) throws IOException {
+    this(core, schema,name,IndexReader.open(directory, readOnly), true, enableCache);
+  }
 
   /** Creates a searcher searching the provided index. */
   public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, IndexReader r, boolean enableCache) {

src/java/org/apache/solr/update/SolrIndexWriter.java

@@ -20,6 +20,7 @@ package org.apache.solr.update;
 import org.apache.lucene.index.*;
 import org.apache.lucene.store.*;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.core.DirectoryFactory;
 import org.apache.solr.schema.IndexSchema;
 
 import org.slf4j.Logger;
@@ -75,12 +76,13 @@ public class SolrIndexWriter extends IndexWriter {
 
   }
 
-  public static Directory getDirectory(String path, SolrIndexConfig config) throws IOException {
+  public static Directory getDirectory(String path, DirectoryFactory directoryFactory, SolrIndexConfig config) throws IOException {
-    Directory d = FSDirectory.getDirectory(path);
+    
+    Directory d = directoryFactory.open(path);
 
     String rawLockType = (null == config) ? null : config.lockType;
     if (null == rawLockType) {
-      // we default to "simple" for backwards compatiblitiy
+      // we default to "simple" for backwards compatibility
       log.warn("No lockType configured for " + path + " assuming 'simple'");
       rawLockType = "simple";
     }
@@ -95,7 +97,7 @@ public class SolrIndexWriter extends IndexWriter {
       if (!(d.getLockFactory() instanceof SingleInstanceLockFactory))
         d.setLockFactory(new SingleInstanceLockFactory());
     } else if ("none".equals(lockType)) {
-      // recipie for disaster
+      // Recipe for disaster
       log.error("CONFIGURATION WARNING: locks are disabled on " + path);      
       d.setLockFactory(new NoLockFactory());
     } else {
@@ -104,12 +106,67 @@ public class SolrIndexWriter extends IndexWriter {
     }
     return d;
   }
+  
+  /**
+   * @deprecated use getDirectory(DirectoryFactory directoryFactory, SolrIndexConfig config)
+   */
+  public static Directory getDirectory(String path, SolrIndexConfig config) throws IOException {
+    Directory d = FSDirectory.getDirectory(path);
 
+    String rawLockType = (null == config) ? null : config.lockType;
+    if (null == rawLockType) {
+      // we default to "simple" for backwards compatibility
+      log.warn("No lockType configured for " + path + " assuming 'simple'");
+      rawLockType = "simple";
+    }
+    final String lockType = rawLockType.toLowerCase().trim();
+
+    if ("simple".equals(lockType)) {
+      // multiple SimpleFSLockFactory instances should be OK
+      d.setLockFactory(new SimpleFSLockFactory(path));
+    } else if ("native".equals(lockType)) {
+      d.setLockFactory(new NativeFSLockFactory(path));
+    } else if ("single".equals(lockType)) {
+      if (!(d.getLockFactory() instanceof SingleInstanceLockFactory))
+        d.setLockFactory(new SingleInstanceLockFactory());
+    } else if ("none".equals(lockType)) {
+      // Recipe for disaster
+      log.error("CONFIGURATION WARNING: locks are disabled on " + path);      
+      d.setLockFactory(new NoLockFactory());
+    } else {
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+              "Unrecognized lockType: " + rawLockType);
+    }
+    return d;
+  }
+  
+  /**
+   *
+   */
+  public SolrIndexWriter(String name, String path, DirectoryFactory dirFactory, boolean create, IndexSchema schema) throws IOException {
+    super(getDirectory(path, dirFactory, null), false, schema.getAnalyzer(), create);
+    init(name, schema, null);
+  }
+
+  /**
+   *
+   */
+  public SolrIndexWriter(String name, String path, DirectoryFactory dirFactory, boolean create, IndexSchema schema, SolrIndexConfig config) throws IOException {
+    super(getDirectory(path, dirFactory, null), config.luceneAutoCommit, schema.getAnalyzer(), create);
+    init(name, schema, config);
+  }
+  
+  /**
+   * @deprecated
+   */
   public SolrIndexWriter(String name, String path, boolean create, IndexSchema schema) throws IOException {
     super(getDirectory(path, null), false, schema.getAnalyzer(), create);
     init(name, schema, null);
   }
 
+  /**
+   * @deprecated
+   */
   public SolrIndexWriter(String name, String path, boolean create, IndexSchema schema, SolrIndexConfig config) throws IOException {
     super(getDirectory(path, config), config.luceneAutoCommit, schema.getAnalyzer(), create);
     init(name, schema, config);

src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java

@@ -183,6 +183,62 @@ public abstract class AbstractPluginLoader<T>
     return defaultPlugin;
   }
   
+  /**
+   * Given a NodeList from XML in the form:
+   * 
+   * <plugin name="name1" class="solr.ClassName" > ... </plugin>
+   * 
+   * This will initialize and register a single plugin. A class will be
+   * generated for the plugin and registered to the given name.
+   * 
+   * If 'preRegister' is true, the plugin will be registered *before* it is
+   * initialized This may be useful for implementations that need to inspect
+   * other registered plugins at startup.
+   * 
+   * The created class for the plugin will be returned from this function.
+   * 
+   */
+  public T loadSingle(ResourceLoader loader, Node node) {
+    List<PluginInitInfo> info = new ArrayList<PluginInitInfo>();
+    T plugin = null;
+
+    try {
+      String name = DOMUtil.getAttr(node, "name", requireName ? type : null);
+      String className = DOMUtil.getAttr(node, "class", type);
+      plugin = create(loader, name, className, node);
+      log.info("created " + name + ": " + plugin.getClass().getName());
+
+      // Either initialize now or wait till everything has been registered
+      if (preRegister) {
+        info.add(new PluginInitInfo(plugin, node));
+      } else {
+        init(plugin, node);
+      }
+
+      T old = register(name, plugin);
+      if (old != null && !(name == null && !requireName)) {
+        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+            "Multiple " + type + " registered to the same name: " + name
+                + " ignoring: " + old);
+      }
+
+    } catch (Exception e) {
+      SolrConfig.severeErrors.add(e);
+      SolrException.logOnce(log, null, e);
+    }
+
+    // If everything needs to be registered *first*, this will initialize later
+    for (PluginInitInfo pinfo : info) {
+      try {
+        init(pinfo.plugin, pinfo.node);
+      } catch (Exception ex) {
+        SolrConfig.severeErrors.add(ex);
+        SolrException.logOnce(log, null, ex);
+      }
+    }
+    return plugin;
+  }
+  
 
   /**
    * Internal class to hold onto initialization info so that it can be initialized