SOLR-2702: add replication support to NRTCachingDirectoryFactory and make it the default

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1358650 13f79535-47bb-0310-9956-ffa450edef68

lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java

@@ -84,6 +84,10 @@ public class NRTCachingDirectory extends Directory {
     maxCachedBytes = (long) (maxCachedMB*1024*1024);
   }
 
+  public Directory getDelegate() {
+    return delegate;
+  }
+
   @Override
   public LockFactory getLockFactory() {
     return delegate.getLockFactory();

solr/CHANGES.txt

@@ -47,6 +47,11 @@ New Features
 * SOLR-3562: Add options to remove instance dir or data dir on core unload.
   (Mark Miller, Per Steffensen)
 
+* SOLR-2702: The default directory factory was changed to NRTCachingDirectoryFactory 
+  which wraps the StandardDirectoryFactory and caches small files for improved
+  Near Real-time (NRT) performance. (Mark Miller, yonik)
+
+
 Bug Fixes
 
 * SOLR-3582: Our ZooKeeper watchers respond to session events as if they are change events, 
@@ -75,6 +80,7 @@ Other Changes
 * SOLR-3599: Add zkClientTimeout to solr.xml so that it's obvious how to change it and so 
   that you can change it with a system property. (Mark Miller)
 
+
 ==================  4.0.0-ALPHA ==================
 More information about this release, including any errata related to the 
 release notes, upgrade instructions, or other changes may be found online at:

solr/core/src/java/org/apache/solr/core/SolrCore.java

@@ -385,7 +385,7 @@ public final class SolrCore implements SolrInfoMBean {
       dirFactory = getResourceLoader().newInstance(info.className, DirectoryFactory.class);
       dirFactory.init(info.initArgs);
     } else {
-      dirFactory = new StandardDirectoryFactory();
+      dirFactory = new NRTCachingDirectoryFactory();
     }
     // And set it
     directoryFactory = dirFactory;

solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java

@@ -37,6 +37,7 @@ import org.apache.lucene.index.*;
 import org.apache.lucene.search.*;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.NRTCachingDirectory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.OpenBitSet;
@@ -118,6 +119,18 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
     this(core, schema,name, core.getIndexReaderFactory().newReader(directoryFactory.get(path, config.lockType), core), true, enableCache, false, directoryFactory);
   }
 
+  private static String getIndexDir(Directory dir) {
+    if (dir instanceof FSDirectory) {
+      return ((FSDirectory)dir).getDirectory().getAbsolutePath();
+    } else if (dir instanceof NRTCachingDirectory) {
+      // recurse on the delegate
+      return getIndexDir(((NRTCachingDirectory) dir).getDelegate());
+    } else {
+      log.warn("WARNING: Directory impl does not support setting indexDir: " + dir.getClass().getName());
+      return null;
+    }
+  }
+
   public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, DirectoryReader r, boolean closeReader, boolean enableCache, boolean reserveDirectory, DirectoryFactory directoryFactory) throws IOException {
     super(r);
     this.directoryFactory = directoryFactory;
@@ -134,13 +147,8 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
       // keep the directory from being released while we use it
       directoryFactory.incRef(dir);
     }
-    
-    if (dir instanceof FSDirectory) {
-      FSDirectory fsDirectory = (FSDirectory) dir;
-      indexDir = fsDirectory.getDirectory().getAbsolutePath();
-    } else {
-      log.warn("WARNING: Directory impl does not support setting indexDir: " + dir.getClass().getName());
-    }
+
+    this.indexDir = getIndexDir(dir);
 
     this.closeReader = closeReader;
     setSimilarity(schema.getSimilarity());

solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java

@@ -33,6 +33,7 @@ import org.apache.solr.servlet.SolrDispatchFilter;
 import org.apache.zookeeper.KeeperException;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.BeforeClass;
 
 public abstract class AbstractDistributedZkTestCase extends BaseDistributedSearchTestCase {
   
@@ -41,6 +42,12 @@ public abstract class AbstractDistributedZkTestCase extends BaseDistributedSearc
   protected ZkTestServer zkServer;
   private AtomicInteger homeCount = new AtomicInteger();
 
+  @BeforeClass
+  public static void beforeThisClass() throws Exception {
+    useFactory(null);
+  }
+
+
   @Before
   @Override
   public void setUp() throws Exception {
@@ -55,9 +62,8 @@ public abstract class AbstractDistributedZkTestCase extends BaseDistributedSearc
     System.setProperty("zkHost", zkServer.getZkAddress());
     System.setProperty("enable.update.log", "true");
     System.setProperty("remove.version.field", "true");
-    System
-    .setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
-    
+
+
     AbstractZkTestCase.buildZooKeeper(zkServer.getZkHost(), zkServer.getZkAddress(), "solrconfig.xml", "schema.xml");
 
     // set some system properties for use by tests

solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java

@@ -153,14 +153,11 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
   
   @BeforeClass
   public static void beforeClass() {
-    System
-        .setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
     System.setProperty("solrcloud.update.delay", "0");
   }
   
   @AfterClass
   public static void afterClass() {
-    System.clearProperty("solr.directoryFactory");
     System.clearProperty("solrcloud.update.delay");
   }
   

solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java

@@ -81,12 +81,10 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
   // index from previous test method
   static int nDocs = 500;
 
-  // TODO: fix this test to not require FSDirectory.. doesnt even work with MockFSDirectory... wtf?
-  static String savedFactory;
+
   @BeforeClass
   public static void beforeClass() throws Exception {
-    savedFactory = System.getProperty("solr.DirectoryFactory");
-    System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
+    useFactory(null); // need an FS factory
     master = new SolrInstance("master", null);
     master.setUp();
     masterJetty = createJetty(master);
@@ -116,11 +114,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
     slaveJetty.stop();
     master.tearDown();
     slave.tearDown();
-    if (savedFactory == null) {
-      System.clearProperty("solr.directoryFactory");
-    } else {
-      System.setProperty("solr.directoryFactory", savedFactory);
-    }
   }
 
   private static JettySolrRunner createJetty(SolrInstance instance) throws Exception {

solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java

@@ -27,6 +27,7 @@ import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SpellingParams;
 import org.apache.solr.common.util.NamedList;
+import org.junit.BeforeClass;
 
 /**
  * Test for SpellCheckComponent's distributed querying
@@ -47,14 +48,14 @@ public class DistributedSpellCheckComponentTest extends BaseDistributedSearchTes
 		//shardCount=2;
 		//stress=0;
 	}
-	
-  private String saveProp;
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    useFactory(null); // need an FS factory
+  }
 
   @Override
   public void setUp() throws Exception {
-    // this test requires FSDir
-    saveProp = System.getProperty("solr.directoryFactory");
-    System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");    
     if(random().nextBoolean()) {
       requestHandlerName = "spellCheckCompRH";
       reqHandlerWithWordbreak = "spellCheckWithWordbreak";      
@@ -68,10 +69,6 @@ public class DistributedSpellCheckComponentTest extends BaseDistributedSearchTes
   @Override
   public void tearDown() throws Exception {
     super.tearDown();
-    if (saveProp == null)
-      System.clearProperty("solr.directoryFactory");
-    else
-      System.setProperty("solr.directoryFactory", saveProp);
   }
   
   private void q(Object... q) throws Exception {

solr/core/src/test/org/apache/solr/search/TestSolrJ.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+
+import org.apache.lucene.util.OpenBitSet;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TestSolrJ extends SolrTestCaseJ4 {
+
+  public void testSolrJ() {
+    // doCommitPerf();
+  }
+
+  public void doCommitPerf() throws Exception {
+    HttpSolrServer client = new HttpSolrServer("http://localhost:8983/solr");
+
+    long start = System.currentTimeMillis();
+
+    for (int i=0; i<10000; i++) {
+      SolrInputDocument doc = new SolrInputDocument();
+      doc.addField("id", Integer.toString(i % 13));
+      client.add(doc);
+      client.commit(true, true, true);
+    }
+
+    long end = System.currentTimeMillis();
+
+    client.shutdown();
+
+    System.out.println("TIME: " + (end-start));
+  }
+
+}

solr/example/solr/collection1/conf/solrconfig.xml

@@ -101,17 +101,20 @@
 
   <!-- The DirectoryFactory to use for indexes.
        
-       solr.StandardDirectoryFactory, the default, is filesystem
+       solr.StandardDirectoryFactory is filesystem
        based and tries to pick the best implementation for the current
-       JVM and platform.  One can force a particular implementation
-       via solr.MMapDirectoryFactory, solr.NIOFSDirectoryFactory, or
-       solr.SimpleFSDirectoryFactory.
+       JVM and platform.  solr.NRTCachingDirectoryFactory, the default,
+       wraps solr.StandardDirectoryFactory and caches small files in memory
+       for better NRT performance.
+
+       One can force a particular implementation via solr.MMapDirectoryFactory,
+       solr.NIOFSDirectoryFactory, or solr.SimpleFSDirectoryFactory.
 
        solr.RAMDirectoryFactory is memory based, not
        persistent, and doesn't work with replication.
     -->
   <directoryFactory name="DirectoryFactory" 
-                    class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
+                    class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/> 
 
   <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        Index Config - These settings control low-level behavior of indexing

solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java

@@ -81,8 +81,34 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
     resetExceptionIgnores();
     endTrackingSearchers();
     endTrackingZkClients();
+    resetFactory();
+    resetFactory();
   }
 
+  private static boolean changedFactory = false;
+  private static String savedFactory;
+  /** Use a different directory factory.  Passing "null" sets to an FS-based factory */
+  public static void useFactory(String factory) throws Exception {
+    assert !changedFactory;
+    changedFactory = true;
+    savedFactory = System.getProperty("solr.DirectoryFactory");
+    if (factory == null) {
+      factory = random().nextInt(100) < 75 ? "solr.NRTCachingDirectoryFactory" : "solr.StandardDirectoryFactory"; // test the default most of the time
+    }
+    System.setProperty("solr.directoryFactory", factory);
+  }
+
+  private static void resetFactory() throws Exception {
+    if (!changedFactory) return;
+    changedFactory = false;
+    if (savedFactory != null) {
+      System.setProperty("solr.directoryFactory", savedFactory);
+    } else {
+      System.clearProperty("solr.directoryFactory");
+    }
+  }
+
+
   @Override
   public void setUp() throws Exception {
     super.setUp();