SOLR-4764: When using NRT, just init the reader from IndexWriter

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1513945 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -122,6 +122,9 @@ Bug Fixes
 
 * SOLR-5135: Harden Collection API deletion of /collections/$collection 
   ZooKeeper node. (Mark Miller)
+
+* SOLR-4764: When using NRT, just init the first reader from IndexWriter.
+  (Robert Muir, Mark Miller)
   
 Optimizations
 ----------------------

solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java

@@ -19,6 +19,7 @@ package org.apache.solr.core;
 import java.io.IOException;
 
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.Directory;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.util.plugin.NamedListInitializedPlugin;
@@ -55,4 +56,21 @@ public abstract class IndexReaderFactory implements NamedListInitializedPlugin {
    */
   public abstract DirectoryReader newReader(Directory indexDir, SolrCore core)
       throws IOException;
+  
+  /**
+   * Creates a new IndexReader instance using the given IndexWriter.
+   * <p>
+   * This is used for opening the initial reader in NRT mode ({@code reopenReaders=true}
+   * in solrconfig.xml)
+   * 
+   * @param writer IndexWriter
+   * @param core {@link SolrCore} instance where this reader will be used. NOTE:
+   * this SolrCore instance may not be fully configured yet, but basic things like
+   * {@link SolrCore#getCoreDescriptor()}, {@link SolrCore#getLatestSchema()} and
+   * {@link SolrCore#getSolrConfig()} are valid.
+   * @return An IndexReader instance
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public abstract DirectoryReader newReader(IndexWriter writer, SolrCore core)
+      throws IOException;
 }

solr/core/src/java/org/apache/solr/core/SolrCore.java

@@ -795,19 +795,6 @@ public final class SolrCore implements SolrInfoMBean {
         }
       }
       
-      // Open the searcher *before* the update handler so we don't end up
-      // opening
-      // one in the middle.
-      // With lockless commits in Lucene now, this probably shouldn't be an
-      // issue anymore
-      
-      try {
-        getSearcher(false, false, null, true);
-      } finally {
-        newReaderCreator = null;
-        if (iwRef != null) iwRef.decref();
-      }
-      
       String updateHandlerClass = solrConfig.getUpdateHandlerInfo().className;
       
       if (updateHandler == null) {
@@ -819,6 +806,13 @@ public final class SolrCore implements SolrInfoMBean {
                 : updateHandlerClass, updateHandler);
       }
       infoRegistry.put("updateHandler", this.updateHandler);
+
+      try {
+        getSearcher(false, false, null, true);
+      } finally {
+        newReaderCreator = null;
+        if (iwRef != null) iwRef.decref();
+      }
       
       // Finally tell anyone who wants to know
       resourceLoader.inform(resourceLoader);
@@ -1432,6 +1426,16 @@ public final class SolrCore implements SolrInfoMBean {
           DirectoryReader newReader = newReaderCreator.call();
           tmp = new SolrIndexSearcher(this, newIndexDir, getLatestSchema(), getSolrConfig().indexConfig, 
               (realtime ? "realtime":"main"), newReader, true, !realtime, true, directoryFactory);
+        } else if (solrConfig.reopenReaders) {
+          RefCounted<IndexWriter> writer = getUpdateHandler().getSolrCoreState().getIndexWriter(this);
+          DirectoryReader newReader = null;
+          try {
+            newReader = indexReaderFactory.newReader(writer.get(), this);
+          } finally {
+            writer.decref();
+          }
+          tmp = new SolrIndexSearcher(this, newIndexDir, getLatestSchema(), getSolrConfig().indexConfig, 
+              (realtime ? "realtime":"main"), newReader, true, !realtime, true, directoryFactory);
         } else {
          // normal open that happens at startup
         // verbose("non-reopen START:");

solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java

@@ -19,6 +19,7 @@ package org.apache.solr.core;
 import java.io.IOException;
 
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.Directory;
 
 /**
@@ -26,6 +27,7 @@ import org.apache.lucene.store.Directory;
  * {@link DirectoryReader}.
  * 
  * @see DirectoryReader#open(Directory)
+ * @see DirectoryReader#open(IndexWriter, boolean)
  */
 public class StandardIndexReaderFactory extends IndexReaderFactory {
   
@@ -33,4 +35,9 @@ public class StandardIndexReaderFactory extends IndexReaderFactory {
   public DirectoryReader newReader(Directory indexDir, SolrCore core) throws IOException {
     return DirectoryReader.open(indexDir);
   }
+
+  @Override
+  public DirectoryReader newReader(IndexWriter writer, SolrCore core) throws IOException {
+    return DirectoryReader.open(writer, true);
+  }
 }

solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml

@@ -35,6 +35,7 @@ A solrconfig.xml snippet containing indexConfig settings for randomized testing.
   <ramBufferSizeMB>${solr.tests.ramBufferSizeMB}</ramBufferSizeMB>
 
   <mergeScheduler class="${solr.tests.mergeScheduler}" />
+  <reopenReaders>${solr.tests.reopenReaders:true}</reopenReaders>
 
   <writeLockTimeout>1000</writeLockTimeout>
   <commitLockTimeout>10000</commitLockTimeout>

solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java

@@ -20,6 +20,7 @@ import java.io.File;
 import java.io.IOException;
 
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.Directory;
 import org.apache.solr.SolrTestCaseJ4;
 import org.junit.BeforeClass;
@@ -71,6 +72,12 @@ public class AlternateDirectoryTest extends SolrTestCaseJ4 {
       TestIndexReaderFactory.newReaderCalled = true;
       return DirectoryReader.open(indexDir);
     }
+
+    @Override
+    public DirectoryReader newReader(IndexWriter writer, SolrCore core) throws IOException {
+      TestIndexReaderFactory.newReaderCalled = true;
+      return DirectoryReader.open(writer, true);
+    }
   }
 
 }

solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java

@@ -53,12 +53,15 @@ public class TestArbitraryIndexDir extends AbstractSolrTestCase{
   static String savedFactory;
   @BeforeClass
   public static void beforeClass() {
+    // this test wants to start solr, and then open a separate indexwriter of its own on the same dir.
+    System.setProperty("solr.tests.reopenReaders", "false");
     System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_
     savedFactory = System.getProperty("solr.DirectoryFactory");
     System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockFSDirectoryFactory");
   }
   @AfterClass
   public static void afterClass() {
+    System.clearProperty("solr.tests.reopenReaders");
     if (savedFactory == null) {
       System.clearProperty("solr.directoryFactory");
     } else {

solr/core/src/test/org/apache/solr/core/TestNRTOpen.java

@@ -0,0 +1,72 @@
+package org.apache.solr.core;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.RefCounted;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+public class TestNRTOpen extends SolrTestCaseJ4 {
+  
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    // use a filesystem, because we need to create an index, then "start up solr"
+    System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
+    // and dont delete it initially
+    System.setProperty("solr.test.leavedatadir", "true");
+    initCore("solrconfig-basic.xml", "schema-minimal.xml");
+    // add a doc
+    assertU(adoc("foo", "bar"));
+    assertU(commit());
+    File myDir = dataDir;
+    deleteCore();
+    // boot up again over the same index
+    dataDir = myDir;
+    initCore("solrconfig-basic.xml", "schema-minimal.xml");
+  }
+  
+  @AfterClass
+  public static void afterClass() throws Exception {
+    // ensure we clean up after ourselves, this will fire before superclass...
+    System.clearProperty("solr.test.leavedatadir");
+    System.clearProperty("solr.directoryFactory");
+  }
+  
+  public void testReaderIsNRT() {
+    assertNRT();
+    String core = h.getCore().getName();
+    h.getCoreContainer().reload(core);
+    assertNRT();
+  }
+  
+  private void assertNRT() {
+    RefCounted<SolrIndexSearcher> searcher = h.getCore().getSearcher();
+    try {
+      DirectoryReader ir = searcher.get().getIndexReader();
+      assertEquals(1, ir.maxDoc());
+      assertTrue("expected NRT reader, got: " + ir, ir.toString().contains(":nrt"));
+    } finally {
+      searcher.decref();
+    }
+  }
+}

solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java

@@ -346,6 +346,9 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
   public void testNoWriter() throws Exception {
     useFactory(null);    // force a persistent directory
 
+    // read-only setting (no opening from indexwriter)
+    System.setProperty("solr.tests.reopenReaders", "false");
+    try {
     // stop and start so they see the new directory setting
     slaveJetty.stop();
     masterJetty.stop();
@@ -356,6 +359,9 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
     slaveClient.commit();
     slaveJetty.stop();
     slaveJetty.start(true);
+    } finally {
+      System.clearProperty("solr.tests.reopenReaders"); // dont mess with other tests
+    }
 
     // Currently we open a writer on-demand.  This is to test that we are correctly testing
     // the code path when SolrDeletionPolicy.getLatestCommit() returns null.

solr/solrj/src/test/org/apache/solr/client/solrj/MultiCoreExampleTestBase.java

@@ -46,11 +46,6 @@ public abstract class MultiCoreExampleTestBase extends SolrExampleTestBase
 
   @Override public String getSolrHome() { return ExternalPaths.EXAMPLE_MULTICORE_HOME; }
 
-  protected void setupCoreContainer() {
-    cores = new CoreContainer();
-    cores.load();
-  }
-  
   @Override public void setUp() throws Exception {
     super.setUp();
 
@@ -65,11 +60,6 @@ public abstract class MultiCoreExampleTestBase extends SolrExampleTestBase
     System.setProperty( "solr.core0.data.dir", this.dataDir1.getCanonicalPath() ); 
     System.setProperty( "solr.core1.data.dir", this.dataDir2.getCanonicalPath() );
 
-    setupCoreContainer();
-
-    SolrCore.log.info("CORES=" + cores + " : " + cores.getCoreNames());
-
-
   }
   
   @Override
@@ -84,8 +74,6 @@ public abstract class MultiCoreExampleTestBase extends SolrExampleTestBase
         System.err.println("!!!! WARNING: best effort to remove " + dataDir2.getAbsolutePath() + " FAILED !!!!!");
       }
     }
-
-    cores.shutdown();
   }
 
   @Override

solr/solrj/src/test/org/apache/solr/client/solrj/embedded/MultiCoreEmbeddedTest.java

@@ -19,6 +19,8 @@ package org.apache.solr.client.solrj.embedded;
 
 import org.apache.solr.client.solrj.MultiCoreExampleTestBase;
 import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
 
 /**
  * This runs SolrServer test using 
@@ -33,6 +35,19 @@ public class MultiCoreEmbeddedTest extends MultiCoreExampleTestBase {
     // TODO: fix this test to use MockDirectoryFactory
     System.clearProperty("solr.directoryFactory");
     super.setUp();
+    setupCoreContainer();
+    SolrCore.log.info("CORES=" + cores + " : " + cores.getCoreNames());
+  }
+  
+  protected void setupCoreContainer() {
+    cores = new CoreContainer();
+    cores.load();
+  }
+  
+  @Override
+  public void tearDown() throws Exception {
+    cores.shutdown();
+    super.tearDown();
   }
 
   @Override