SOLR-4709: The core reload after replication if config files have changed can fail due to a race condition.

SOLR-5489: TestIndexAndConfigAliasReplication commonly fails because it tries to get a lock for a locked index.
SOLR-5343: TestReplicationHandler.doTestStressReplication fails ~ 33% of the time

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1544220 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -127,6 +127,9 @@ Bug Fixes
 * SOLR-5481: SolrCmdDistributor should not let the http client do it's own 
   retries. (Mark Miller)
 
+* SOLR-4709: The core reload after replication if config files have changed
+  can fail due to a race condition. (Mark Miller, Hossman))
+
 Optimizations
 ----------------------
 

solr/core/src/java/org/apache/solr/handler/SnapPuller.java

@@ -57,6 +57,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -416,6 +417,9 @@ public class SnapPuller {
           solrCore.getUpdateHandler().getSolrCoreState()
           .closeIndexWriter(core, true);
         }
+        
+        boolean reloadCore = false;
+        
         try {
           LOG.info("Starting download to " + tmpIndexDir + " fullCopy="
               + isFullCopyNeeded);
@@ -450,7 +454,7 @@ public class SnapPuller {
               logReplicationTimeAndConfFiles(modifiedConfFiles,
                   successfulInstall);// write to a file time of replication and
                                      // conf files.
-              reloadCore();
+              reloadCore = true;
             }
           } else {
             terminateAndWaitFsyncService();
@@ -470,6 +474,11 @@ public class SnapPuller {
             solrCore.getUpdateHandler().getSolrCoreState().openIndexWriter(core);
           }
         }
+        
+        // we must reload the core after we open the IW back up
+        if (reloadCore) {
+          reloadCore();
+        }
 
         if (successfulInstall) {
           if (isFullCopyNeeded) {
@@ -699,6 +708,7 @@ public class SnapPuller {
   }
 
   private void reloadCore() {
+    final CountDownLatch latch = new CountDownLatch(1);
     new Thread() {
       @Override
       public void run() {
@@ -706,9 +716,17 @@ public class SnapPuller {
           solrCore.getCoreDescriptor().getCoreContainer().reload(solrCore.getName());
         } catch (Exception e) {
           LOG.error("Could not reload core ", e);
+        } finally {
+          latch.countDown();
         }
       }
     }.start();
+    try {
+      latch.await();
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new RuntimeException("Interrupted while waiting for core reload to finish", e);
+    }
   }
 
   private void downloadConfFiles(List<Map<String, Object>> confFilesToDownload, long latestGeneration) throws Exception {

solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java

@@ -751,7 +751,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
   }
   
   
-  @Test @Ignore("https://issues.apache.org/jira/browse/SOLR-5343")
+  @Test 
   public void doTestStressReplication() throws Exception {
     // change solrconfig on slave
     // this has no entry for pollinginterval