From 03ed51acef121b7ab6ac5129dd7b93c2bdf7c7b0 Mon Sep 17 00:00:00 2001
From: Mark Robert Miller <markrmiller@apache.org>
Date: Mon, 28 Oct 2013 20:43:46 +0000
Subject: [PATCH] SOLR-5397: If a retry fails, *always* finish the rest of the
 retries

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1536511 13f79535-47bb-0310-9956-ffa450edef68
---
 .../solr/update/SolrCmdDistributor.java       | 75 ++++++++++---------
 .../solr/update/StreamingSolrServers.java     |  5 +-
 2 files changed, 42 insertions(+), 38 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index 3c68870ab3e..7826437f49c 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -72,46 +72,51 @@ public class SolrCmdDistributor {
     allErrors.addAll(errors);
     boolean blockUntilFinishedAgain = false;
     for (Error err : errors) {
-      String oldNodeUrl = err.req.node.getUrl();
-      
-      // if there is a retry url, we want to retry...
-      boolean isRetry = err.req.node.checkRetry();
-      boolean doRetry = false;
-      int rspCode = err.statusCode;
-      
-      if (testing_errorHook != null) Diagnostics.call(testing_errorHook, err.e);
-      
-      // this can happen in certain situations such as shutdown
-      if (isRetry) {
-        if (rspCode == 404 || rspCode == 403 || rspCode == 503
-            || rspCode == 500) {
-          doRetry = true;
-        }
+      try {
+        String oldNodeUrl = err.req.node.getUrl();
         
-        // if its an ioexception, lets try again
-        if (err.e instanceof IOException) {
-          doRetry = true;
-        } else if (err.e instanceof SolrServerException) {
-          if (((SolrServerException) err.e).getRootCause() instanceof IOException) {
+        // if there is a retry url, we want to retry...
+        boolean isRetry = err.req.node.checkRetry();
+        boolean doRetry = false;
+        int rspCode = err.statusCode;
+        
+        if (testing_errorHook != null) Diagnostics.call(testing_errorHook,
+            err.e);
+        
+        // this can happen in certain situations such as shutdown
+        if (isRetry) {
+          if (rspCode == 404 || rspCode == 403 || rspCode == 503
+              || rspCode == 500) {
             doRetry = true;
           }
-        }
-      }
-      
-      if (isRetry && err.req.retries < MAX_RETRIES_ON_FORWARD && doRetry) {
-        err.req.retries++;
-
-        SolrException.log(SolrCmdDistributor.log, "forwarding update to "
-            + oldNodeUrl + " failed - retrying ... ");
-        try {
-          Thread.sleep(500);
-        } catch (InterruptedException e) {
-          Thread.currentThread().interrupt();
-          log.warn(null, e);
+          
+          // if its an ioexception, lets try again
+          if (err.e instanceof IOException) {
+            doRetry = true;
+          } else if (err.e instanceof SolrServerException) {
+            if (((SolrServerException) err.e).getRootCause() instanceof IOException) {
+              doRetry = true;
+            }
+          }
         }
         
-        submit(err.req);
-        blockUntilFinishedAgain = true;
+        if (isRetry && err.req.retries < MAX_RETRIES_ON_FORWARD && doRetry) {
+          err.req.retries++;
+          
+          SolrException.log(SolrCmdDistributor.log, "forwarding update to "
+              + oldNodeUrl + " failed - retrying ... ");
+          try {
+            Thread.sleep(500);
+          } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            log.warn(null, e);
+          }
+          
+          submit(err.req);
+          blockUntilFinishedAgain = true;
+        }
+      } catch (Exception e) {
+        SolrException.log(log, "Retry attempt failed", e);
       }
     }
     
diff --git a/solr/core/src/java/org/apache/solr/update/StreamingSolrServers.java b/solr/core/src/java/org/apache/solr/update/StreamingSolrServers.java
index 9c1d2caafd0..02ec08939e6 100644
--- a/solr/core/src/java/org/apache/solr/update/StreamingSolrServers.java
+++ b/solr/core/src/java/org/apache/solr/update/StreamingSolrServers.java
@@ -32,13 +32,12 @@ import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.JavaBinCodec;
 import org.apache.solr.update.SolrCmdDistributor.Error;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class StreamingSolrServers {
-  public static Logger scdlog = LoggerFactory.getLogger(SolrCmdDistributor.class);
+  public static Logger log = LoggerFactory.getLogger(StreamingSolrServers.class);
   
   private static HttpClient httpClient;
   static {
@@ -73,7 +72,7 @@ public class StreamingSolrServers {
       server = new ConcurrentUpdateSolrServer(url, httpClient, 100, 1, updateExecutor) {
         @Override
         public void handleError(Throwable ex) {
-          scdlog.error("error", ex);
+          log.error("error", ex);
           Error error = new Error();
           error.e = (Exception) ex;
           if (ex instanceof SolrException) {