From 4a5b914eaa8683009191748bf6c0b1be14d59661 Mon Sep 17 00:00:00 2001
From: Tomas Fernandez Lobbe <tflobbe@apache.org>
Date: Wed, 12 Sep 2018 16:29:17 -0700
Subject: [PATCH] SOLR-12766: Improve backoff for internal retries

When retrying internal update requests, backoff only once for the full batch of retries
instead of for every request.
Make backoff linear with the number of retries
---
 solr/CHANGES.txt                              |  2 ++
 .../solr/update/SolrCmdDistributor.java       | 21 ++++++++++++-------
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 113ca1370d9..64b8e58a492 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -385,6 +385,8 @@ Optimizations
 
 * SOLR-12723: Reduce object creation in HashBasedRouter. (ab)
 
+* SOLR-12766: When retrying internal requests, backoff only once for the full batch of retries (Tomás Fernández Löbbe)
+
 Other Changes
 ----------------------
 
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index d5aafeca49e..3a65f170d7b 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -79,7 +79,8 @@ public class SolrCmdDistributor implements Closeable {
     this.completionService = new ExecutorCompletionService<>(updateShardHandler.getUpdateExecutor());
   }
   
-  public SolrCmdDistributor(StreamingSolrClients clients, int retryPause) {
+  /* For tests only */
+  SolrCmdDistributor(StreamingSolrClients clients, int retryPause) {
     this.clients = clients;
     this.retryPause = retryPause;
     completionService = new ExecutorCompletionService<>(clients.getUpdateExecutor());
@@ -156,12 +157,6 @@ public class SolrCmdDistributor implements Closeable {
                 + err.req.cmd.toString() + " params:"
                 + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
           }
-          try {
-            Thread.sleep(retryPause); //TODO: Do we want this wait for every error?
-          } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            log.warn(null, e);
-          }
           resubmitList.add(err);
         } else {
           allErrors.add(err);
@@ -172,6 +167,18 @@ public class SolrCmdDistributor implements Closeable {
       }
     }
     
+    if (resubmitList.size() > 0) {
+      // Only backoff once for the full batch
+      try {
+        int backoffTime = retryPause * resubmitList.get(0).req.retries;
+        log.debug("Sleeping {}ms before re-submitting {} requests", backoffTime, resubmitList.size());
+        Thread.sleep(backoffTime);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        log.warn(null, e);
+      }
+    }
+    
     clients.clearErrors();
     this.errors.clear();
     for (Error err : resubmitList) {