SOLR-12766: Improve backoff for internal retries

When retrying internal update requests, backoff only once for the full batch of retries
instead of for every request.
Make backoff linear with the number of retries

solr/CHANGES.txt

@@ -385,6 +385,8 @@ Optimizations
 
 * SOLR-12723: Reduce object creation in HashBasedRouter. (ab)
 
+* SOLR-12766: When retrying internal requests, backoff only once for the full batch of retries (Tomás Fernández Löbbe)
+
 Other Changes
 ----------------------
 

solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java

@@ -79,7 +79,8 @@ public class SolrCmdDistributor implements Closeable {
     this.completionService = new ExecutorCompletionService<>(updateShardHandler.getUpdateExecutor());
   }
   
-  public SolrCmdDistributor(StreamingSolrClients clients, int retryPause) {
+  /* For tests only */
+  SolrCmdDistributor(StreamingSolrClients clients, int retryPause) {
     this.clients = clients;
     this.retryPause = retryPause;
     completionService = new ExecutorCompletionService<>(clients.getUpdateExecutor());
@@ -156,12 +157,6 @@ public class SolrCmdDistributor implements Closeable {
                 + err.req.cmd.toString() + " params:"
                 + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
           }
-          try {
-            Thread.sleep(retryPause); //TODO: Do we want this wait for every error?
-          } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            log.warn(null, e);
-          }
           resubmitList.add(err);
         } else {
           allErrors.add(err);
@@ -172,6 +167,18 @@ public class SolrCmdDistributor implements Closeable {
       }
     }
     
+    if (resubmitList.size() > 0) {
+      // Only backoff once for the full batch
+      try {
+        int backoffTime = retryPause * resubmitList.get(0).req.retries;
+        log.debug("Sleeping {}ms before re-submitting {} requests", backoffTime, resubmitList.size());
+        Thread.sleep(backoffTime);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        log.warn(null, e);
+      }
+    }
+    
     clients.clearErrors();
     this.errors.clear();
     for (Error err : resubmitList) {