[reindex] Make search failure cause rest failure

Indexing failures have caused the reindex http request to fail for a while
now. Both search and indexing failures cause it to abort. But search
failures didn't cause a non-200 response code from the http api. This
fixes that.

Also slips in a fix to some infrequently failing rest tests.

Closes #16037

modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java

@@ -19,7 +19,9 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestToXContentListener;
@@ -35,6 +37,10 @@ public class BulkIndexByScrollResponseContentListener<R extends BulkIndexByScrol
 
     @Override
     protected RestStatus getStatus(R response) {
+        /*
+         * Return the highest numbered rest status under the assumption that higher numbered statuses are "more error" and thus more
+         * interesting to the user.
+         */
         RestStatus status = RestStatus.OK;
         if (response.isTimedOut()) {
             status = RestStatus.REQUEST_TIMEOUT;
@@ -44,6 +50,12 @@ public class BulkIndexByScrollResponseContentListener<R extends BulkIndexByScrol
                 status = failure.getStatus();
             }
         }
+        for (ShardSearchFailure failure: response.getSearchFailures()) {
+            RestStatus failureStatus = ExceptionsHelper.status(failure.getCause());
+            if (failureStatus.getStatus() > status.getStatus()) {
+                status = failureStatus;
+            }
+        }
         return status;
     }
 }

modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml

@@ -126,7 +126,8 @@
   - match: {failures.0.id:     "1"}
   - match: {failures.0.status: 409}
   - match: {failures.0.cause.type:   version_conflict_engine_exception}
-  - match: {failures.0.cause.reason: "[foo][1]: version conflict, document already exists (current version [1])"}
+  # Use a regex so we don't mind if the version isn't always 1. Sometimes it comes out 2.
+  - match: {failures.0.cause.reason: "/\\[foo\\]\\[1\\]:.version.conflict,.document.already.exists.\\(current.version.\\[\\d+\\]\\)/"}
   - match: {failures.0.cause.shard:  /\d+/}
   - match: {failures.0.cause.index:  dest}
   - is_true: took

modules/reindex/src/test/resources/rest-api-spec/test/update-by-query/10_basic.yaml

@@ -87,7 +87,8 @@
   - match: {failures.0.id:     "1"}
   - match: {failures.0.status: 409}
   - match: {failures.0.cause.type:   version_conflict_engine_exception}
-  - match: {failures.0.cause.reason: "[foo][1]: version conflict, current version [2] is different than the one provided [1]"}
+  # Use a regex so we don't mind if the current version isn't always 1. Sometimes it comes out 2.
+  - match: {failures.0.cause.reason: "/\\[foo\\]\\[1\\]:.version.conflict,.current.version.\\[\\d+\\].is.different.than.the.one.provided.\\[\\d+\\]/"}
   - match: {failures.0.cause.shard:  /\d+/}
   - match: {failures.0.cause.index:  test}
   - is_true: took

qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml

@@ -0,0 +1,34 @@
+---
+"Response format search failures":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: request
+      reindex:
+        body:
+          source:
+            index:   source
+            query:
+              script:
+                script: 1/0 # Divide by 0 to cause a search time exception
+          dest:
+            index:   dest
+  - match: {created: 0}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 0}
+  - is_true: failures.0.shard
+  - match: {failures.0.index:  source}
+  - is_true: failures.0.node
+  - match: {failures.0.reason.type:   script_exception}
+  - match: {failures.0.reason.reason: "failed to run inline script [1/0] using lang [groovy]"}
+  - match: {failures.0.reason.caused_by.type:   arithmetic_exception}
+  - match: {failures.0.reason.caused_by.reason: Division by zero}
+  - is_true: took

qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update-by-query/40_search_failure.yaml

@@ -0,0 +1,30 @@
+---
+"Response format search failures":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: request
+      update-by-query:
+        index:   source
+        body:
+          query:
+            script:
+              script: 1/0 # Divide by 0 to cause a search time exception
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 0}
+  - is_true: failures.0.shard
+  - match: {failures.0.index:  source}
+  - is_true: failures.0.node
+  - match: {failures.0.reason.type:   script_exception}
+  - match: {failures.0.reason.reason: "failed to run inline script [1/0] using lang [groovy]"}
+  - match: {failures.0.reason.caused_by.type:   arithmetic_exception}
+  - match: {failures.0.reason.caused_by.reason: Division by zero}
+  - is_true: took