From 5bf4e746476f37f4effa210bd031f8076e74a247 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Thu, 19 Dec 2013 22:19:20 +0100
Subject: [PATCH] Failed search on a shard tries a local replica on a network
 thread When a search on a shard to a remove node fails, and then replica
 exists on the local node, then the execution of the search is done on the
 network thread. This is problematic since we need to execute it on the actual
 search thread pool, but can also explain #4519, where the get happens on the
 network thread and it waits to send the get request till the network thread
 we use is freed (deadlock...) fixes #4526

note, re-enable the geo shape fetch test, this fix should solve it as well
---
 .../search/type/TransportSearchTypeAction.java      | 13 +++++++++++--
 .../search/geo/GeoShapeIntegrationTests.java        |  3 ---
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java
index b8e01100594..e0d4f83c6ca 100644
--- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java
+++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java
@@ -278,14 +278,23 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
                     }
                 }
             } else {
-                ShardRouting nextShard = shardIt.nextOrNull();
+                final ShardRouting nextShard = shardIt.nextOrNull();
                 final boolean lastShard = nextShard == null;
                 // trace log this exception
                 if (logger.isTraceEnabled() && t != null) {
                     logger.trace(executionFailureMsg(shard, shardIt, request, lastShard), t);
                 }
                 if (!lastShard) {
-                    performFirstPhase(shardIndex, shardIt, nextShard);
+                    try {
+                        threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
+                            @Override
+                            public void run() {
+                                performFirstPhase(shardIndex, shardIt, nextShard);
+                            }
+                        });
+                    } catch (Throwable t1) {
+                        onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, t1);
+                    }
                 } else {
                     // no more shards active, add a failure
                     if (logger.isDebugEnabled() && !logger.isTraceEnabled()) { // do not double log this exception
diff --git a/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationTests.java b/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationTests.java
index 716bac1cfae..dbbd9e42cdc 100644
--- a/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationTests.java
+++ b/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.geo;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
@@ -147,9 +146,7 @@ public class GeoShapeIntegrationTests extends ElasticsearchIntegrationTest {
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo("blakely"));
     }
 
-    // TODO this test causes hangs, blocking on the action get when fetching the shape for some reason
     @Test
-    @AwaitsFix(bugUrl = "this test causes hangs, blocking on the action get when fetching the shape for some reason")
     public void testIndexedShapeReference() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("location")