Failed search on a shard tries a local replica on a network thread

When a search on a shard to a remove node fails, and then replica exists on the local node, then the execution of the search is done on the network thread. This is problematic since we need to execute it on the actual search thread pool, but can also explain #4519, where the get happens on the network thread and it waits to send the get request till the network thread we use is freed (deadlock...)
fixes #4526

note, re-enable the geo shape fetch test, this fix should solve it as well

src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java

@@ -278,14 +278,23 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
                     }
                 }
             } else {
-                ShardRouting nextShard = shardIt.nextOrNull();
+                final ShardRouting nextShard = shardIt.nextOrNull();
                 final boolean lastShard = nextShard == null;
                 // trace log this exception
                 if (logger.isTraceEnabled() && t != null) {
                     logger.trace(executionFailureMsg(shard, shardIt, request, lastShard), t);
                 }
                 if (!lastShard) {
+                    try {
+                        threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
+                            @Override
+                            public void run() {
                                 performFirstPhase(shardIndex, shardIt, nextShard);
+                            }
+                        });
+                    } catch (Throwable t1) {
+                        onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, t1);
+                    }
                 } else {
                     // no more shards active, add a failure
                     if (logger.isDebugEnabled() && !logger.isTraceEnabled()) { // do not double log this exception

src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationTests.java

@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.geo;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
@@ -147,9 +146,7 @@ public class GeoShapeIntegrationTests extends ElasticsearchIntegrationTest {
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo("blakely"));
     }
 
-    // TODO this test causes hangs, blocking on the action get when fetching the shape for some reason
     @Test
-    @AwaitsFix(bugUrl = "this test causes hangs, blocking on the action get when fetching the shape for some reason")
     public void testIndexedShapeReference() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("location")