honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

search reducer to use atomic reference arrays 

move away from maps to correlate between responses from different shards to unique incremental integer representing a shardRequestId (unique for the specific search request)

 this allows to no longer require using maps (or CHM), and simply use atomic reference arrays, which rely on volatiles. it also removes the need to use a cache for heavy data structures since we don't really have them around anymore...
This commit is contained in:
Shay Banon 2013-07-14 00:51:54 +02:00
parent 2762fed04f
commit 8e0d23b147
20 changed files with 522 additions and 532 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/org/elasticsearch/action/ActionModule.java
View File

@ -231,7 +231,6 @@ public class ActionModule extends AbstractModule {
registerAction(DeleteByQueryAction.INSTANCE, TransportDeleteByQueryAction.class, registerAction(DeleteByQueryAction.INSTANCE, TransportDeleteByQueryAction.class,
TransportIndexDeleteByQueryAction.class, TransportShardDeleteByQueryAction.class); TransportIndexDeleteByQueryAction.class, TransportShardDeleteByQueryAction.class);
registerAction(SearchAction.INSTANCE, TransportSearchAction.class, registerAction(SearchAction.INSTANCE, TransportSearchAction.class,
TransportSearchCache.class,
TransportSearchDfsQueryThenFetchAction.class, TransportSearchDfsQueryThenFetchAction.class,
TransportSearchQueryThenFetchAction.class, TransportSearchQueryThenFetchAction.class,
TransportSearchDfsQueryAndFetchAction.class, TransportSearchDfsQueryAndFetchAction.class,

99
src/main/java/org/elasticsearch/action/search/type/TransportSearchCache.java
View File

@ -1,99 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.search.type;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.search.dfs.DfsSearchResult;
import org.elasticsearch.search.fetch.FetchSearchResult;
import org.elasticsearch.search.fetch.QueryFetchSearchResult;
import org.elasticsearch.search.query.QuerySearchResultProvider;
import java.util.Collection;
import java.util.Map;
import java.util.Queue;
/**
*
*/
public class TransportSearchCache {
private final Queue<Collection<DfsSearchResult>> cacheDfsResults = ConcurrentCollections.newQueue();
private final Queue<Map<SearchShardTarget, QuerySearchResultProvider>> cacheQueryResults = ConcurrentCollections.newQueue();
private final Queue<Map<SearchShardTarget, FetchSearchResult>> cacheFetchResults = ConcurrentCollections.newQueue();
private final Queue<Map<SearchShardTarget, QueryFetchSearchResult>> cacheQueryFetchResults = ConcurrentCollections.newQueue();
public Collection<DfsSearchResult> obtainDfsResults() {
Collection<DfsSearchResult> dfsSearchResults;
while ((dfsSearchResults = cacheDfsResults.poll()) == null) {
Queue<DfsSearchResult> offer = ConcurrentCollections.newQueue();
cacheDfsResults.offer(offer);
}
return dfsSearchResults;
}
public void releaseDfsResults(Collection<DfsSearchResult> dfsResults) {
dfsResults.clear();
cacheDfsResults.offer(dfsResults);
}
public Map<SearchShardTarget, QuerySearchResultProvider> obtainQueryResults() {
Map<SearchShardTarget, QuerySearchResultProvider> queryResults;
while ((queryResults = cacheQueryResults.poll()) == null) {
cacheQueryResults.offer(ConcurrentCollections.<SearchShardTarget, QuerySearchResultProvider>newConcurrentMap());
}
return queryResults;
}
public void releaseQueryResults(Map<SearchShardTarget, QuerySearchResultProvider> queryResults) {
queryResults.clear();
cacheQueryResults.offer(queryResults);
}
public Map<SearchShardTarget, FetchSearchResult> obtainFetchResults() {
Map<SearchShardTarget, FetchSearchResult> fetchResults;
while ((fetchResults = cacheFetchResults.poll()) == null) {
cacheFetchResults.offer(ConcurrentCollections.<SearchShardTarget, FetchSearchResult>newConcurrentMap());
}
return fetchResults;
}
public void releaseFetchResults(Map<SearchShardTarget, FetchSearchResult> fetchResults) {
fetchResults.clear();
cacheFetchResults.offer(fetchResults);
}
public Map<SearchShardTarget, QueryFetchSearchResult> obtainQueryFetchResults() {
Map<SearchShardTarget, QueryFetchSearchResult> fetchResults;
while ((fetchResults = cacheQueryFetchResults.poll()) == null) {
cacheQueryFetchResults.offer(ConcurrentCollections.<SearchShardTarget, QueryFetchSearchResult>newConcurrentMap());
}
return fetchResults;
}
public void releaseQueryFetchResults(Map<SearchShardTarget, QueryFetchSearchResult> fetchResults) {
fetchResults.clear();
cacheQueryFetchResults.offer(fetchResults);
}
}

23
src/main/java/org/elasticsearch/action/search/type/TransportSearchCountAction.java
View File

@ -19,16 +19,14 @@
package org.elasticsearch.action.search.type; package org.elasticsearch.action.search.type;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.search.action.SearchServiceListener; import org.elasticsearch.search.action.SearchServiceListener;
import org.elasticsearch.search.action.SearchServiceTransportAction; import org.elasticsearch.search.action.SearchServiceTransportAction;
import org.elasticsearch.search.controller.SearchPhaseController; import org.elasticsearch.search.controller.SearchPhaseController;
@ -37,11 +35,8 @@ import org.elasticsearch.search.fetch.FetchSearchResultProvider;
import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.internal.ShardSearchRequest;
import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.query.QuerySearchResult;
import org.elasticsearch.search.query.QuerySearchResultProvider;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import java.util.Map;
import static org.elasticsearch.action.search.type.TransportSearchHelper.buildScrollId; import static org.elasticsearch.action.search.type.TransportSearchHelper.buildScrollId;
/** /**
@ -51,8 +46,8 @@ public class TransportSearchCountAction extends TransportSearchTypeAction {
@Inject @Inject
public TransportSearchCountAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, public TransportSearchCountAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
TransportSearchCache transportSearchCache, SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) { SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) {
super(settings, threadPool, clusterService, transportSearchCache, searchService, searchPhaseController); super(settings, threadPool, clusterService, searchService, searchPhaseController);
} }
@Override @Override
@ -62,8 +57,6 @@ public class TransportSearchCountAction extends TransportSearchTypeAction {
private class AsyncAction extends BaseAsyncAction<QuerySearchResult> { private class AsyncAction extends BaseAsyncAction<QuerySearchResult> {
private final Map<SearchShardTarget, QuerySearchResultProvider> queryFetchResults = searchCache.obtainQueryResults();
private AsyncAction(SearchRequest request, ActionListener<SearchResponse> listener) { private AsyncAction(SearchRequest request, ActionListener<SearchResponse> listener) {
super(request, listener); super(request, listener);
} }
@ -78,21 +71,15 @@ public class TransportSearchCountAction extends TransportSearchTypeAction {
searchService.sendExecuteQuery(node, request, listener); searchService.sendExecuteQuery(node, request, listener);
} }
@Override
protected void processFirstPhaseResult(ShardRouting shard, QuerySearchResult result) {
queryFetchResults.put(result.shardTarget(), result);
}
@Override @Override
protected void moveToSecondPhase() throws Exception { protected void moveToSecondPhase() throws Exception {
// no need to sort, since we know we have no hits back // no need to sort, since we know we have no hits back
final InternalSearchResponse internalResponse = searchPhaseController.merge(EMPTY_DOCS, queryFetchResults, ImmutableMap.<SearchShardTarget, FetchSearchResultProvider>of()); final InternalSearchResponse internalResponse = searchPhaseController.merge(EMPTY_DOCS, firstResults, (AtomicArray<? extends FetchSearchResultProvider>) AtomicArray.empty());
String scrollId = null; String scrollId = null;
if (request.scroll() != null) { if (request.scroll() != null) {
scrollId = buildScrollId(request.searchType(), queryFetchResults.values(), null); scrollId = buildScrollId(request.searchType(), firstResults, null);
} }
listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successulOps.get(), buildTookInMillis(), buildShardFailures())); listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successulOps.get(), buildTookInMillis(), buildShardFailures()));
searchCache.releaseQueryResults(queryFetchResults);
} }
} }

56
src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryAndFetchAction.java
View File

@ -23,10 +23,9 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.search.*; import org.elasticsearch.action.search.*;
import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.search.action.SearchServiceListener; import org.elasticsearch.search.action.SearchServiceListener;
import org.elasticsearch.search.action.SearchServiceTransportAction; import org.elasticsearch.search.action.SearchServiceTransportAction;
import org.elasticsearch.search.controller.SearchPhaseController; import org.elasticsearch.search.controller.SearchPhaseController;
@ -38,12 +37,8 @@ import org.elasticsearch.search.internal.ShardSearchRequest;
import org.elasticsearch.search.query.QuerySearchRequest; import org.elasticsearch.search.query.QuerySearchRequest;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import static org.elasticsearch.action.search.type.TransportSearchHelper.buildScrollId;
/** /**
* *
*/ */
@ -51,8 +46,8 @@ public class TransportSearchDfsQueryAndFetchAction extends TransportSearchTypeAc
@Inject @Inject
public TransportSearchDfsQueryAndFetchAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, public TransportSearchDfsQueryAndFetchAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
TransportSearchCache transportSearchCache, SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) { SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) {
super(settings, threadPool, clusterService, transportSearchCache, searchService, searchPhaseController); super(settings, threadPool, clusterService, searchService, searchPhaseController);
} }
@Override @Override
@ -62,13 +57,11 @@ public class TransportSearchDfsQueryAndFetchAction extends TransportSearchTypeAc
private class AsyncAction extends BaseAsyncAction<DfsSearchResult> { private class AsyncAction extends BaseAsyncAction<DfsSearchResult> {
private final Collection<DfsSearchResult> dfsResults = searchCache.obtainDfsResults(); private final AtomicArray<QueryFetchSearchResult> queryFetchResults;
private final Map<SearchShardTarget, QueryFetchSearchResult> queryFetchResults = searchCache.obtainQueryFetchResults();
private AsyncAction(SearchRequest request, ActionListener<SearchResponse> listener) { private AsyncAction(SearchRequest request, ActionListener<SearchResponse> listener) {
super(request, listener); super(request, listener);
queryFetchResults = new AtomicArray<QueryFetchSearchResult>(firstResults.length());
} }
@Override @Override
@ -81,24 +74,20 @@ public class TransportSearchDfsQueryAndFetchAction extends TransportSearchTypeAc
searchService.sendExecuteDfs(node, request, listener); searchService.sendExecuteDfs(node, request, listener);
} }
@Override
protected void processFirstPhaseResult(ShardRouting shard, DfsSearchResult result) {
dfsResults.add(result);
}
@Override @Override
protected void moveToSecondPhase() { protected void moveToSecondPhase() {
final AggregatedDfs dfs = searchPhaseController.aggregateDfs(dfsResults); final AggregatedDfs dfs = searchPhaseController.aggregateDfs(firstResults);
final AtomicInteger counter = new AtomicInteger(dfsResults.size()); final AtomicInteger counter = new AtomicInteger(firstResults.asList().size());
int localOperations = 0; int localOperations = 0;
for (final DfsSearchResult dfsResult : dfsResults) { for (final AtomicArray.Entry<DfsSearchResult> entry : firstResults.asList()) {
DfsSearchResult dfsResult = entry.value;
DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId()); DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId());
if (node.id().equals(nodes.localNodeId())) { if (node.id().equals(nodes.localNodeId())) {
localOperations++; localOperations++;
} else { } else {
QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs); QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs);
executeSecondPhase(dfsResult, counter, node, querySearchRequest); executeSecondPhase(entry.index, dfsResult, counter, node, querySearchRequest);
} }
} }
if (localOperations > 0) { if (localOperations > 0) {
@ -106,18 +95,20 @@ public class TransportSearchDfsQueryAndFetchAction extends TransportSearchTypeAc
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
for (final DfsSearchResult dfsResult : dfsResults) { for (final AtomicArray.Entry<DfsSearchResult> entry : firstResults.asList()) {
DfsSearchResult dfsResult = entry.value;
DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId()); DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId());
if (node.id().equals(nodes.localNodeId())) { if (node.id().equals(nodes.localNodeId())) {
QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs); QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs);
executeSecondPhase(dfsResult, counter, node, querySearchRequest); executeSecondPhase(entry.index, dfsResult, counter, node, querySearchRequest);
} }
} }
} }
}); });
} else { } else {
boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD; boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD;
for (final DfsSearchResult dfsResult : dfsResults) { for (final AtomicArray.Entry<DfsSearchResult> entry : firstResults.asList()) {
final DfsSearchResult dfsResult = entry.value;
final DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId()); final DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId());
if (node.id().equals(nodes.localNodeId())) { if (node.id().equals(nodes.localNodeId())) {
final QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs); final QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs);
@ -125,11 +116,11 @@ public class TransportSearchDfsQueryAndFetchAction extends TransportSearchTypeAc
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
executeSecondPhase(dfsResult, counter, node, querySearchRequest); executeSecondPhase(entry.index, dfsResult, counter, node, querySearchRequest);
} }
}); });
} else { } else {
executeSecondPhase(dfsResult, counter, node, querySearchRequest); executeSecondPhase(entry.index, dfsResult, counter, node, querySearchRequest);
} }
} }
} }
@ -137,12 +128,12 @@ public class TransportSearchDfsQueryAndFetchAction extends TransportSearchTypeAc
} }
} }
void executeSecondPhase(final DfsSearchResult dfsResult, final AtomicInteger counter, DiscoveryNode node, final QuerySearchRequest querySearchRequest) { void executeSecondPhase(final int shardRequestId, final DfsSearchResult dfsResult, final AtomicInteger counter, DiscoveryNode node, final QuerySearchRequest querySearchRequest) {
searchService.sendExecuteFetch(node, querySearchRequest, new SearchServiceListener<QueryFetchSearchResult>() { searchService.sendExecuteFetch(node, querySearchRequest, new SearchServiceListener<QueryFetchSearchResult>() {
@Override @Override
public void onResult(QueryFetchSearchResult result) { public void onResult(QueryFetchSearchResult result) {
result.shardTarget(dfsResult.shardTarget()); result.shardTarget(dfsResult.shardTarget());
queryFetchResults.put(result.shardTarget(), result); queryFetchResults.set(shardRequestId, result);
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
finishHim(); finishHim();
} }
@ -153,7 +144,7 @@ public class TransportSearchDfsQueryAndFetchAction extends TransportSearchTypeAc
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("[{}] Failed to execute query phase", t, querySearchRequest.id()); logger.debug("[{}] Failed to execute query phase", t, querySearchRequest.id());
} }
AsyncAction.this.addShardFailure(new ShardSearchFailure(t)); AsyncAction.this.addShardFailure(shardRequestId, new ShardSearchFailure(t));
successulOps.decrementAndGet(); successulOps.decrementAndGet();
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
finishHim(); finishHim();
@ -172,17 +163,16 @@ public class TransportSearchDfsQueryAndFetchAction extends TransportSearchTypeAc
} }
listener.onFailure(failure); listener.onFailure(failure);
} finally { } finally {
searchCache.releaseDfsResults(dfsResults); //
searchCache.releaseQueryFetchResults(queryFetchResults);
} }
} }
void innerFinishHim() throws Exception { void innerFinishHim() throws Exception {
sortedShardList = searchPhaseController.sortDocs(queryFetchResults.values()); sortedShardList = searchPhaseController.sortDocs(queryFetchResults);
final InternalSearchResponse internalResponse = searchPhaseController.merge(sortedShardList, queryFetchResults, queryFetchResults); final InternalSearchResponse internalResponse = searchPhaseController.merge(sortedShardList, queryFetchResults, queryFetchResults);
String scrollId = null; String scrollId = null;
if (request.scroll() != null) { if (request.scroll() != null) {
scrollId = buildScrollId(request.searchType(), dfsResults, null); scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults, null);
} }
listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successulOps.get(), buildTookInMillis(), buildShardFailures())); listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successulOps.get(), buildTookInMillis(), buildShardFailures()));
} }

106
src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java
View File

@ -23,10 +23,10 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.search.*; import org.elasticsearch.action.search.*;
import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.common.trove.ExtTIntArrayList;
import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.search.action.SearchServiceListener; import org.elasticsearch.search.action.SearchServiceListener;
import org.elasticsearch.search.action.SearchServiceTransportAction; import org.elasticsearch.search.action.SearchServiceTransportAction;
@ -39,11 +39,8 @@ import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.internal.ShardSearchRequest;
import org.elasticsearch.search.query.QuerySearchRequest; import org.elasticsearch.search.query.QuerySearchRequest;
import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.query.QuerySearchResult;
import org.elasticsearch.search.query.QuerySearchResultProvider;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
/** /**
@ -53,8 +50,8 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
@Inject @Inject
public TransportSearchDfsQueryThenFetchAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, public TransportSearchDfsQueryThenFetchAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
TransportSearchCache transportSearchCache, SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) { SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) {
super(settings, threadPool, clusterService, transportSearchCache, searchService, searchPhaseController); super(settings, threadPool, clusterService, searchService, searchPhaseController);
} }
@Override @Override
@ -64,16 +61,15 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
private class AsyncAction extends BaseAsyncAction<DfsSearchResult> { private class AsyncAction extends BaseAsyncAction<DfsSearchResult> {
private final Collection<DfsSearchResult> dfsResults = searchCache.obtainDfsResults(); final AtomicArray<QuerySearchResult> queryResults;
final AtomicArray<FetchSearchResult> fetchResults;
private final Map<SearchShardTarget, QuerySearchResultProvider> queryResults = searchCache.obtainQueryResults(); final AtomicArray<ExtTIntArrayList> docIdsToLoad;
private final Map<SearchShardTarget, FetchSearchResult> fetchResults = searchCache.obtainFetchResults();
private volatile Map<SearchShardTarget, ExtTIntArrayList> docIdsToLoad;
private AsyncAction(SearchRequest request, ActionListener<SearchResponse> listener) { private AsyncAction(SearchRequest request, ActionListener<SearchResponse> listener) {
super(request, listener); super(request, listener);
queryResults = new AtomicArray<QuerySearchResult>(firstResults.length());
fetchResults = new AtomicArray<FetchSearchResult>(firstResults.length());
docIdsToLoad = new AtomicArray<ExtTIntArrayList>(firstResults.length());
} }
@Override @Override
@ -86,25 +82,20 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
searchService.sendExecuteDfs(node, request, listener); searchService.sendExecuteDfs(node, request, listener);
} }
@Override
protected void processFirstPhaseResult(ShardRouting shard, DfsSearchResult result) {
dfsResults.add(result);
}
@Override @Override
protected void moveToSecondPhase() { protected void moveToSecondPhase() {
final AggregatedDfs dfs = searchPhaseController.aggregateDfs(dfsResults); final AggregatedDfs dfs = searchPhaseController.aggregateDfs(firstResults);
final AtomicInteger counter = new AtomicInteger(dfsResults.size()); final AtomicInteger counter = new AtomicInteger(firstResults.asList().size());
int localOperations = 0; int localOperations = 0;
for (final DfsSearchResult dfsResult : dfsResults) { for (final AtomicArray.Entry<DfsSearchResult> entry : firstResults.asList()) {
DfsSearchResult dfsResult = entry.value;
DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId()); DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId());
if (node.id().equals(nodes.localNodeId())) { if (node.id().equals(nodes.localNodeId())) {
localOperations++; localOperations++;
} else { } else {
QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs); QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs);
executeQuery(dfsResult, counter, querySearchRequest, node); executeQuery(entry.index, dfsResult, counter, querySearchRequest, node);
} }
} }
@ -113,18 +104,20 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
for (final DfsSearchResult dfsResult : dfsResults) { for (final AtomicArray.Entry<DfsSearchResult> entry : firstResults.asList()) {
DfsSearchResult dfsResult = entry.value;
DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId()); DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId());
if (node.id().equals(nodes.localNodeId())) { if (node.id().equals(nodes.localNodeId())) {
QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs); QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs);
executeQuery(dfsResult, counter, querySearchRequest, node); executeQuery(entry.index, dfsResult, counter, querySearchRequest, node);
} }
} }
} }
}); });
} else { } else {
boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD; boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD;
for (final DfsSearchResult dfsResult : dfsResults) { for (final AtomicArray.Entry<DfsSearchResult> entry : firstResults.asList()) {
final DfsSearchResult dfsResult = entry.value;
final DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId()); final DiscoveryNode node = nodes.get(dfsResult.shardTarget().nodeId());
if (node.id().equals(nodes.localNodeId())) { if (node.id().equals(nodes.localNodeId())) {
final QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs); final QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs);
@ -132,11 +125,11 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
executeQuery(dfsResult, counter, querySearchRequest, node); executeQuery(entry.index, dfsResult, counter, querySearchRequest, node);
} }
}); });
} else { } else {
executeQuery(dfsResult, counter, querySearchRequest, node); executeQuery(entry.index, dfsResult, counter, querySearchRequest, node);
} }
} }
} }
@ -144,12 +137,12 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
} }
} }
void executeQuery(final DfsSearchResult dfsResult, final AtomicInteger counter, final QuerySearchRequest querySearchRequest, DiscoveryNode node) { void executeQuery(final int shardRequestId, final DfsSearchResult dfsResult, final AtomicInteger counter, final QuerySearchRequest querySearchRequest, DiscoveryNode node) {
searchService.sendExecuteQuery(node, querySearchRequest, new SearchServiceListener<QuerySearchResult>() { searchService.sendExecuteQuery(node, querySearchRequest, new SearchServiceListener<QuerySearchResult>() {
@Override @Override
public void onResult(QuerySearchResult result) { public void onResult(QuerySearchResult result) {
result.shardTarget(dfsResult.shardTarget()); result.shardTarget(dfsResult.shardTarget());
queryResults.put(result.shardTarget(), result); queryResults.set(shardRequestId, result);
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
executeFetchPhase(); executeFetchPhase();
} }
@ -160,7 +153,7 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("[{}] Failed to execute query phase", t, querySearchRequest.id()); logger.debug("[{}] Failed to execute query phase", t, querySearchRequest.id());
} }
AsyncAction.this.addShardFailure(new ShardSearchFailure(t)); AsyncAction.this.addShardFailure(shardRequestId, new ShardSearchFailure(t));
successulOps.decrementAndGet(); successulOps.decrementAndGet();
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
executeFetchPhase(); executeFetchPhase();
@ -178,24 +171,24 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
} }
void innerExecuteFetchPhase() { void innerExecuteFetchPhase() {
sortedShardList = searchPhaseController.sortDocs(queryResults.values()); sortedShardList = searchPhaseController.sortDocs(queryResults);
final Map<SearchShardTarget, ExtTIntArrayList> docIdsToLoad = searchPhaseController.docIdsToLoad(sortedShardList); searchPhaseController.fillDocIdsToLoad(docIdsToLoad, sortedShardList);
this.docIdsToLoad = docIdsToLoad;
if (docIdsToLoad.isEmpty()) { if (docIdsToLoad.asList().isEmpty()) {
finishHim(); finishHim();
return; return;
} }
final AtomicInteger counter = new AtomicInteger(docIdsToLoad.size()); final AtomicInteger counter = new AtomicInteger(docIdsToLoad.asList().size());
int localOperations = 0; int localOperations = 0;
for (final Map.Entry<SearchShardTarget, ExtTIntArrayList> entry : docIdsToLoad.entrySet()) { for (final AtomicArray.Entry<ExtTIntArrayList> entry : docIdsToLoad.asList()) {
DiscoveryNode node = nodes.get(entry.getKey().nodeId()); QuerySearchResult queryResult = queryResults.get(entry.index);
DiscoveryNode node = nodes.get(queryResult.shardTarget().nodeId());
if (node.id().equals(nodes.localNodeId())) { if (node.id().equals(nodes.localNodeId())) {
localOperations++; localOperations++;
} else { } else {
FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResults.get(entry.getKey()).id(), entry.getValue()); FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResult.id(), entry.value);
executeFetch(entry.getKey(), counter, fetchSearchRequest, node); executeFetch(entry.index, queryResult.shardTarget(), counter, fetchSearchRequest, node);
} }
} }
@ -204,30 +197,32 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
for (final Map.Entry<SearchShardTarget, ExtTIntArrayList> entry : docIdsToLoad.entrySet()) { for (final AtomicArray.Entry<ExtTIntArrayList> entry : docIdsToLoad.asList()) {
DiscoveryNode node = nodes.get(entry.getKey().nodeId()); QuerySearchResult queryResult = queryResults.get(entry.index);
DiscoveryNode node = nodes.get(queryResult.shardTarget().nodeId());
if (node.id().equals(nodes.localNodeId())) { if (node.id().equals(nodes.localNodeId())) {
FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResults.get(entry.getKey()).id(), entry.getValue()); FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResult.id(), entry.value);
executeFetch(entry.getKey(), counter, fetchSearchRequest, node); executeFetch(entry.index, queryResult.shardTarget(), counter, fetchSearchRequest, node);
} }
} }
} }
}); });
} else { } else {
boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD; boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD;
for (final Map.Entry<SearchShardTarget, ExtTIntArrayList> entry : docIdsToLoad.entrySet()) { for (final AtomicArray.Entry<ExtTIntArrayList> entry : docIdsToLoad.asList()) {
final DiscoveryNode node = nodes.get(entry.getKey().nodeId()); final QuerySearchResult queryResult = queryResults.get(entry.index);
final DiscoveryNode node = nodes.get(queryResult.shardTarget().nodeId());
if (node.id().equals(nodes.localNodeId())) { if (node.id().equals(nodes.localNodeId())) {
final FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResults.get(entry.getKey()).id(), entry.getValue()); final FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResult.id(), entry.value);
if (localAsync) { if (localAsync) {
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
executeFetch(entry.getKey(), counter, fetchSearchRequest, node); executeFetch(entry.index, queryResult.shardTarget(), counter, fetchSearchRequest, node);
} }
}); });
} else { } else {
executeFetch(entry.getKey(), counter, fetchSearchRequest, node); executeFetch(entry.index, queryResult.shardTarget(), counter, fetchSearchRequest, node);
} }
} }
} }
@ -235,12 +230,12 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
} }
} }
void executeFetch(final SearchShardTarget shardTarget, final AtomicInteger counter, final FetchSearchRequest fetchSearchRequest, DiscoveryNode node) { void executeFetch(final int shardRequestId, final SearchShardTarget shardTarget, final AtomicInteger counter, final FetchSearchRequest fetchSearchRequest, DiscoveryNode node) {
searchService.sendExecuteFetch(node, fetchSearchRequest, new SearchServiceListener<FetchSearchResult>() { searchService.sendExecuteFetch(node, fetchSearchRequest, new SearchServiceListener<FetchSearchResult>() {
@Override @Override
public void onResult(FetchSearchResult result) { public void onResult(FetchSearchResult result) {
result.shardTarget(shardTarget); result.shardTarget(shardTarget);
fetchResults.put(result.shardTarget(), result); fetchResults.set(shardRequestId, result);
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
finishHim(); finishHim();
} }
@ -251,7 +246,7 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("[{}] Failed to execute fetch phase", t, fetchSearchRequest.id()); logger.debug("[{}] Failed to execute fetch phase", t, fetchSearchRequest.id());
} }
AsyncAction.this.addShardFailure(new ShardSearchFailure(t)); AsyncAction.this.addShardFailure(shardRequestId, new ShardSearchFailure(t));
successulOps.decrementAndGet(); successulOps.decrementAndGet();
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
finishHim(); finishHim();
@ -264,16 +259,13 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
try { try {
innerFinishHim(); innerFinishHim();
} catch (Throwable e) { } catch (Throwable e) {
ReduceSearchPhaseException failure = new ReduceSearchPhaseException("fetch", "", e, buildShardFailures()); ReduceSearchPhaseException failure = new ReduceSearchPhaseException("merge", "", e, buildShardFailures());
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("failed to reduce search", failure); logger.debug("failed to reduce search", failure);
} }
listener.onFailure(failure); listener.onFailure(failure);
} finally { } finally {
releaseIrrelevantSearchContexts(queryResults, docIdsToLoad); releaseIrrelevantSearchContexts(queryResults, docIdsToLoad);
searchCache.releaseDfsResults(dfsResults);
searchCache.releaseQueryResults(queryResults);
searchCache.releaseFetchResults(fetchResults);
} }
} }
@ -281,7 +273,7 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA
final InternalSearchResponse internalResponse = searchPhaseController.merge(sortedShardList, queryResults, fetchResults); final InternalSearchResponse internalResponse = searchPhaseController.merge(sortedShardList, queryResults, fetchResults);
String scrollId = null; String scrollId = null;
if (request.scroll() != null) { if (request.scroll() != null) {
scrollId = TransportSearchHelper.buildScrollId(request.searchType(), dfsResults, null); scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults, null);
} }
listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successulOps.get(), buildTookInMillis(), buildShardFailures())); listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successulOps.get(), buildTookInMillis(), buildShardFailures()));
} }

12
src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.action.search.type;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util.UnicodeUtil;
@ -35,12 +34,12 @@ import org.elasticsearch.common.Base64;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchPhaseResult;
import org.elasticsearch.search.internal.InternalScrollSearchRequest; import org.elasticsearch.search.internal.InternalScrollSearchRequest;
import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.internal.ShardSearchRequest;
import java.io.IOException; import java.io.IOException;
import java.util.Collection;
import java.util.Map; import java.util.Map;
/** /**
@ -59,7 +58,7 @@ public abstract class TransportSearchHelper {
return new InternalScrollSearchRequest(request, id); return new InternalScrollSearchRequest(request, id);
} }
public static String buildScrollId(SearchType searchType, Collection<? extends SearchPhaseResult> searchPhaseResults, @Nullable Map<String, String> attributes) throws IOException { public static String buildScrollId(SearchType searchType, AtomicArray<? extends SearchPhaseResult> searchPhaseResults, @Nullable Map<String, String> attributes) throws IOException {
if (searchType == SearchType.DFS_QUERY_THEN_FETCH || searchType == SearchType.QUERY_THEN_FETCH) { if (searchType == SearchType.DFS_QUERY_THEN_FETCH || searchType == SearchType.QUERY_THEN_FETCH) {
return buildScrollId(ParsedScrollId.QUERY_THEN_FETCH_TYPE, searchPhaseResults, attributes); return buildScrollId(ParsedScrollId.QUERY_THEN_FETCH_TYPE, searchPhaseResults, attributes);
} else if (searchType == SearchType.QUERY_AND_FETCH || searchType == SearchType.DFS_QUERY_AND_FETCH) { } else if (searchType == SearchType.QUERY_AND_FETCH || searchType == SearchType.DFS_QUERY_AND_FETCH) {
@ -71,10 +70,11 @@ public abstract class TransportSearchHelper {
} }
} }
public static String buildScrollId(String type, Collection<? extends SearchPhaseResult> searchPhaseResults, @Nullable Map<String, String> attributes) throws IOException { public static String buildScrollId(String type, AtomicArray<? extends SearchPhaseResult> searchPhaseResults, @Nullable Map<String, String> attributes) throws IOException {
StringBuilder sb = new StringBuilder().append(type).append(';'); StringBuilder sb = new StringBuilder().append(type).append(';');
sb.append(searchPhaseResults.size()).append(';'); sb.append(searchPhaseResults.asList().size()).append(';');
for (SearchPhaseResult searchPhaseResult : searchPhaseResults) { for (AtomicArray.Entry<? extends SearchPhaseResult> entry : searchPhaseResults.asList()) {
SearchPhaseResult searchPhaseResult = entry.value;
sb.append(searchPhaseResult.id()).append(':').append(searchPhaseResult.shardTarget().nodeId()).append(';'); sb.append(searchPhaseResult.id()).append(':').append(searchPhaseResult.shardTarget().nodeId()).append(';');
} }
if (attributes == null) { if (attributes == null) {

34
src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryAndFetchAction.java
View File

@ -20,14 +20,13 @@
package org.elasticsearch.action.search.type; package org.elasticsearch.action.search.type;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.search.ReduceSearchPhaseException;
import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.search.action.SearchServiceListener; import org.elasticsearch.search.action.SearchServiceListener;
import org.elasticsearch.search.action.SearchServiceTransportAction; import org.elasticsearch.search.action.SearchServiceTransportAction;
import org.elasticsearch.search.controller.SearchPhaseController; import org.elasticsearch.search.controller.SearchPhaseController;
@ -36,7 +35,7 @@ import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.internal.ShardSearchRequest;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import java.util.Map; import java.io.IOException;
import static org.elasticsearch.action.search.type.TransportSearchHelper.buildScrollId; import static org.elasticsearch.action.search.type.TransportSearchHelper.buildScrollId;
@ -47,8 +46,8 @@ public class TransportSearchQueryAndFetchAction extends TransportSearchTypeActio
@Inject @Inject
public TransportSearchQueryAndFetchAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, public TransportSearchQueryAndFetchAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
TransportSearchCache transportSearchCache, SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) { SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) {
super(settings, threadPool, clusterService, transportSearchCache, searchService, searchPhaseController); super(settings, threadPool, clusterService, searchService, searchPhaseController);
} }
@Override @Override
@ -58,9 +57,6 @@ public class TransportSearchQueryAndFetchAction extends TransportSearchTypeActio
private class AsyncAction extends BaseAsyncAction<QueryFetchSearchResult> { private class AsyncAction extends BaseAsyncAction<QueryFetchSearchResult> {
private final Map<SearchShardTarget, QueryFetchSearchResult> queryFetchResults = searchCache.obtainQueryFetchResults();
private AsyncAction(SearchRequest request, ActionListener<SearchResponse> listener) { private AsyncAction(SearchRequest request, ActionListener<SearchResponse> listener) {
super(request, listener); super(request, listener);
} }
@ -76,20 +72,26 @@ public class TransportSearchQueryAndFetchAction extends TransportSearchTypeActio
} }
@Override @Override
protected void processFirstPhaseResult(ShardRouting shard, QueryFetchSearchResult result) { protected void moveToSecondPhase() throws Exception {
queryFetchResults.put(result.shardTarget(), result); try {
innerFinishHim();
} catch (Throwable e) {
ReduceSearchPhaseException failure = new ReduceSearchPhaseException("merge", "", e, buildShardFailures());
if (logger.isDebugEnabled()) {
logger.debug("failed to reduce search", failure);
}
listener.onFailure(failure);
}
} }
@Override private void innerFinishHim() throws IOException {
protected void moveToSecondPhase() throws Exception { sortedShardList = searchPhaseController.sortDocs(firstResults);
sortedShardList = searchPhaseController.sortDocs(queryFetchResults.values()); final InternalSearchResponse internalResponse = searchPhaseController.merge(sortedShardList, firstResults, firstResults);
final InternalSearchResponse internalResponse = searchPhaseController.merge(sortedShardList, queryFetchResults, queryFetchResults);
String scrollId = null; String scrollId = null;
if (request.scroll() != null) { if (request.scroll() != null) {
scrollId = buildScrollId(request.searchType(), queryFetchResults.values(), null); scrollId = buildScrollId(request.searchType(), firstResults, null);
} }
listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successulOps.get(), buildTookInMillis(), buildShardFailures())); listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successulOps.get(), buildTookInMillis(), buildShardFailures()));
searchCache.releaseQueryFetchResults(queryFetchResults);
} }
} }
} }

74
src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java
View File

@ -23,10 +23,10 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.search.*; import org.elasticsearch.action.search.*;
import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.common.trove.ExtTIntArrayList;
import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.search.action.SearchServiceListener; import org.elasticsearch.search.action.SearchServiceListener;
import org.elasticsearch.search.action.SearchServiceTransportAction; import org.elasticsearch.search.action.SearchServiceTransportAction;
@ -36,10 +36,8 @@ import org.elasticsearch.search.fetch.FetchSearchResult;
import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.internal.ShardSearchRequest;
import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.query.QuerySearchResult;
import org.elasticsearch.search.query.QuerySearchResultProvider;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
/** /**
@ -49,8 +47,8 @@ public class TransportSearchQueryThenFetchAction extends TransportSearchTypeActi
@Inject @Inject
public TransportSearchQueryThenFetchAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, public TransportSearchQueryThenFetchAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
TransportSearchCache transportSearchCache, SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) { SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) {
super(settings, threadPool, clusterService, transportSearchCache, searchService, searchPhaseController); super(settings, threadPool, clusterService, searchService, searchPhaseController);
} }
@Override @Override
@ -60,14 +58,13 @@ public class TransportSearchQueryThenFetchAction extends TransportSearchTypeActi
private class AsyncAction extends BaseAsyncAction<QuerySearchResult> { private class AsyncAction extends BaseAsyncAction<QuerySearchResult> {
private final Map<SearchShardTarget, QuerySearchResultProvider> queryResults = searchCache.obtainQueryResults(); final AtomicArray<FetchSearchResult> fetchResults;
final AtomicArray<ExtTIntArrayList> docIdsToLoad;
private final Map<SearchShardTarget, FetchSearchResult> fetchResults = searchCache.obtainFetchResults();
private volatile Map<SearchShardTarget, ExtTIntArrayList> docIdsToLoad;
private AsyncAction(SearchRequest request, ActionListener<SearchResponse> listener) { private AsyncAction(SearchRequest request, ActionListener<SearchResponse> listener) {
super(request, listener); super(request, listener);
fetchResults = new AtomicArray<FetchSearchResult>(firstResults.length());
docIdsToLoad = new AtomicArray<ExtTIntArrayList>(firstResults.length());
} }
@Override @Override
@ -80,32 +77,27 @@ public class TransportSearchQueryThenFetchAction extends TransportSearchTypeActi
searchService.sendExecuteQuery(node, request, listener); searchService.sendExecuteQuery(node, request, listener);
} }
@Override
protected void processFirstPhaseResult(ShardRouting shard, QuerySearchResult result) {
queryResults.put(result.shardTarget(), result);
}
@Override @Override
protected void moveToSecondPhase() { protected void moveToSecondPhase() {
sortedShardList = searchPhaseController.sortDocs(queryResults.values()); sortedShardList = searchPhaseController.sortDocs(firstResults);
final Map<SearchShardTarget, ExtTIntArrayList> docIdsToLoad = searchPhaseController.docIdsToLoad(sortedShardList); searchPhaseController.fillDocIdsToLoad(docIdsToLoad, sortedShardList);
this.docIdsToLoad = docIdsToLoad;
if (docIdsToLoad.isEmpty()) { if (docIdsToLoad.asList().isEmpty()) {
finishHim(); finishHim();
return; return;
} }
final AtomicInteger counter = new AtomicInteger(docIdsToLoad.size()); final AtomicInteger counter = new AtomicInteger(docIdsToLoad.asList().size());
int localOperations = 0; int localOperations = 0;
for (final Map.Entry<SearchShardTarget, ExtTIntArrayList> entry : docIdsToLoad.entrySet()) { for (AtomicArray.Entry<ExtTIntArrayList> entry : docIdsToLoad.asList()) {
DiscoveryNode node = nodes.get(entry.getKey().nodeId()); QuerySearchResult queryResult = firstResults.get(entry.index);
DiscoveryNode node = nodes.get(queryResult.shardTarget().nodeId());
if (node.id().equals(nodes.localNodeId())) { if (node.id().equals(nodes.localNodeId())) {
localOperations++; localOperations++;
} else { } else {
FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResults.get(entry.getKey()).id(), entry.getValue()); FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResult.id(), entry.value);
executeFetch(entry.getKey(), counter, fetchSearchRequest, node); executeFetch(entry.index, queryResult.shardTarget(), counter, fetchSearchRequest, node);
} }
} }
@ -114,30 +106,32 @@ public class TransportSearchQueryThenFetchAction extends TransportSearchTypeActi
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
for (final Map.Entry<SearchShardTarget, ExtTIntArrayList> entry : docIdsToLoad.entrySet()) { for (AtomicArray.Entry<ExtTIntArrayList> entry : docIdsToLoad.asList()) {
DiscoveryNode node = nodes.get(entry.getKey().nodeId()); QuerySearchResult queryResult = firstResults.get(entry.index);
DiscoveryNode node = nodes.get(queryResult.shardTarget().nodeId());
if (node.id().equals(nodes.localNodeId())) { if (node.id().equals(nodes.localNodeId())) {
FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResults.get(entry.getKey()).id(), entry.getValue()); FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResult.id(), entry.value);
executeFetch(entry.getKey(), counter, fetchSearchRequest, node); executeFetch(entry.index, queryResult.shardTarget(), counter, fetchSearchRequest, node);
} }
} }
} }
}); });
} else { } else {
boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD; boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD;
for (final Map.Entry<SearchShardTarget, ExtTIntArrayList> entry : docIdsToLoad.entrySet()) { for (final AtomicArray.Entry<ExtTIntArrayList> entry : docIdsToLoad.asList()) {
final DiscoveryNode node = nodes.get(entry.getKey().nodeId()); final QuerySearchResult queryResult = firstResults.get(entry.index);
final DiscoveryNode node = nodes.get(queryResult.shardTarget().nodeId());
if (node.id().equals(nodes.localNodeId())) { if (node.id().equals(nodes.localNodeId())) {
final FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResults.get(entry.getKey()).id(), entry.getValue()); final FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResult.id(), entry.value);
if (localAsync) { if (localAsync) {
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
executeFetch(entry.getKey(), counter, fetchSearchRequest, node); executeFetch(entry.index, queryResult.shardTarget(), counter, fetchSearchRequest, node);
} }
}); });
} else { } else {
executeFetch(entry.getKey(), counter, fetchSearchRequest, node); executeFetch(entry.index, queryResult.shardTarget(), counter, fetchSearchRequest, node);
} }
} }
} }
@ -145,12 +139,12 @@ public class TransportSearchQueryThenFetchAction extends TransportSearchTypeActi
} }
} }
void executeFetch(final SearchShardTarget shardTarget, final AtomicInteger counter, final FetchSearchRequest fetchSearchRequest, DiscoveryNode node) { void executeFetch(final int shardRequestId, final SearchShardTarget shardTarget, final AtomicInteger counter, final FetchSearchRequest fetchSearchRequest, DiscoveryNode node) {
searchService.sendExecuteFetch(node, fetchSearchRequest, new SearchServiceListener<FetchSearchResult>() { searchService.sendExecuteFetch(node, fetchSearchRequest, new SearchServiceListener<FetchSearchResult>() {
@Override @Override
public void onResult(FetchSearchResult result) { public void onResult(FetchSearchResult result) {
result.shardTarget(shardTarget); result.shardTarget(shardTarget);
fetchResults.put(result.shardTarget(), result); fetchResults.set(shardRequestId, result);
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
finishHim(); finishHim();
} }
@ -161,7 +155,7 @@ public class TransportSearchQueryThenFetchAction extends TransportSearchTypeActi
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("[{}] Failed to execute fetch phase", t, fetchSearchRequest.id()); logger.debug("[{}] Failed to execute fetch phase", t, fetchSearchRequest.id());
} }
AsyncAction.this.addShardFailure(new ShardSearchFailure(t)); AsyncAction.this.addShardFailure(shardRequestId, new ShardSearchFailure(t));
successulOps.decrementAndGet(); successulOps.decrementAndGet();
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
finishHim(); finishHim();
@ -180,17 +174,15 @@ public class TransportSearchQueryThenFetchAction extends TransportSearchTypeActi
} }
listener.onFailure(failure); listener.onFailure(failure);
} finally { } finally {
releaseIrrelevantSearchContexts(queryResults, docIdsToLoad); releaseIrrelevantSearchContexts(firstResults, docIdsToLoad);
searchCache.releaseQueryResults(queryResults);
searchCache.releaseFetchResults(fetchResults);
} }
} }
void innerFinishHim() throws Exception { void innerFinishHim() throws Exception {
InternalSearchResponse internalResponse = searchPhaseController.merge(sortedShardList, queryResults, fetchResults); InternalSearchResponse internalResponse = searchPhaseController.merge(sortedShardList, firstResults, fetchResults);
String scrollId = null; String scrollId = null;
if (request.scroll() != null) { if (request.scroll() != null) {
scrollId = TransportSearchHelper.buildScrollId(request.searchType(), queryResults.values(), null); scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults, null);
} }
listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successulOps.get(), buildTookInMillis(), buildShardFailures())); listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successulOps.get(), buildTookInMillis(), buildShardFailures()));
} }

22
src/main/java/org/elasticsearch/action/search/type/TransportSearchScanAction.java
View File

@ -25,10 +25,9 @@ import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.search.action.SearchServiceListener; import org.elasticsearch.search.action.SearchServiceListener;
import org.elasticsearch.search.action.SearchServiceTransportAction; import org.elasticsearch.search.action.SearchServiceTransportAction;
import org.elasticsearch.search.controller.SearchPhaseController; import org.elasticsearch.search.controller.SearchPhaseController;
@ -37,19 +36,16 @@ import org.elasticsearch.search.fetch.FetchSearchResultProvider;
import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.internal.ShardSearchRequest;
import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.query.QuerySearchResult;
import org.elasticsearch.search.query.QuerySearchResultProvider;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import java.util.Map;
import static org.elasticsearch.action.search.type.TransportSearchHelper.buildScrollId; import static org.elasticsearch.action.search.type.TransportSearchHelper.buildScrollId;
public class TransportSearchScanAction extends TransportSearchTypeAction { public class TransportSearchScanAction extends TransportSearchTypeAction {
@Inject @Inject
public TransportSearchScanAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, public TransportSearchScanAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
TransportSearchCache transportSearchCache, SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) { SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) {
super(settings, threadPool, clusterService, transportSearchCache, searchService, searchPhaseController); super(settings, threadPool, clusterService, searchService, searchPhaseController);
} }
@Override @Override
@ -59,8 +55,6 @@ public class TransportSearchScanAction extends TransportSearchTypeAction {
private class AsyncAction extends BaseAsyncAction<QuerySearchResult> { private class AsyncAction extends BaseAsyncAction<QuerySearchResult> {
private final Map<SearchShardTarget, QuerySearchResultProvider> queryResults = searchCache.obtainQueryResults();
private AsyncAction(SearchRequest request, ActionListener<SearchResponse> listener) { private AsyncAction(SearchRequest request, ActionListener<SearchResponse> listener) {
super(request, listener); super(request, listener);
} }
@ -75,20 +69,14 @@ public class TransportSearchScanAction extends TransportSearchTypeAction {
searchService.sendExecuteScan(node, request, listener); searchService.sendExecuteScan(node, request, listener);
} }
@Override
protected void processFirstPhaseResult(ShardRouting shard, QuerySearchResult result) {
queryResults.put(result.shardTarget(), result);
}
@Override @Override
protected void moveToSecondPhase() throws Exception { protected void moveToSecondPhase() throws Exception {
final InternalSearchResponse internalResponse = searchPhaseController.merge(EMPTY_DOCS, queryResults, ImmutableMap.<SearchShardTarget, FetchSearchResultProvider>of()); final InternalSearchResponse internalResponse = searchPhaseController.merge(EMPTY_DOCS, firstResults, (AtomicArray<? extends FetchSearchResultProvider>) AtomicArray.empty());
String scrollId = null; String scrollId = null;
if (request.scroll() != null) { if (request.scroll() != null) {
scrollId = buildScrollId(request.searchType(), queryResults.values(), ImmutableMap.of("total_hits", Long.toString(internalResponse.hits().totalHits()))); scrollId = buildScrollId(request.searchType(), firstResults, ImmutableMap.of("total_hits", Long.toString(internalResponse.hits().totalHits())));
} }
listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successulOps.get(), buildTookInMillis(), buildShardFailures())); listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successulOps.get(), buildTookInMillis(), buildShardFailures()));
searchCache.releaseQueryResults(queryResults);
} }
} }

64
src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java
View File

@ -28,8 +28,7 @@ import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.search.action.SearchServiceListener; import org.elasticsearch.search.action.SearchServiceListener;
import org.elasticsearch.search.action.SearchServiceTransportAction; import org.elasticsearch.search.action.SearchServiceTransportAction;
import org.elasticsearch.search.controller.SearchPhaseController; import org.elasticsearch.search.controller.SearchPhaseController;
@ -38,8 +37,7 @@ import org.elasticsearch.search.fetch.QueryFetchSearchResult;
import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import java.util.Map; import java.util.List;
import java.util.Queue;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import static org.elasticsearch.action.search.type.TransportSearchHelper.internalScrollSearchRequest; import static org.elasticsearch.action.search.type.TransportSearchHelper.internalScrollSearchRequest;
@ -57,16 +55,12 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent
private final SearchPhaseController searchPhaseController; private final SearchPhaseController searchPhaseController;
private final TransportSearchCache searchCache;
@Inject @Inject
public TransportSearchScrollQueryAndFetchAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, public TransportSearchScrollQueryAndFetchAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
TransportSearchCache searchCache,
SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) { SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) {
super(settings); super(settings);
this.threadPool = threadPool; this.threadPool = threadPool;
this.clusterService = clusterService; this.clusterService = clusterService;
this.searchCache = searchCache;
this.searchService = searchService; this.searchService = searchService;
this.searchPhaseController = searchPhaseController; this.searchPhaseController = searchPhaseController;
} }
@ -85,12 +79,10 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent
private final DiscoveryNodes nodes; private final DiscoveryNodes nodes;
private volatile Queue<ShardSearchFailure> shardFailures; private volatile AtomicArray<ShardSearchFailure> shardFailures;
private final AtomicArray<QueryFetchSearchResult> queryFetchResults;
private final Map<SearchShardTarget, QueryFetchSearchResult> queryFetchResults = searchCache.obtainQueryFetchResults();
private final AtomicInteger successfulOps; private final AtomicInteger successfulOps;
private final AtomicInteger counter; private final AtomicInteger counter;
private final long startTime = System.currentTimeMillis(); private final long startTime = System.currentTimeMillis();
@ -102,23 +94,29 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent
this.nodes = clusterService.state().nodes(); this.nodes = clusterService.state().nodes();
this.successfulOps = new AtomicInteger(scrollId.getContext().length); this.successfulOps = new AtomicInteger(scrollId.getContext().length);
this.counter = new AtomicInteger(scrollId.getContext().length); this.counter = new AtomicInteger(scrollId.getContext().length);
this.queryFetchResults = new AtomicArray<QueryFetchSearchResult>(scrollId.getContext().length);
} }
protected final ShardSearchFailure[] buildShardFailures() { protected final ShardSearchFailure[] buildShardFailures() {
Queue<ShardSearchFailure> localFailures = shardFailures; if (shardFailures == null) {
if (localFailures == null) {
return ShardSearchFailure.EMPTY_ARRAY; return ShardSearchFailure.EMPTY_ARRAY;
} }
return localFailures.toArray(ShardSearchFailure.EMPTY_ARRAY); List<AtomicArray.Entry<ShardSearchFailure>> entries = shardFailures.asList();
ShardSearchFailure[] failures = new ShardSearchFailure[entries.size()];
for (int i = 0; i < failures.length; i++) {
failures[i] = entries.get(i).value;
}
return failures;
} }
// we do our best to return the shard failures, but its ok if its not fully concurrently safe // we do our best to return the shard failures, but its ok if its not fully concurrently safe
// we simply try and return as much as possible // we simply try and return as much as possible
protected final void addShardFailure(ShardSearchFailure failure) { protected final void addShardFailure(final int shardRequestId, ShardSearchFailure failure) {
if (shardFailures == null) { if (shardFailures == null) {
shardFailures = ConcurrentCollections.newQueue(); shardFailures = new AtomicArray<ShardSearchFailure>(scrollId.getContext().length);
} }
shardFailures.add(failure); shardFailures.set(shardRequestId, failure);
} }
public void start() { public void start() {
@ -128,13 +126,15 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent
} }
int localOperations = 0; int localOperations = 0;
for (Tuple<String, Long> target : scrollId.getContext()) { Tuple<String, Long>[] context = scrollId.getContext();
for (int i = 0; i < context.length; i++) {
Tuple<String, Long> target = context[i];
DiscoveryNode node = nodes.get(target.v1()); DiscoveryNode node = nodes.get(target.v1());
if (node != null) { if (node != null) {
if (nodes.localNodeId().equals(node.id())) { if (nodes.localNodeId().equals(node.id())) {
localOperations++; localOperations++;
} else { } else {
executePhase(node, target.v2()); executePhase(i, node, target.v2());
} }
} else { } else {
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
@ -152,28 +152,33 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
for (Tuple<String, Long> target : scrollId.getContext()) { Tuple<String, Long>[] context1 = scrollId.getContext();
for (int i = 0; i < context1.length; i++) {
Tuple<String, Long> target = context1[i];
DiscoveryNode node = nodes.get(target.v1()); DiscoveryNode node = nodes.get(target.v1());
if (node != null && nodes.localNodeId().equals(node.id())) { if (node != null && nodes.localNodeId().equals(node.id())) {
executePhase(node, target.v2()); executePhase(i, node, target.v2());
} }
} }
} }
}); });
} else { } else {
boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD; boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD;
for (final Tuple<String, Long> target : scrollId.getContext()) { Tuple<String, Long>[] context1 = scrollId.getContext();
for (int i = 0; i < context1.length; i++) {
final Tuple<String, Long> target = context1[i];
final int shardRequestId = i;
final DiscoveryNode node = nodes.get(target.v1()); final DiscoveryNode node = nodes.get(target.v1());
if (node != null && nodes.localNodeId().equals(node.id())) { if (node != null && nodes.localNodeId().equals(node.id())) {
if (localAsync) { if (localAsync) {
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
executePhase(node, target.v2()); executePhase(shardRequestId, node, target.v2());
} }
}); });
} else { } else {
executePhase(node, target.v2()); executePhase(shardRequestId, node, target.v2());
} }
} }
} }
@ -195,11 +200,11 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent
} }
} }
private void executePhase(DiscoveryNode node, final long searchId) { private void executePhase(final int shardRequestId, DiscoveryNode node, final long searchId) {
searchService.sendExecuteFetch(node, internalScrollSearchRequest(searchId, request), new SearchServiceListener<QueryFetchSearchResult>() { searchService.sendExecuteFetch(node, internalScrollSearchRequest(searchId, request), new SearchServiceListener<QueryFetchSearchResult>() {
@Override @Override
public void onResult(QueryFetchSearchResult result) { public void onResult(QueryFetchSearchResult result) {
queryFetchResults.put(result.shardTarget(), result); queryFetchResults.set(shardRequestId, result);
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
finishHim(); finishHim();
} }
@ -210,7 +215,7 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("[{}] Failed to execute query phase", t, searchId); logger.debug("[{}] Failed to execute query phase", t, searchId);
} }
addShardFailure(new ShardSearchFailure(t)); addShardFailure(shardRequestId, new ShardSearchFailure(t));
successfulOps.decrementAndGet(); successfulOps.decrementAndGet();
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
finishHim(); finishHim();
@ -228,13 +233,12 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent
} }
private void innerFinishHim() { private void innerFinishHim() {
ShardDoc[] sortedShardList = searchPhaseController.sortDocs(queryFetchResults.values()); ShardDoc[] sortedShardList = searchPhaseController.sortDocs(queryFetchResults);
final InternalSearchResponse internalResponse = searchPhaseController.merge(sortedShardList, queryFetchResults, queryFetchResults); final InternalSearchResponse internalResponse = searchPhaseController.merge(sortedShardList, queryFetchResults, queryFetchResults);
String scrollId = null; String scrollId = null;
if (request.scroll() != null) { if (request.scroll() != null) {
scrollId = request.scrollId(); scrollId = request.scrollId();
} }
searchCache.releaseQueryFetchResults(queryFetchResults);
listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.getContext().length, successfulOps.get(), listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.getContext().length, successfulOps.get(),
System.currentTimeMillis() - startTime, buildShardFailures())); System.currentTimeMillis() - startTime, buildShardFailures()));
} }

90
src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java
View File

@ -29,8 +29,7 @@ import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.common.trove.ExtTIntArrayList;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.search.action.SearchServiceListener; import org.elasticsearch.search.action.SearchServiceListener;
import org.elasticsearch.search.action.SearchServiceTransportAction; import org.elasticsearch.search.action.SearchServiceTransportAction;
import org.elasticsearch.search.controller.SearchPhaseController; import org.elasticsearch.search.controller.SearchPhaseController;
@ -39,11 +38,9 @@ import org.elasticsearch.search.fetch.FetchSearchRequest;
import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.fetch.FetchSearchResult;
import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.query.QuerySearchResult;
import org.elasticsearch.search.query.QuerySearchResultProvider;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import java.util.Map; import java.util.List;
import java.util.Queue;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import static org.elasticsearch.action.search.type.TransportSearchHelper.internalScrollSearchRequest; import static org.elasticsearch.action.search.type.TransportSearchHelper.internalScrollSearchRequest;
@ -61,16 +58,12 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent
private final SearchPhaseController searchPhaseController; private final SearchPhaseController searchPhaseController;
private final TransportSearchCache searchCache;
@Inject @Inject
public TransportSearchScrollQueryThenFetchAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, public TransportSearchScrollQueryThenFetchAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
TransportSearchCache searchCache,
SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) { SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) {
super(settings); super(settings);
this.threadPool = threadPool; this.threadPool = threadPool;
this.clusterService = clusterService; this.clusterService = clusterService;
this.searchCache = searchCache;
this.searchService = searchService; this.searchService = searchService;
this.searchPhaseController = searchPhaseController; this.searchPhaseController = searchPhaseController;
} }
@ -89,11 +82,9 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent
private final DiscoveryNodes nodes; private final DiscoveryNodes nodes;
protected volatile Queue<ShardSearchFailure> shardFailures; private volatile AtomicArray<ShardSearchFailure> shardFailures;
final AtomicArray<QuerySearchResult> queryResults;
private final Map<SearchShardTarget, QuerySearchResultProvider> queryResults = searchCache.obtainQueryResults(); final AtomicArray<FetchSearchResult> fetchResults;
private final Map<SearchShardTarget, FetchSearchResult> fetchResults = searchCache.obtainFetchResults();
private volatile ShardDoc[] sortedShardList; private volatile ShardDoc[] sortedShardList;
@ -107,23 +98,30 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent
this.scrollId = scrollId; this.scrollId = scrollId;
this.nodes = clusterService.state().nodes(); this.nodes = clusterService.state().nodes();
this.successfulOps = new AtomicInteger(scrollId.getContext().length); this.successfulOps = new AtomicInteger(scrollId.getContext().length);
this.queryResults = new AtomicArray<QuerySearchResult>(scrollId.getContext().length);
this.fetchResults = new AtomicArray<FetchSearchResult>(scrollId.getContext().length);
} }
protected final ShardSearchFailure[] buildShardFailures() { protected final ShardSearchFailure[] buildShardFailures() {
Queue<ShardSearchFailure> localFailures = shardFailures; if (shardFailures == null) {
if (localFailures == null) {
return ShardSearchFailure.EMPTY_ARRAY; return ShardSearchFailure.EMPTY_ARRAY;
} }
return localFailures.toArray(ShardSearchFailure.EMPTY_ARRAY); List<AtomicArray.Entry<ShardSearchFailure>> entries = shardFailures.asList();
ShardSearchFailure[] failures = new ShardSearchFailure[entries.size()];
for (int i = 0; i < failures.length; i++) {
failures[i] = entries.get(i).value;
}
return failures;
} }
// we do our best to return the shard failures, but its ok if its not fully concurrently safe // we do our best to return the shard failures, but its ok if its not fully concurrently safe
// we simply try and return as much as possible // we simply try and return as much as possible
protected final void addShardFailure(ShardSearchFailure failure) { protected final void addShardFailure(final int shardRequestId, ShardSearchFailure failure) {
if (shardFailures == null) { if (shardFailures == null) {
shardFailures = ConcurrentCollections.newQueue(); shardFailures = new AtomicArray<ShardSearchFailure>(scrollId.getContext().length);
} }
shardFailures.add(failure); shardFailures.set(shardRequestId, failure);
} }
public void start() { public void start() {
@ -134,13 +132,15 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent
final AtomicInteger counter = new AtomicInteger(scrollId.getContext().length); final AtomicInteger counter = new AtomicInteger(scrollId.getContext().length);
int localOperations = 0; int localOperations = 0;
for (Tuple<String, Long> target : scrollId.getContext()) { Tuple<String, Long>[] context = scrollId.getContext();
for (int i = 0; i < context.length; i++) {
Tuple<String, Long> target = context[i];
DiscoveryNode node = nodes.get(target.v1()); DiscoveryNode node = nodes.get(target.v1());
if (node != null) { if (node != null) {
if (nodes.localNodeId().equals(node.id())) { if (nodes.localNodeId().equals(node.id())) {
localOperations++; localOperations++;
} else { } else {
executeQueryPhase(counter, node, target.v2()); executeQueryPhase(i, counter, node, target.v2());
} }
} else { } else {
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
@ -158,28 +158,33 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
for (Tuple<String, Long> target : scrollId.getContext()) { Tuple<String, Long>[] context1 = scrollId.getContext();
for (int i = 0; i < context1.length; i++) {
Tuple<String, Long> target = context1[i];
DiscoveryNode node = nodes.get(target.v1()); DiscoveryNode node = nodes.get(target.v1());
if (node != null && nodes.localNodeId().equals(node.id())) { if (node != null && nodes.localNodeId().equals(node.id())) {
executeQueryPhase(counter, node, target.v2()); executeQueryPhase(i, counter, node, target.v2());
} }
} }
} }
}); });
} else { } else {
boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD; boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD;
for (final Tuple<String, Long> target : scrollId.getContext()) { Tuple<String, Long>[] context1 = scrollId.getContext();
for (int i = 0; i < context1.length; i++) {
final Tuple<String, Long> target = context1[i];
final int shardRequestId = i;
final DiscoveryNode node = nodes.get(target.v1()); final DiscoveryNode node = nodes.get(target.v1());
if (node != null && nodes.localNodeId().equals(node.id())) { if (node != null && nodes.localNodeId().equals(node.id())) {
if (localAsync) { if (localAsync) {
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
executeQueryPhase(counter, node, target.v2()); executeQueryPhase(shardRequestId, counter, node, target.v2());
} }
}); });
} else { } else {
executeQueryPhase(counter, node, target.v2()); executeQueryPhase(shardRequestId, counter, node, target.v2());
} }
} }
} }
@ -187,11 +192,11 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent
} }
} }
private void executeQueryPhase(final AtomicInteger counter, DiscoveryNode node, final long searchId) { private void executeQueryPhase(final int shardRequestId, final AtomicInteger counter, DiscoveryNode node, final long searchId) {
searchService.sendExecuteQuery(node, internalScrollSearchRequest(searchId, request), new SearchServiceListener<QuerySearchResult>() { searchService.sendExecuteQuery(node, internalScrollSearchRequest(searchId, request), new SearchServiceListener<QuerySearchResult>() {
@Override @Override
public void onResult(QuerySearchResult result) { public void onResult(QuerySearchResult result) {
queryResults.put(result.shardTarget(), result); queryResults.set(shardRequestId, result);
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
executeFetchPhase(); executeFetchPhase();
} }
@ -202,7 +207,7 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("[{}] Failed to execute query phase", t, searchId); logger.debug("[{}] Failed to execute query phase", t, searchId);
} }
addShardFailure(new ShardSearchFailure(t)); addShardFailure(shardRequestId, new ShardSearchFailure(t));
successfulOps.decrementAndGet(); successfulOps.decrementAndGet();
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
executeFetchPhase(); executeFetchPhase();
@ -212,25 +217,26 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent
} }
private void executeFetchPhase() { private void executeFetchPhase() {
sortedShardList = searchPhaseController.sortDocs(queryResults.values()); sortedShardList = searchPhaseController.sortDocs(queryResults);
Map<SearchShardTarget, ExtTIntArrayList> docIdsToLoad = searchPhaseController.docIdsToLoad(sortedShardList); AtomicArray<ExtTIntArrayList> docIdsToLoad = new AtomicArray<ExtTIntArrayList>(queryResults.length());
searchPhaseController.fillDocIdsToLoad(docIdsToLoad, sortedShardList);
if (docIdsToLoad.isEmpty()) { if (docIdsToLoad.asList().isEmpty()) {
finishHim(); finishHim();
} }
final AtomicInteger counter = new AtomicInteger(docIdsToLoad.size()); final AtomicInteger counter = new AtomicInteger(docIdsToLoad.asList().size());
for (final Map.Entry<SearchShardTarget, ExtTIntArrayList> entry : docIdsToLoad.entrySet()) { for (final AtomicArray.Entry<ExtTIntArrayList> entry : docIdsToLoad.asList()) {
SearchShardTarget shardTarget = entry.getKey(); ExtTIntArrayList docIds = entry.value;
ExtTIntArrayList docIds = entry.getValue(); final QuerySearchResult querySearchResult = queryResults.get(entry.index);
FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, queryResults.get(shardTarget).id(), docIds); FetchSearchRequest fetchSearchRequest = new FetchSearchRequest(request, querySearchResult.id(), docIds);
DiscoveryNode node = nodes.get(shardTarget.nodeId()); DiscoveryNode node = nodes.get(querySearchResult.shardTarget().nodeId());
searchService.sendExecuteFetch(node, fetchSearchRequest, new SearchServiceListener<FetchSearchResult>() { searchService.sendExecuteFetch(node, fetchSearchRequest, new SearchServiceListener<FetchSearchResult>() {
@Override @Override
public void onResult(FetchSearchResult result) { public void onResult(FetchSearchResult result) {
result.shardTarget(entry.getKey()); result.shardTarget(querySearchResult.shardTarget());
fetchResults.put(result.shardTarget(), result); fetchResults.set(entry.index, result);
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
finishHim(); finishHim();
} }
@ -266,8 +272,6 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent
} }
listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.getContext().length, successfulOps.get(), listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.getContext().length, successfulOps.get(),
System.currentTimeMillis() - startTime, buildShardFailures())); System.currentTimeMillis() - startTime, buildShardFailures()));
searchCache.releaseQueryResults(queryResults);
searchCache.releaseFetchResults(fetchResults);
} }
} }
} }

83
src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollScanAction.java
View File

@ -29,8 +29,7 @@ import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.search.action.SearchServiceListener; import org.elasticsearch.search.action.SearchServiceListener;
import org.elasticsearch.search.action.SearchServiceTransportAction; import org.elasticsearch.search.action.SearchServiceTransportAction;
import org.elasticsearch.search.controller.SearchPhaseController; import org.elasticsearch.search.controller.SearchPhaseController;
@ -42,8 +41,7 @@ import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import java.io.IOException; import java.io.IOException;
import java.util.Map; import java.util.List;
import java.util.Queue;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import static org.elasticsearch.action.search.type.TransportSearchHelper.internalScrollSearchRequest; import static org.elasticsearch.action.search.type.TransportSearchHelper.internalScrollSearchRequest;
@ -61,16 +59,12 @@ public class TransportSearchScrollScanAction extends AbstractComponent {
private final SearchPhaseController searchPhaseController; private final SearchPhaseController searchPhaseController;
private final TransportSearchCache searchCache;
@Inject @Inject
public TransportSearchScrollScanAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, public TransportSearchScrollScanAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
TransportSearchCache searchCache,
SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) { SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) {
super(settings); super(settings);
this.threadPool = threadPool; this.threadPool = threadPool;
this.clusterService = clusterService; this.clusterService = clusterService;
this.searchCache = searchCache;
this.searchService = searchService; this.searchService = searchService;
this.searchPhaseController = searchPhaseController; this.searchPhaseController = searchPhaseController;
} }
@ -89,14 +83,11 @@ public class TransportSearchScrollScanAction extends AbstractComponent {
private final DiscoveryNodes nodes; private final DiscoveryNodes nodes;
protected volatile Queue<ShardSearchFailure> shardFailures; private volatile AtomicArray<ShardSearchFailure> shardFailures;
private final AtomicArray<QueryFetchSearchResult> queryFetchResults;
private final Map<SearchShardTarget, QueryFetchSearchResult> queryFetchResults = searchCache.obtainQueryFetchResults();
private final AtomicInteger successfulOps; private final AtomicInteger successfulOps;
private final AtomicInteger counter; private final AtomicInteger counter;
private final long startTime = System.currentTimeMillis(); private final long startTime = System.currentTimeMillis();
private AsyncAction(SearchScrollRequest request, ParsedScrollId scrollId, ActionListener<SearchResponse> listener) { private AsyncAction(SearchScrollRequest request, ParsedScrollId scrollId, ActionListener<SearchResponse> listener) {
@ -106,41 +97,48 @@ public class TransportSearchScrollScanAction extends AbstractComponent {
this.nodes = clusterService.state().nodes(); this.nodes = clusterService.state().nodes();
this.successfulOps = new AtomicInteger(scrollId.getContext().length); this.successfulOps = new AtomicInteger(scrollId.getContext().length);
this.counter = new AtomicInteger(scrollId.getContext().length); this.counter = new AtomicInteger(scrollId.getContext().length);
this.queryFetchResults = new AtomicArray<QueryFetchSearchResult>(scrollId.getContext().length);
} }
protected final ShardSearchFailure[] buildShardFailures() { protected final ShardSearchFailure[] buildShardFailures() {
Queue<ShardSearchFailure> localFailures = shardFailures; if (shardFailures == null) {
if (localFailures == null) {
return ShardSearchFailure.EMPTY_ARRAY; return ShardSearchFailure.EMPTY_ARRAY;
} }
return localFailures.toArray(ShardSearchFailure.EMPTY_ARRAY); List<AtomicArray.Entry<ShardSearchFailure>> entries = shardFailures.asList();
ShardSearchFailure[] failures = new ShardSearchFailure[entries.size()];
for (int i = 0; i < failures.length; i++) {
failures[i] = entries.get(i).value;
}
return failures;
} }
// we do our best to return the shard failures, but its ok if its not fully concurrently safe // we do our best to return the shard failures, but its ok if its not fully concurrently safe
// we simply try and return as much as possible // we simply try and return as much as possible
protected final void addShardFailure(ShardSearchFailure failure) { protected final void addShardFailure(final int shardRequestId, ShardSearchFailure failure) {
if (shardFailures == null) { if (shardFailures == null) {
shardFailures = ConcurrentCollections.newQueue(); shardFailures = new AtomicArray<ShardSearchFailure>(scrollId.getContext().length);
} }
shardFailures.add(failure); shardFailures.set(shardRequestId, failure);
} }
public void start() { public void start() {
if (scrollId.getContext().length == 0) { if (scrollId.getContext().length == 0) {
final InternalSearchResponse internalResponse = new InternalSearchResponse(new InternalSearchHits(InternalSearchHits.EMPTY, Long.parseLong(this.scrollId.getAttributes().get("total_hits")), 0.0f), null, null, false); final InternalSearchResponse internalResponse = new InternalSearchResponse(new InternalSearchHits(InternalSearchHits.EMPTY, Long.parseLong(this.scrollId.getAttributes().get("total_hits")), 0.0f), null, null, false);
searchCache.releaseQueryFetchResults(queryFetchResults);
listener.onResponse(new SearchResponse(internalResponse, request.scrollId(), 0, 0, 0l, buildShardFailures())); listener.onResponse(new SearchResponse(internalResponse, request.scrollId(), 0, 0, 0l, buildShardFailures()));
return; return;
} }
int localOperations = 0; int localOperations = 0;
for (Tuple<String, Long> target : scrollId.getContext()) { Tuple<String, Long>[] context = scrollId.getContext();
for (int i = 0; i < context.length; i++) {
Tuple<String, Long> target = context[i];
DiscoveryNode node = nodes.get(target.v1()); DiscoveryNode node = nodes.get(target.v1());
if (node != null) { if (node != null) {
if (nodes.localNodeId().equals(node.id())) { if (nodes.localNodeId().equals(node.id())) {
localOperations++; localOperations++;
} else { } else {
executePhase(node, target.v2()); executePhase(i, node, target.v2());
} }
} else { } else {
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
@ -158,28 +156,33 @@ public class TransportSearchScrollScanAction extends AbstractComponent {
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
for (Tuple<String, Long> target : scrollId.getContext()) { Tuple<String, Long>[] context1 = scrollId.getContext();
for (int i = 0; i < context1.length; i++) {
Tuple<String, Long> target = context1[i];
DiscoveryNode node = nodes.get(target.v1()); DiscoveryNode node = nodes.get(target.v1());
if (node != null && nodes.localNodeId().equals(node.id())) { if (node != null && nodes.localNodeId().equals(node.id())) {
executePhase(node, target.v2()); executePhase(i, node, target.v2());
} }
} }
} }
}); });
} else { } else {
boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD; boolean localAsync = request.operationThreading() == SearchOperationThreading.THREAD_PER_SHARD;
for (final Tuple<String, Long> target : scrollId.getContext()) { Tuple<String, Long>[] context1 = scrollId.getContext();
for (int i = 0; i < context1.length; i++) {
final Tuple<String, Long> target = context1[i];
final int shardRequestId = i;
final DiscoveryNode node = nodes.get(target.v1()); final DiscoveryNode node = nodes.get(target.v1());
if (node != null && nodes.localNodeId().equals(node.id())) { if (node != null && nodes.localNodeId().equals(node.id())) {
if (localAsync) { if (localAsync) {
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
executePhase(node, target.v2()); executePhase(shardRequestId, node, target.v2());
} }
}); });
} else { } else {
executePhase(node, target.v2()); executePhase(shardRequestId, node, target.v2());
} }
} }
} }
@ -201,11 +204,11 @@ public class TransportSearchScrollScanAction extends AbstractComponent {
} }
} }
private void executePhase(DiscoveryNode node, final long searchId) { private void executePhase(final int shardRequestId, DiscoveryNode node, final long searchId) {
searchService.sendExecuteScan(node, internalScrollSearchRequest(searchId, request), new SearchServiceListener<QueryFetchSearchResult>() { searchService.sendExecuteScan(node, internalScrollSearchRequest(searchId, request), new SearchServiceListener<QueryFetchSearchResult>() {
@Override @Override
public void onResult(QueryFetchSearchResult result) { public void onResult(QueryFetchSearchResult result) {
queryFetchResults.put(result.shardTarget(), result); queryFetchResults.set(shardRequestId, result);
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
finishHim(); finishHim();
} }
@ -216,7 +219,7 @@ public class TransportSearchScrollScanAction extends AbstractComponent {
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("[{}] Failed to execute query phase", t, searchId); logger.debug("[{}] Failed to execute query phase", t, searchId);
} }
addShardFailure(new ShardSearchFailure(t)); addShardFailure(shardRequestId, new ShardSearchFailure(t));
successfulOps.decrementAndGet(); successfulOps.decrementAndGet();
if (counter.decrementAndGet() == 0) { if (counter.decrementAndGet() == 0) {
finishHim(); finishHim();
@ -234,39 +237,37 @@ public class TransportSearchScrollScanAction extends AbstractComponent {
logger.debug("failed to reduce search", failure); logger.debug("failed to reduce search", failure);
} }
listener.onFailure(failure); listener.onFailure(failure);
} finally {
searchCache.releaseQueryFetchResults(queryFetchResults);
} }
} }
private void innerFinishHim() throws IOException { private void innerFinishHim() throws IOException {
int numberOfHits = 0; int numberOfHits = 0;
for (QueryFetchSearchResult shardResult : queryFetchResults.values()) { for (AtomicArray.Entry<QueryFetchSearchResult> entry : queryFetchResults.asList()) {
numberOfHits += shardResult.queryResult().topDocs().scoreDocs.length; numberOfHits += entry.value.queryResult().topDocs().scoreDocs.length;
} }
ShardDoc[] docs = new ShardDoc[numberOfHits]; ShardDoc[] docs = new ShardDoc[numberOfHits];
int counter = 0; int counter = 0;
for (QueryFetchSearchResult shardResult : queryFetchResults.values()) { for (AtomicArray.Entry<QueryFetchSearchResult> entry : queryFetchResults.asList()) {
ScoreDoc[] scoreDocs = shardResult.queryResult().topDocs().scoreDocs; ScoreDoc[] scoreDocs = entry.value.queryResult().topDocs().scoreDocs;
for (ScoreDoc scoreDoc : scoreDocs) { for (ScoreDoc scoreDoc : scoreDocs) {
docs[counter++] = new ShardScoreDoc(shardResult.shardTarget(), scoreDoc.doc, 0.0f); docs[counter++] = new ShardScoreDoc(entry.index, scoreDoc.doc, 0.0f);
} }
} }
final InternalSearchResponse internalResponse = searchPhaseController.merge(docs, queryFetchResults, queryFetchResults); final InternalSearchResponse internalResponse = searchPhaseController.merge(docs, queryFetchResults, queryFetchResults);
((InternalSearchHits) internalResponse.hits()).totalHits = Long.parseLong(this.scrollId.getAttributes().get("total_hits")); ((InternalSearchHits) internalResponse.hits()).totalHits = Long.parseLong(this.scrollId.getAttributes().get("total_hits"));
for (QueryFetchSearchResult shardResult : queryFetchResults.values()) { for (AtomicArray.Entry<QueryFetchSearchResult> entry : queryFetchResults.asList()) {
if (shardResult.queryResult().topDocs().scoreDocs.length < shardResult.queryResult().size()) { if (entry.value.queryResult().topDocs().scoreDocs.length < entry.value.queryResult().size()) {
// we found more than we want for this round, remove this from our scrolling // we found more than we want for this round, remove this from our scrolling
queryFetchResults.remove(shardResult.shardTarget()); queryFetchResults.set(entry.index, null);
} }
} }
String scrollId = null; String scrollId = null;
if (request.scroll() != null) { if (request.scroll() != null) {
// we rebuild the scroll id since we remove shards that we finished scrolling on // we rebuild the scroll id since we remove shards that we finished scrolling on
scrollId = TransportSearchHelper.buildScrollId(this.scrollId.getType(), queryFetchResults.values(), this.scrollId.getAttributes()); // continue moving the total_hits scrollId = TransportSearchHelper.buildScrollId(this.scrollId.getType(), queryFetchResults, this.scrollId.getAttributes()); // continue moving the total_hits
} }
listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.getContext().length, successfulOps.get(), listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.getContext().length, successfulOps.get(),
System.currentTimeMillis() - startTime, buildShardFailures())); System.currentTimeMillis() - startTime, buildShardFailures()));

106
src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java
View File

@ -33,7 +33,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.common.trove.ExtTIntArrayList;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchPhaseResult;
import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.SearchShardTarget;
@ -46,8 +46,8 @@ import org.elasticsearch.search.query.QuerySearchResultProvider;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Queue;
import java.util.Set; import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
@ -64,13 +64,10 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
protected final SearchPhaseController searchPhaseController; protected final SearchPhaseController searchPhaseController;
protected final TransportSearchCache searchCache;
public TransportSearchTypeAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, public TransportSearchTypeAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
TransportSearchCache searchCache, SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) { SearchServiceTransportAction searchService, SearchPhaseController searchPhaseController) {
super(settings, threadPool); super(settings, threadPool);
this.clusterService = clusterService; this.clusterService = clusterService;
this.searchCache = searchCache;
this.searchService = searchService; this.searchService = searchService;
this.searchPhaseController = searchPhaseController; this.searchPhaseController = searchPhaseController;
} }
@ -79,7 +76,7 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
protected final ActionListener<SearchResponse> listener; protected final ActionListener<SearchResponse> listener;
private final GroupShardsIterator shardsIts; protected final GroupShardsIterator shardsIts;
protected final SearchRequest request; protected final SearchRequest request;
@ -87,15 +84,13 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
protected final DiscoveryNodes nodes; protected final DiscoveryNodes nodes;
protected final int expectedSuccessfulOps; protected final int expectedSuccessfulOps;
private final int expectedTotalOps; private final int expectedTotalOps;
protected final AtomicInteger successulOps = new AtomicInteger(); protected final AtomicInteger successulOps = new AtomicInteger();
private final AtomicInteger totalOps = new AtomicInteger(); private final AtomicInteger totalOps = new AtomicInteger();
private volatile Queue<ShardSearchFailure> shardFailures; protected final AtomicArray<FirstResult> firstResults;
private volatile AtomicArray<ShardSearchFailure> shardFailures;
protected volatile ShardDoc[] sortedShardList; protected volatile ShardDoc[] sortedShardList;
protected final long startTime = System.currentTimeMillis(); protected final long startTime = System.currentTimeMillis();
@ -126,24 +121,28 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
// not search shards to search on... // not search shards to search on...
throw new SearchPhaseExecutionException("initial", "No indices / shards to search on, requested indices are " + Arrays.toString(request.indices()), buildShardFailures()); throw new SearchPhaseExecutionException("initial", "No indices / shards to search on, requested indices are " + Arrays.toString(request.indices()), buildShardFailures());
} }
firstResults = new AtomicArray<FirstResult>(shardsIts.size());
} }
public void start() { public void start() {
request.beforeStart(); request.beforeStart();
// count the local operations, and perform the non local ones // count the local operations, and perform the non local ones
int localOperations = 0; int localOperations = 0;
int shardRequestId = -1;
for (final ShardIterator shardIt : shardsIts) { for (final ShardIterator shardIt : shardsIts) {
shardRequestId++;
final ShardRouting shard = shardIt.firstOrNull(); final ShardRouting shard = shardIt.firstOrNull();
if (shard != null) { if (shard != null) {
if (shard.currentNodeId().equals(nodes.localNodeId())) { if (shard.currentNodeId().equals(nodes.localNodeId())) {
localOperations++; localOperations++;
} else { } else {
// do the remote operation here, the localAsync flag is not relevant // do the remote operation here, the localAsync flag is not relevant
performFirstPhase(shardIt); performFirstPhase(shardRequestId, shardIt);
} }
} else { } else {
// really, no shards active in this group // really, no shards active in this group
onFirstPhaseResult(null, shardIt, null); onFirstPhaseResult(shardRequestId, null, shardIt, null);
} }
} }
// we have local operations, perform them now // we have local operations, perform them now
@ -153,11 +152,13 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
int shardRequestId = -1;
for (final ShardIterator shardIt : shardsIts) { for (final ShardIterator shardIt : shardsIts) {
shardRequestId++;
final ShardRouting shard = shardIt.firstOrNull(); final ShardRouting shard = shardIt.firstOrNull();
if (shard != null) { if (shard != null) {
if (shard.currentNodeId().equals(nodes.localNodeId())) { if (shard.currentNodeId().equals(nodes.localNodeId())) {
performFirstPhase(shardIt); performFirstPhase(shardRequestId, shardIt);
} }
} }
} }
@ -168,7 +169,10 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
if (localAsync) { if (localAsync) {
request.beforeLocalFork(); request.beforeLocalFork();
} }
shardRequestId = -1;
for (final ShardIterator shardIt : shardsIts) { for (final ShardIterator shardIt : shardsIts) {
shardRequestId++;
final int fShardRequestId = shardRequestId;
final ShardRouting shard = shardIt.firstOrNull(); final ShardRouting shard = shardIt.firstOrNull();
if (shard != null) { if (shard != null) {
if (shard.currentNodeId().equals(nodes.localNodeId())) { if (shard.currentNodeId().equals(nodes.localNodeId())) {
@ -176,11 +180,11 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() { threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
performFirstPhase(shardIt); performFirstPhase(fShardRequestId, shardIt);
} }
}); });
} else { } else {
performFirstPhase(shardIt); performFirstPhase(fShardRequestId, shardIt);
} }
} }
} }
@ -189,45 +193,45 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
} }
} }
void performFirstPhase(final ShardIterator shardIt) { void performFirstPhase(final int shardRequestId, final ShardIterator shardIt) {
performFirstPhase(shardIt, shardIt.nextOrNull()); performFirstPhase(shardRequestId, shardIt, shardIt.nextOrNull());
} }
void performFirstPhase(final ShardIterator shardIt, final ShardRouting shard) { void performFirstPhase(final int shardRequestId, final ShardIterator shardIt, final ShardRouting shard) {
if (shard == null) { if (shard == null) {
// no more active shards... (we should not really get here, but just for safety) // no more active shards... (we should not really get here, but just for safety)
onFirstPhaseResult(null, shardIt, null); onFirstPhaseResult(shardRequestId, null, shardIt, null);
} else { } else {
DiscoveryNode node = nodes.get(shard.currentNodeId()); DiscoveryNode node = nodes.get(shard.currentNodeId());
if (node == null) { if (node == null) {
onFirstPhaseResult(shard, shardIt, null); onFirstPhaseResult(shardRequestId, shard, shardIt, null);
} else { } else {
String[] filteringAliases = clusterState.metaData().filteringAliases(shard.index(), request.indices()); String[] filteringAliases = clusterState.metaData().filteringAliases(shard.index(), request.indices());
sendExecuteFirstPhase(node, internalSearchRequest(shard, shardsIts.size(), request, filteringAliases, startTime), new SearchServiceListener<FirstResult>() { sendExecuteFirstPhase(node, internalSearchRequest(shard, shardsIts.size(), request, filteringAliases, startTime), new SearchServiceListener<FirstResult>() {
@Override @Override
public void onResult(FirstResult result) { public void onResult(FirstResult result) {
onFirstPhaseResult(shard, result, shardIt); onFirstPhaseResult(shardRequestId, shard, result, shardIt);
} }
@Override @Override
public void onFailure(Throwable t) { public void onFailure(Throwable t) {
onFirstPhaseResult(shard, shardIt, t); onFirstPhaseResult(shardRequestId, shard, shardIt, t);
} }
}); });
} }
} }
} }
void onFirstPhaseResult(ShardRouting shard, FirstResult result, ShardIterator shardIt) { void onFirstPhaseResult(int shardRequestId, ShardRouting shard, FirstResult result, ShardIterator shardIt) {
result.shardTarget(new SearchShardTarget(shard.currentNodeId(), shard.index(), shard.id())); result.shardTarget(new SearchShardTarget(shard.currentNodeId(), shard.index(), shard.id()));
processFirstPhaseResult(shard, result); processFirstPhaseResult(shardRequestId, shard, result);
// increment all the "future" shards to update the total ops since we some may work and some may not... // increment all the "future" shards to update the total ops since we some may work and some may not...
// and when that happens, we break on total ops, so we must maintain them // and when that happens, we break on total ops, so we must maintain them
int xTotalOps = totalOps.addAndGet(shardIt.remaining() + 1); int xTotalOps = totalOps.addAndGet(shardIt.remaining() + 1);
successulOps.incrementAndGet(); successulOps.incrementAndGet();
if (xTotalOps == expectedTotalOps) { if (xTotalOps == expectedTotalOps) {
try { try {
moveToSecondPhase(); innerMoveToSecondPhase();
} catch (Throwable e) { } catch (Throwable e) {
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug(shardIt.shardId() + ": Failed to execute [" + request + "] while moving to second phase", e); logger.debug(shardIt.shardId() + ": Failed to execute [" + request + "] while moving to second phase", e);
@ -237,7 +241,7 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
} }
} }
void onFirstPhaseResult(@Nullable ShardRouting shard, final ShardIterator shardIt, Throwable t) { void onFirstPhaseResult(final int shardRequestId, @Nullable ShardRouting shard, final ShardIterator shardIt, Throwable t) {
if (totalOps.incrementAndGet() == expectedTotalOps) { if (totalOps.incrementAndGet() == expectedTotalOps) {
// e is null when there is no next active.... // e is null when there is no next active....
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
@ -252,16 +256,16 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
// no more shards, add a failure // no more shards, add a failure
if (t == null) { if (t == null) {
// no active shards // no active shards
addShardFailure(new ShardSearchFailure("No active shards", new SearchShardTarget(null, shardIt.shardId().index().name(), shardIt.shardId().id()), RestStatus.SERVICE_UNAVAILABLE)); addShardFailure(shardRequestId, new ShardSearchFailure("No active shards", new SearchShardTarget(null, shardIt.shardId().index().name(), shardIt.shardId().id()), RestStatus.SERVICE_UNAVAILABLE));
} else { } else {
addShardFailure(new ShardSearchFailure(t)); addShardFailure(shardRequestId, new ShardSearchFailure(t));
} }
if (successulOps.get() == 0) { if (successulOps.get() == 0) {
// no successful ops, raise an exception // no successful ops, raise an exception
listener.onFailure(new SearchPhaseExecutionException(firstPhaseName(), "total failure", buildShardFailures())); listener.onFailure(new SearchPhaseExecutionException(firstPhaseName(), "total failure", buildShardFailures()));
} else { } else {
try { try {
moveToSecondPhase(); innerMoveToSecondPhase();
} catch (Throwable e) { } catch (Throwable e) {
listener.onFailure(new ReduceSearchPhaseException(firstPhaseName(), "", e, buildShardFailures())); listener.onFailure(new ReduceSearchPhaseException(firstPhaseName(), "", e, buildShardFailures()));
} }
@ -279,7 +283,7 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
} }
} }
} }
performFirstPhase(shardIt, nextShard); performFirstPhase(shardRequestId, shardIt, nextShard);
} else { } else {
// no more shards active, add a failure // no more shards active, add a failure
// e is null when there is no next active.... // e is null when there is no next active....
@ -294,9 +298,9 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
} }
if (t == null) { if (t == null) {
// no active shards // no active shards
addShardFailure(new ShardSearchFailure("No active shards", new SearchShardTarget(null, shardIt.shardId().index().name(), shardIt.shardId().id()), RestStatus.SERVICE_UNAVAILABLE)); addShardFailure(shardRequestId, new ShardSearchFailure("No active shards", new SearchShardTarget(null, shardIt.shardId().index().name(), shardIt.shardId().id()), RestStatus.SERVICE_UNAVAILABLE));
} else { } else {
addShardFailure(new ShardSearchFailure(t)); addShardFailure(shardRequestId, new ShardSearchFailure(t));
} }
} }
} }
@ -310,37 +314,41 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
} }
protected final ShardSearchFailure[] buildShardFailures() { protected final ShardSearchFailure[] buildShardFailures() {
Queue<ShardSearchFailure> localFailures = shardFailures; if (shardFailures == null) {
if (localFailures == null) {
return ShardSearchFailure.EMPTY_ARRAY; return ShardSearchFailure.EMPTY_ARRAY;
} }
return localFailures.toArray(ShardSearchFailure.EMPTY_ARRAY); List<AtomicArray.Entry<ShardSearchFailure>> entries = shardFailures.asList();
ShardSearchFailure[] failures = new ShardSearchFailure[entries.size()];
for (int i = 0; i < failures.length; i++) {
failures[i] = entries.get(i).value;
}
return failures;
} }
// we do our best to return the shard failures, but its ok if its not fully concurrently safe // we do our best to return the shard failures, but its ok if its not fully concurrently safe
// we simply try and return as much as possible // we simply try and return as much as possible
protected final void addShardFailure(ShardSearchFailure failure) { protected final void addShardFailure(final int shardRequestId, ShardSearchFailure failure) {
if (shardFailures == null) { if (shardFailures == null) {
shardFailures = ConcurrentCollections.newQueue(); shardFailures = new AtomicArray<ShardSearchFailure>(shardsIts.size());
} }
shardFailures.add(failure); shardFailures.set(shardRequestId, failure);
} }
/** /**
* Releases shard targets that are not used in the docsIdsToLoad. * Releases shard targets that are not used in the docsIdsToLoad.
*/ */
protected void releaseIrrelevantSearchContexts(Map<SearchShardTarget, QuerySearchResultProvider> queryResults, protected void releaseIrrelevantSearchContexts(AtomicArray<? extends QuerySearchResultProvider> queryResults,
Map<SearchShardTarget, ExtTIntArrayList> docIdsToLoad) { AtomicArray<ExtTIntArrayList> docIdsToLoad) {
if (docIdsToLoad == null) { if (docIdsToLoad == null) {
return; return;
} }
// we only release search context that we did not fetch from if we are not scrolling // we only release search context that we did not fetch from if we are not scrolling
if (request.scroll() == null) { if (request.scroll() == null) {
for (Map.Entry<SearchShardTarget, QuerySearchResultProvider> entry : queryResults.entrySet()) { for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults.asList()) {
if (!docIdsToLoad.containsKey(entry.getKey())) { if (docIdsToLoad.get(entry.index) == null) {
DiscoveryNode node = nodes.get(entry.getKey().nodeId()); DiscoveryNode node = nodes.get(entry.value.queryResult().shardTarget().nodeId());
if (node != null) { // should not happen (==null) but safeguard anyhow if (node != null) { // should not happen (==null) but safeguard anyhow
searchService.sendFreeContext(node, entry.getValue().id(), request); searchService.sendFreeContext(node, entry.value.queryResult().id(), request);
} }
} }
} }
@ -349,7 +357,13 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
protected abstract void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchRequest request, SearchServiceListener<FirstResult> listener); protected abstract void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchRequest request, SearchServiceListener<FirstResult> listener);
protected abstract void processFirstPhaseResult(ShardRouting shard, FirstResult result); protected final void processFirstPhaseResult(int shardRequestId, ShardRouting shard, FirstResult result) {
firstResults.set(shardRequestId, result);
}
final void innerMoveToSecondPhase() throws Exception {
moveToSecondPhase();
}
protected abstract void moveToSecondPhase() throws Exception; protected abstract void moveToSecondPhase() throws Exception;

2
src/main/java/org/elasticsearch/common/lucene/search/ShardFieldDocSortedHitQueue.java
View File

@ -126,7 +126,7 @@ public class ShardFieldDocSortedHitQueue extends PriorityQueue<ShardFieldDoc> {
// avoid random sort order that could lead to duplicates (bug #31241): // avoid random sort order that could lead to duplicates (bug #31241):
if (c == 0) { if (c == 0) {
// CHANGE: Add shard base tie breaking // CHANGE: Add shard base tie breaking
c = docA.shardTarget().compareTo(docB.shardTarget()); c = docA.shardRequestId() - docB.shardRequestId();
if (c == 0) { if (c == 0) {
return docA.doc > docB.doc; return docA.doc > docB.doc;
} }

119
src/main/java/org/elasticsearch/common/util/concurrent/AtomicArray.java Normal file
View File

@ -0,0 +1,119 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.util.concurrent;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
* A list backed by an {@link AtomicReferenceArray} with potential null values, easily allowing
* to get the concrete values as a list using {@link #asList()}.
*/
public class AtomicArray<E> {
private static final AtomicArray EMPTY = new AtomicArray(0);
@SuppressWarnings("unchecked")
public static <E> E empty() {
return (E) EMPTY;
}
private final AtomicReferenceArray<E> array;
private volatile List<Entry<E>> nonNullList;
public AtomicArray(int size) {
array = new AtomicReferenceArray<E>(size);
}
/**
* The size of the expected results, including potential null values.
*/
public int length() {
return array.length();
}
/**
* Sets the element at position {@code i} to the given value.
*
* @param i the index
* @param value the new value
*/
public void set(int i, E value) {
array.set(i, value);
if (nonNullList != null) { // read first, lighter, and most times it will be null...
nonNullList = null;
}
}
/**
* Gets the current value at position {@code i}.
*
* @param i the index
* @return the current value
*/
public E get(int i) {
return array.get(i);
}
/**
* Returns the it as a non null list, with an Entry wrapping each value allowing to
* retain its index.
*/
public List<Entry<E>> asList() {
if (nonNullList == null) {
if (array == null || array.length() == 0) {
nonNullList = ImmutableList.of();
} else {
List<Entry<E>> list = new ArrayList<Entry<E>>(array.length());
for (int i = 0; i < array.length(); i++) {
E e = array.get(i);
if (e != null) {
list.add(new Entry<E>(i, e));
}
}
nonNullList = list;
}
}
return nonNullList;
}
/**
* An entry within the array.
*/
public static class Entry<E> {
/**
* The original index of the value within the array.
*/
public final int index;
/**
* The value.
*/
public final E value;
public Entry(int index, E value) {
this.index = index;
this.value = value;
}
}
}

4
src/main/java/org/elasticsearch/search/controller/ScoreDocQueue.java
View File

@ -23,8 +23,6 @@ import org.apache.lucene.util.PriorityQueue;
/** /**
* <p>Same as lucene {@link org.apache.lucene.search.HitQueue}. * <p>Same as lucene {@link org.apache.lucene.search.HitQueue}.
*
*
*/ */
public class ScoreDocQueue extends PriorityQueue<ShardScoreDoc> { public class ScoreDocQueue extends PriorityQueue<ShardScoreDoc> {
@ -34,7 +32,7 @@ public class ScoreDocQueue extends PriorityQueue<ShardScoreDoc> {
protected final boolean lessThan(ShardScoreDoc hitA, ShardScoreDoc hitB) { protected final boolean lessThan(ShardScoreDoc hitA, ShardScoreDoc hitB) {
if (hitA.score == hitB.score) { if (hitA.score == hitB.score) {
int c = hitA.shardTarget().compareTo(hitB.shardTarget()); int c = hitA.shardRequestId() - hitB.shardRequestId();
if (c == 0) { if (c == 0) {
return hitA.doc > hitB.doc; return hitA.doc > hitB.doc;
} }

132
src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.search.controller; package org.elasticsearch.search.controller;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.Ordering; import com.google.common.collect.Ordering;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
@ -33,7 +32,7 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.ShardFieldDocSortedHitQueue; import org.elasticsearch.common.lucene.search.ShardFieldDocSortedHitQueue;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.common.trove.ExtTIntArrayList;
import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.search.dfs.AggregatedDfs; import org.elasticsearch.search.dfs.AggregatedDfs;
import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.dfs.DfsSearchResult;
import org.elasticsearch.search.facet.Facet; import org.elasticsearch.search.facet.Facet;
@ -49,7 +48,6 @@ import org.elasticsearch.search.query.QuerySearchResultProvider;
import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.Suggest;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -59,12 +57,12 @@ import java.util.Map;
*/ */
public class SearchPhaseController extends AbstractComponent { public class SearchPhaseController extends AbstractComponent {
public static Ordering<QuerySearchResultProvider> QUERY_RESULT_ORDERING = new Ordering<QuerySearchResultProvider>() { public static Ordering<AtomicArray.Entry<? extends QuerySearchResultProvider>> QUERY_RESULT_ORDERING = new Ordering<AtomicArray.Entry<? extends QuerySearchResultProvider>>() {
@Override @Override
public int compare(@Nullable QuerySearchResultProvider o1, @Nullable QuerySearchResultProvider o2) { public int compare(@Nullable AtomicArray.Entry<? extends QuerySearchResultProvider> o1, @Nullable AtomicArray.Entry<? extends QuerySearchResultProvider> o2) {
int i = o1.shardTarget().index().compareTo(o2.shardTarget().index()); int i = o1.value.shardTarget().index().compareTo(o2.value.shardTarget().index());
if (i == 0) { if (i == 0) {
i = o1.shardTarget().shardId() - o2.shardTarget().shardId(); i = o1.value.shardTarget().shardId() - o2.value.shardTarget().shardId();
} }
return i; return i;
} }
@ -86,13 +84,13 @@ public class SearchPhaseController extends AbstractComponent {
return optimizeSingleShard; return optimizeSingleShard;
} }
public AggregatedDfs aggregateDfs(Iterable<DfsSearchResult> results) { public AggregatedDfs aggregateDfs(AtomicArray<DfsSearchResult> results) {
Map<Term, TermStatistics> termStatistics = XMaps.newNoNullKeysMap(); Map<Term, TermStatistics> termStatistics = XMaps.newNoNullKeysMap();
Map<String, CollectionStatistics> fieldStatistics = XMaps.newNoNullKeysMap(); Map<String, CollectionStatistics> fieldStatistics = XMaps.newNoNullKeysMap();
long aggMaxDoc = 0; long aggMaxDoc = 0;
for (DfsSearchResult result : results) { for (AtomicArray.Entry<DfsSearchResult> lEntry : results.asList()) {
final Term[] terms = result.terms(); final Term[] terms = lEntry.value.terms();
final TermStatistics[] stats = result.termStatistics(); final TermStatistics[] stats = lEntry.value.termStatistics();
assert terms.length == stats.length; assert terms.length == stats.length;
for (int i = 0; i < terms.length; i++) { for (int i = 0; i < terms.length; i++) {
assert terms[i] != null; assert terms[i] != null;
@ -102,14 +100,14 @@ public class SearchPhaseController extends AbstractComponent {
// totalTermFrequency is an optional statistic we need to check if either one or both // totalTermFrequency is an optional statistic we need to check if either one or both
// are set to -1 which means not present and then set it globally to -1 // are set to -1 which means not present and then set it globally to -1
termStatistics.put(terms[i], new TermStatistics(existing.term(), termStatistics.put(terms[i], new TermStatistics(existing.term(),
existing.docFreq() + stats[i].docFreq(), existing.docFreq() + stats[i].docFreq(),
optionalSum(existing.totalTermFreq(), stats[i].totalTermFreq()))); optionalSum(existing.totalTermFreq(), stats[i].totalTermFreq())));
} else { } else {
termStatistics.put(terms[i], stats[i]); termStatistics.put(terms[i], stats[i]);
} }
} }
for (Map.Entry<String, CollectionStatistics> entry : result.fieldStatistics().entrySet()) { for (Map.Entry<String, CollectionStatistics> entry : lEntry.value.fieldStatistics().entrySet()) {
assert entry.getKey() != null; assert entry.getKey() != null;
CollectionStatistics existing = fieldStatistics.get(entry.getKey()); CollectionStatistics existing = fieldStatistics.get(entry.getKey());
if (existing != null) { if (existing != null) {
@ -124,36 +122,39 @@ public class SearchPhaseController extends AbstractComponent {
fieldStatistics.put(entry.getKey(), entry.getValue()); fieldStatistics.put(entry.getKey(), entry.getValue());
} }
} }
aggMaxDoc += result.maxDoc(); aggMaxDoc += lEntry.value.maxDoc();
} }
return new AggregatedDfs(termStatistics, fieldStatistics, aggMaxDoc); return new AggregatedDfs(termStatistics, fieldStatistics, aggMaxDoc);
} }
private static long optionalSum(long left, long right) { private static long optionalSum(long left, long right) {
return Math.min(left, right) == -1 ? -1 : left + right; return Math.min(left, right) == -1 ? -1 : left + right;
} }
public ShardDoc[] sortDocs(Collection<? extends QuerySearchResultProvider> results1) { public ShardDoc[] sortDocs(AtomicArray<? extends QuerySearchResultProvider> results1) {
if (results1.isEmpty()) { if (results1.asList().isEmpty()) {
return EMPTY; return EMPTY;
} }
if (optimizeSingleShard) { if (optimizeSingleShard) {
boolean canOptimize = false; boolean canOptimize = false;
QuerySearchResult result = null; QuerySearchResult result = null;
if (results1.size() == 1) { int shardRequestId = -1;
if (results1.asList().size() == 1) {
canOptimize = true; canOptimize = true;
result = results1.iterator().next().queryResult(); result = results1.asList().get(0).value.queryResult();
shardRequestId = results1.asList().get(0).index;
} else { } else {
// lets see if we only got hits from a single shard, if so, we can optimize... // lets see if we only got hits from a single shard, if so, we can optimize...
for (QuerySearchResultProvider queryResult : results1) { for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : results1.asList()) {
if (queryResult.queryResult().topDocs().scoreDocs.length > 0) { if (entry.value.queryResult().topDocs().scoreDocs.length > 0) {
if (result != null) { // we already have one, can't really optimize if (result != null) { // we already have one, can't really optimize
canOptimize = false; canOptimize = false;
break; break;
} }
canOptimize = true; canOptimize = true;
result = queryResult.queryResult(); result = entry.value.queryResult();
shardRequestId = entry.index;
} }
} }
} }
@ -170,23 +171,22 @@ public class SearchPhaseController extends AbstractComponent {
ShardDoc[] docs = new ShardDoc[resultDocsSize]; ShardDoc[] docs = new ShardDoc[resultDocsSize];
for (int i = 0; i < resultDocsSize; i++) { for (int i = 0; i < resultDocsSize; i++) {
ScoreDoc scoreDoc = scoreDocs[result.from() + i]; ScoreDoc scoreDoc = scoreDocs[result.from() + i];
docs[i] = new ShardFieldDoc(result.shardTarget(), scoreDoc.doc, scoreDoc.score, ((FieldDoc) scoreDoc).fields); docs[i] = new ShardFieldDoc(shardRequestId, scoreDoc.doc, scoreDoc.score, ((FieldDoc) scoreDoc).fields);
} }
return docs; return docs;
} else { } else {
ShardDoc[] docs = new ShardDoc[resultDocsSize]; ShardDoc[] docs = new ShardDoc[resultDocsSize];
for (int i = 0; i < resultDocsSize; i++) { for (int i = 0; i < resultDocsSize; i++) {
ScoreDoc scoreDoc = scoreDocs[result.from() + i]; ScoreDoc scoreDoc = scoreDocs[result.from() + i];
docs[i] = new ShardScoreDoc(result.shardTarget(), scoreDoc.doc, scoreDoc.score); docs[i] = new ShardScoreDoc(shardRequestId, scoreDoc.doc, scoreDoc.score);
} }
return docs; return docs;
} }
} }
} }
List<? extends QuerySearchResultProvider> results = QUERY_RESULT_ORDERING.sortedCopy(results1); List<? extends AtomicArray.Entry<? extends QuerySearchResultProvider>> results = QUERY_RESULT_ORDERING.sortedCopy(results1.asList());
QuerySearchResultProvider queryResultProvider = results.get(0).value;
QuerySearchResultProvider queryResultProvider = results.get(0);
int totalNumDocs = 0; int totalNumDocs = 0;
@ -203,8 +203,8 @@ public class SearchPhaseController extends AbstractComponent {
for (int i = 0; i < fieldDocs.fields.length; i++) { for (int i = 0; i < fieldDocs.fields.length; i++) {
boolean allValuesAreNull = true; boolean allValuesAreNull = true;
boolean resolvedField = false; boolean resolvedField = false;
for (QuerySearchResultProvider resultProvider : results) { for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : results) {
for (ScoreDoc doc : resultProvider.queryResult().topDocs().scoreDocs) { for (ScoreDoc doc : entry.value.queryResult().topDocs().scoreDocs) {
FieldDoc fDoc = (FieldDoc) doc; FieldDoc fDoc = (FieldDoc) doc;
if (fDoc.fields[i] != null) { if (fDoc.fields[i] != null) {
allValuesAreNull = false; allValuesAreNull = false;
@ -227,12 +227,12 @@ public class SearchPhaseController extends AbstractComponent {
queue = new ShardFieldDocSortedHitQueue(fieldDocs.fields, queueSize); queue = new ShardFieldDocSortedHitQueue(fieldDocs.fields, queueSize);
// we need to accumulate for all and then filter the from // we need to accumulate for all and then filter the from
for (QuerySearchResultProvider resultProvider : results) { for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : results) {
QuerySearchResult result = resultProvider.queryResult(); QuerySearchResult result = entry.value.queryResult();
ScoreDoc[] scoreDocs = result.topDocs().scoreDocs; ScoreDoc[] scoreDocs = result.topDocs().scoreDocs;
totalNumDocs += scoreDocs.length; totalNumDocs += scoreDocs.length;
for (ScoreDoc doc : scoreDocs) { for (ScoreDoc doc : scoreDocs) {
ShardFieldDoc nodeFieldDoc = new ShardFieldDoc(result.shardTarget(), doc.doc, doc.score, ((FieldDoc) doc).fields); ShardFieldDoc nodeFieldDoc = new ShardFieldDoc(entry.index, doc.doc, doc.score, ((FieldDoc) doc).fields);
if (queue.insertWithOverflow(nodeFieldDoc) == nodeFieldDoc) { if (queue.insertWithOverflow(nodeFieldDoc) == nodeFieldDoc) {
// filled the queue, break // filled the queue, break
break; break;
@ -241,12 +241,12 @@ public class SearchPhaseController extends AbstractComponent {
} }
} else { } else {
queue = new ScoreDocQueue(queueSize); // we need to accumulate for all and then filter the from queue = new ScoreDocQueue(queueSize); // we need to accumulate for all and then filter the from
for (QuerySearchResultProvider resultProvider : results) { for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : results) {
QuerySearchResult result = resultProvider.queryResult(); QuerySearchResult result = entry.value.queryResult();
ScoreDoc[] scoreDocs = result.topDocs().scoreDocs; ScoreDoc[] scoreDocs = result.topDocs().scoreDocs;
totalNumDocs += scoreDocs.length; totalNumDocs += scoreDocs.length;
for (ScoreDoc doc : scoreDocs) { for (ScoreDoc doc : scoreDocs) {
ShardScoreDoc nodeScoreDoc = new ShardScoreDoc(result.shardTarget(), doc.doc, doc.score); ShardScoreDoc nodeScoreDoc = new ShardScoreDoc(entry.index, doc.doc, doc.score);
if (queue.insertWithOverflow(nodeScoreDoc) == nodeScoreDoc) { if (queue.insertWithOverflow(nodeScoreDoc) == nodeScoreDoc) {
// filled the queue, break // filled the queue, break
break; break;
@ -277,31 +277,31 @@ public class SearchPhaseController extends AbstractComponent {
return shardDocs; return shardDocs;
} }
public Map<SearchShardTarget, ExtTIntArrayList> docIdsToLoad(ShardDoc[] shardDocs) { /**
Map<SearchShardTarget, ExtTIntArrayList> result = XMaps.newMap(); * Builds an array, with potential null elements, with docs to load.
*/
public void fillDocIdsToLoad(AtomicArray<ExtTIntArrayList> docsIdsToLoad, ShardDoc[] shardDocs) {
for (ShardDoc shardDoc : shardDocs) { for (ShardDoc shardDoc : shardDocs) {
ExtTIntArrayList list = result.get(shardDoc.shardTarget()); ExtTIntArrayList list = docsIdsToLoad.get(shardDoc.shardRequestId());
if (list == null) { if (list == null) {
list = new ExtTIntArrayList(); // can't be shared!, uses unsafe on it later on list = new ExtTIntArrayList(); // can't be shared!, uses unsafe on it later on
result.put(shardDoc.shardTarget(), list); docsIdsToLoad.set(shardDoc.shardRequestId(), list);
} }
list.add(shardDoc.docId()); list.add(shardDoc.docId());
} }
return result;
} }
public InternalSearchResponse merge(ShardDoc[] sortedDocs, Map<SearchShardTarget, ? extends QuerySearchResultProvider> queryResults, Map<SearchShardTarget, ? extends FetchSearchResultProvider> fetchResults) { public InternalSearchResponse merge(ShardDoc[] sortedDocs, AtomicArray<? extends QuerySearchResultProvider> queryResults, AtomicArray<? extends FetchSearchResultProvider> fetchResults) {
boolean sorted = false; boolean sorted = false;
int sortScoreIndex = -1; int sortScoreIndex = -1;
QuerySearchResult querySearchResult;
try { if (queryResults.asList().isEmpty()) {
querySearchResult = Iterables.get(queryResults.values(), 0).queryResult();
} catch (IndexOutOfBoundsException e) {
// no results, return an empty response
return InternalSearchResponse.EMPTY; return InternalSearchResponse.EMPTY;
} }
QuerySearchResult querySearchResult = queryResults.asList().get(0).value.queryResult();
if (querySearchResult.topDocs() instanceof TopFieldDocs) { if (querySearchResult.topDocs() instanceof TopFieldDocs) {
sorted = true; sorted = true;
TopFieldDocs fieldDocs = (TopFieldDocs) querySearchResult.queryResult().topDocs(); TopFieldDocs fieldDocs = (TopFieldDocs) querySearchResult.queryResult().topDocs();
@ -314,7 +314,7 @@ public class SearchPhaseController extends AbstractComponent {
// merge facets // merge facets
InternalFacets facets = null; InternalFacets facets = null;
if (!queryResults.isEmpty()) { if (!queryResults.asList().isEmpty()) {
// we rely on the fact that the order of facets is the same on all query results // we rely on the fact that the order of facets is the same on all query results
if (querySearchResult.facets() != null && querySearchResult.facets().facets() != null && !querySearchResult.facets().facets().isEmpty()) { if (querySearchResult.facets() != null && querySearchResult.facets().facets() != null && !querySearchResult.facets().facets().isEmpty()) {
List<Facet> aggregatedFacets = Lists.newArrayList(); List<Facet> aggregatedFacets = Lists.newArrayList();
@ -322,8 +322,8 @@ public class SearchPhaseController extends AbstractComponent {
for (Facet facet : querySearchResult.facets()) { for (Facet facet : querySearchResult.facets()) {
// aggregate each facet name into a single list, and aggregate it // aggregate each facet name into a single list, and aggregate it
namedFacets.clear(); namedFacets.clear();
for (QuerySearchResultProvider queryResultProvider : queryResults.values()) { for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults.asList()) {
for (Facet facet1 : queryResultProvider.queryResult().facets()) { for (Facet facet1 : entry.value.queryResult().facets()) {
if (facet.getName().equals(facet1.getName())) { if (facet.getName().equals(facet1.getName())) {
namedFacets.add(facet1); namedFacets.add(facet1);
} }
@ -342,13 +342,14 @@ public class SearchPhaseController extends AbstractComponent {
long totalHits = 0; long totalHits = 0;
float maxScore = Float.NEGATIVE_INFINITY; float maxScore = Float.NEGATIVE_INFINITY;
boolean timedOut = false; boolean timedOut = false;
for (QuerySearchResultProvider queryResultProvider : queryResults.values()) { for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults.asList()) {
if (queryResultProvider.queryResult().searchTimedOut()) { QuerySearchResult result = entry.value.queryResult();
if (result.searchTimedOut()) {
timedOut = true; timedOut = true;
} }
totalHits += queryResultProvider.queryResult().topDocs().totalHits; totalHits += result.topDocs().totalHits;
if (!Float.isNaN(queryResultProvider.queryResult().topDocs().getMaxScore())) { if (!Float.isNaN(result.topDocs().getMaxScore())) {
maxScore = Math.max(maxScore, queryResultProvider.queryResult().topDocs().getMaxScore()); maxScore = Math.max(maxScore, result.topDocs().getMaxScore());
} }
} }
if (Float.isInfinite(maxScore)) { if (Float.isInfinite(maxScore)) {
@ -356,15 +357,15 @@ public class SearchPhaseController extends AbstractComponent {
} }
// clean the fetch counter // clean the fetch counter
for (FetchSearchResultProvider fetchSearchResultProvider : fetchResults.values()) { for (AtomicArray.Entry<? extends FetchSearchResultProvider> entry : fetchResults.asList()) {
fetchSearchResultProvider.fetchResult().initCounter(); entry.value.fetchResult().initCounter();
} }
// merge hits // merge hits
List<InternalSearchHit> hits = new ArrayList<InternalSearchHit>(); List<InternalSearchHit> hits = new ArrayList<InternalSearchHit>();
if (!fetchResults.isEmpty()) { if (!fetchResults.asList().isEmpty()) {
for (ShardDoc shardDoc : sortedDocs) { for (ShardDoc shardDoc : sortedDocs) {
FetchSearchResultProvider fetchResultProvider = fetchResults.get(shardDoc.shardTarget()); FetchSearchResultProvider fetchResultProvider = fetchResults.get(shardDoc.shardRequestId());
if (fetchResultProvider == null) { if (fetchResultProvider == null) {
continue; continue;
} }
@ -390,13 +391,12 @@ public class SearchPhaseController extends AbstractComponent {
// merge suggest results // merge suggest results
Suggest suggest = null; Suggest suggest = null;
if (!queryResults.isEmpty()) { if (!queryResults.asList().isEmpty()) {
final Map<String, List<Suggest.Suggestion>> groupedSuggestions = new HashMap<String, List<Suggest.Suggestion>>(); final Map<String, List<Suggest.Suggestion>> groupedSuggestions = new HashMap<String, List<Suggest.Suggestion>>();
boolean hasSuggestions = false; boolean hasSuggestions = false;
for (QuerySearchResultProvider resultProvider : queryResults.values()) { for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults.asList()) {
Suggest shardResult = resultProvider.queryResult().suggest(); Suggest shardResult = entry.value.queryResult().queryResult().suggest();
if (shardResult == null) { if (shardResult == null) {
continue; continue;
} }
@ -410,5 +410,5 @@ public class SearchPhaseController extends AbstractComponent {
InternalSearchHits searchHits = new InternalSearchHits(hits.toArray(new InternalSearchHit[hits.size()]), totalHits, maxScore); InternalSearchHits searchHits = new InternalSearchHits(hits.toArray(new InternalSearchHit[hits.size()]), totalHits, maxScore);
return new InternalSearchResponse(searchHits, facets, suggest, timedOut); return new InternalSearchResponse(searchHits, facets, suggest, timedOut);
} }
} }

2
src/main/java/org/elasticsearch/search/controller/ShardDoc.java
View File

@ -26,7 +26,7 @@ import org.elasticsearch.search.SearchShardTarget;
*/ */
public interface ShardDoc { public interface ShardDoc {
SearchShardTarget shardTarget(); int shardRequestId();
int docId(); int docId();

14
src/main/java/org/elasticsearch/search/controller/ShardFieldDoc.java
View File

@ -27,21 +27,21 @@ import org.elasticsearch.search.SearchShardTarget;
*/ */
public class ShardFieldDoc extends FieldDoc implements ShardDoc { public class ShardFieldDoc extends FieldDoc implements ShardDoc {
private final SearchShardTarget shardTarget; private final int shardRequestId;
public ShardFieldDoc(SearchShardTarget shardTarget, int doc, float score) { public ShardFieldDoc(int shardRequestId, int doc, float score) {
super(doc, score); super(doc, score);
this.shardTarget = shardTarget; this.shardRequestId = shardRequestId;
} }
public ShardFieldDoc(SearchShardTarget shardTarget, int doc, float score, Object[] fields) { public ShardFieldDoc(int shardRequestId, int doc, float score, Object[] fields) {
super(doc, score, fields); super(doc, score, fields);
this.shardTarget = shardTarget; this.shardRequestId = shardRequestId;
} }
@Override @Override
public SearchShardTarget shardTarget() { public int shardRequestId() {
return this.shardTarget; return this.shardRequestId;
} }
@Override @Override

11
src/main/java/org/elasticsearch/search/controller/ShardScoreDoc.java
View File

@ -20,23 +20,22 @@
package org.elasticsearch.search.controller; package org.elasticsearch.search.controller;
import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreDoc;
import org.elasticsearch.search.SearchShardTarget;
/** /**
* *
*/ */
public class ShardScoreDoc extends ScoreDoc implements ShardDoc { public class ShardScoreDoc extends ScoreDoc implements ShardDoc {
private final SearchShardTarget shardTarget; private final int shardRequestId;
public ShardScoreDoc(SearchShardTarget shardTarget, int doc, float score) { public ShardScoreDoc(int shardRequestId, int doc, float score) {
super(doc, score); super(doc, score);
this.shardTarget = shardTarget; this.shardRequestId = shardRequestId;
} }
@Override @Override
public SearchShardTarget shardTarget() { public int shardRequestId() {
return this.shardTarget; return this.shardRequestId;
} }
@Override @Override