Stop async search maintenance service on restart (#56982)

This change ensures that we stop the maintenance service on all nodes
when a data node is restarted. This ensures that we don't send
update_by_query requests on the node that is restarted.
This commit also raises the log level to trace for some packages
in order to investigate the failures to acquire a shard lock
after a restart.

Relates #56765
This commit is contained in:
Jim Ferenczi 2020-05-26 09:29:56 +02:00 committed by jimczi
parent 184338ed61
commit 52443d41cf
2 changed files with 9 additions and 10 deletions

View File

@ -6,7 +6,6 @@
package org.elasticsearch.xpack.search; package org.elasticsearch.xpack.search;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
@ -17,7 +16,8 @@ import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.metrics.InternalMax; import org.elasticsearch.search.aggregations.metrics.InternalMax;
import org.elasticsearch.search.aggregations.metrics.InternalMin; import org.elasticsearch.search.aggregations.metrics.InternalMin;
import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.SuiteScopeTestCase;
import org.elasticsearch.test.junit.annotations.TestIssueLogging;
import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
@ -37,8 +37,10 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThan;
import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo;
@ESIntegTestCase.SuiteScopeTestCase @SuiteScopeTestCase
@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/56765") @TestIssueLogging(
value = "org.elasticsearch.index:TRACE,org.elasticsearch.env:TRACE",
issueUrl = "https://github.com/elastic/elasticsearch/issues/56765")
public class AsyncSearchActionIT extends AsyncSearchIntegTestCase { public class AsyncSearchActionIT extends AsyncSearchIntegTestCase {
private static String indexName; private static String indexName;
private static int numShards; private static int numShards;

View File

@ -134,12 +134,9 @@ public abstract class AsyncSearchIntegTestCase extends ESIntegTestCase {
final ClusterStateResponse clusterState = client().admin().cluster() final ClusterStateResponse clusterState = client().admin().cluster()
.prepareState().clear().setNodes(true).get(); .prepareState().clear().setNodes(true).get();
DiscoveryNode node = clusterState.getState().nodes().get(searchId.getTaskId().getNodeId()); DiscoveryNode node = clusterState.getState().nodes().get(searchId.getTaskId().getNodeId());
internalCluster().restartNode(node.getName(), new InternalTestCluster.RestartCallback() { stopMaintenanceService();
@Override internalCluster().restartNode(node.getName(), new InternalTestCluster.RestartCallback() {});
public Settings onNodeStopped(String nodeName) throws Exception { startMaintenanceService();
return super.onNodeStopped(nodeName);
}
});
ensureYellow(INDEX, indexName); ensureYellow(INDEX, indexName);
} }