honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Rollup] improve stopping tests (#55666) 

improve tests related to stopping using a client that answers and can be
synchronized with the test thread in order to test special situations

relates #55011
This commit is contained in:
Hendrik Muhs 2020-04-24 08:46:05 +02:00
parent 30f8c326fe
commit b213209f0c
1 changed files with 179 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

316
x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java
View File

@ -6,6 +6,9 @@
package org.elasticsearch.xpack.rollup.job; package org.elasticsearch.xpack.rollup.job;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.ActionType;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.client.Client; import org.elasticsearch.client.Client;
@ -19,6 +22,7 @@ import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregati
import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskId;
import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.tasks.TaskManager;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.client.NoOpClient;
import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.core.indexing.IndexerState; import org.elasticsearch.xpack.core.indexing.IndexerState;
@ -198,83 +202,95 @@ public class RollupJobTaskTests extends ESTestCase {
public void testStartWhenStopping() throws InterruptedException { public void testStartWhenStopping() throws InterruptedException {
RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
Client client = mock(Client.class);
when(client.settings()).thenReturn(Settings.EMPTY);
when(client.threadPool()).thenReturn(pool);
SchedulerEngine schedulerEngine = mock(SchedulerEngine.class);
AtomicInteger counter = new AtomicInteger(0); final CountDownLatch block = new CountDownLatch(1);
TaskId taskId = new TaskId("node", 123); final CountDownLatch unblock = new CountDownLatch(1);
RollupJobTask task = new RollupJobTask(1, "type", "action", taskId, job, try (NoOpClient client = getEmptySearchResponseClient(block, unblock)) {
null, client, schedulerEngine, pool, Collections.emptyMap()) { SchedulerEngine schedulerEngine = mock(SchedulerEngine.class);
@Override
public void updatePersistentTaskState(PersistentTaskState taskState, AtomicInteger counter = new AtomicInteger(0);
ActionListener<PersistentTasksCustomMetadata.PersistentTask<?>> listener) { TaskId taskId = new TaskId("node", 123);
assertThat(taskState, instanceOf(RollupJobStatus.class)); RollupJobTask task = new RollupJobTask(1, "type", "action", taskId, job,
int c = counter.get(); null, client, schedulerEngine, pool, Collections.emptyMap()) {
if (c == 0) { @Override
assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STARTED)); public void updatePersistentTaskState(PersistentTaskState taskState,
} else if (c == 1) { ActionListener<PersistentTasksCustomMetadata.PersistentTask<?>> listener) {
assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STOPPED)); assertThat(taskState, instanceOf(RollupJobStatus.class));
} else { int c = counter.get();
fail("Should not have updated persistent statuses > 2 times"); if (c == 0) {
assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STARTED));
} else if (c == 1) {
assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STOPPED));
} else if (c == 2) {
assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STOPPED));
} else {
fail("Should not have updated persistent statuses > 3 times");
}
listener.onResponse(new PersistentTasksCustomMetadata.PersistentTask<>("foo", RollupField.TASK_NAME, job, 1,
new PersistentTasksCustomMetadata.Assignment("foo", "foo")));
counter.incrementAndGet();
} }
listener.onResponse(new PersistentTasksCustomMetadata.PersistentTask<>("foo", RollupField.TASK_NAME, job, 1, };
new PersistentTasksCustomMetadata.Assignment("foo", "foo"))); task.init(null, mock(TaskManager.class), taskId.toString(), 123);
counter.incrementAndGet(); assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED));
} assertNull(((RollupJobStatus)task.getStatus()).getPosition());
};
task.init(null, mock(TaskManager.class), taskId.toString(), 123);
assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED));
assertNull(((RollupJobStatus)task.getStatus()).getPosition());
CountDownLatch latch = new CountDownLatch(1); CountDownLatch latch = new CountDownLatch(1);
task.start(new ActionListener<StartRollupJobAction.Response>() { task.start(new ActionListener<StartRollupJobAction.Response>() {
@Override @Override
public void onResponse(StartRollupJobAction.Response response) { public void onResponse(StartRollupJobAction.Response response) {
assertTrue(response.isStarted()); assertTrue(response.isStarted());
assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STARTED)); assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STARTED));
latch.countDown(); latch.countDown();
} }
@Override @Override
public void onFailure(Exception e) { public void onFailure(Exception e) {
fail("Should not have entered onFailure"); fail("Should not have entered onFailure");
} }
}); });
latch.await(3, TimeUnit.SECONDS); latch.await(3, TimeUnit.SECONDS);
task.triggered(new SchedulerEngine.Event(RollupJobTask.SCHEDULE_NAME + "_" + job.getConfig().getId(), 123, 123)); task.triggered(new SchedulerEngine.Event(RollupJobTask.SCHEDULE_NAME + "_" + job.getConfig().getId(), 123, 123));
assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.INDEXING)); assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.INDEXING));
assertThat(task.getStats().getNumInvocations(), equalTo(1L)); assertThat(task.getStats().getNumInvocations(), equalTo(1L));
task.stop(new ActionListener<StopRollupJobAction.Response>() { // wait until the search request is send, this is unblocked in the client
@Override block.await(3, TimeUnit.SECONDS);
public void onResponse(StopRollupJobAction.Response response) { task.stop(new ActionListener<StopRollupJobAction.Response>() {
assertTrue(response.isStopped()); @Override
} public void onResponse(StopRollupJobAction.Response response) {
assertTrue(response.isStopped());
}
@Override @Override
public void onFailure(Exception e) { public void onFailure(Exception e) {
fail("should not have entered onFailure"); fail("should not have entered onFailure");
} }
}); });
CountDownLatch latch2 = new CountDownLatch(1); // we issued stop but the indexer is waiting for the search response, therefore we should be in stopping state
task.start(new ActionListener<StartRollupJobAction.Response>() { assertThat(((RollupJobStatus) task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPING));
@Override
public void onResponse(StartRollupJobAction.Response response) {
fail("should not have entered onResponse");
}
@Override CountDownLatch latch2 = new CountDownLatch(1);
public void onFailure(Exception e) { task.start(new ActionListener<StartRollupJobAction.Response>() {
assertThat(e.getMessage(), equalTo("Cannot start task for Rollup Job [" @Override
+ job.getConfig().getId() + "] because state was [STOPPING]")); public void onResponse(StartRollupJobAction.Response response) {
latch2.countDown(); fail("should not have entered onResponse");
} }
});
latch2.await(3, TimeUnit.SECONDS); @Override
public void onFailure(Exception e) {
assertThat(e.getMessage(), equalTo("Cannot start task for Rollup Job ["
+ job.getConfig().getId() + "] because state was [STOPPING]"));
latch2.countDown();
}
});
latch2.await(3, TimeUnit.SECONDS);
// the the client answer
unblock.countDown();
}
} }
public void testStartWhenStopped() throws InterruptedException { public void testStartWhenStopped() throws InterruptedException {
@ -698,85 +714,94 @@ public class RollupJobTaskTests extends ESTestCase {
public void testStopWhenStopping() throws InterruptedException { public void testStopWhenStopping() throws InterruptedException {
RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap());
Client client = mock(Client.class); final CountDownLatch block = new CountDownLatch(1);
when(client.settings()).thenReturn(Settings.EMPTY); final CountDownLatch unblock = new CountDownLatch(1);
when(client.threadPool()).thenReturn(pool); try (NoOpClient client = getEmptySearchResponseClient(block, unblock)) {
SchedulerEngine schedulerEngine = mock(SchedulerEngine.class); SchedulerEngine schedulerEngine = mock(SchedulerEngine.class);
AtomicInteger counter = new AtomicInteger(0); AtomicInteger counter = new AtomicInteger(0);
TaskId taskId = new TaskId("node", 123); TaskId taskId = new TaskId("node", 123);
RollupJobTask task = new RollupJobTask(1, "type", "action", taskId, job, RollupJobTask task = new RollupJobTask(1, "type", "action", taskId, job,
null, client, schedulerEngine, pool, Collections.emptyMap()) { null, client, schedulerEngine, pool, Collections.emptyMap()) {
@Override @Override
public void updatePersistentTaskState(PersistentTaskState taskState, public void updatePersistentTaskState(PersistentTaskState taskState,
ActionListener<PersistentTasksCustomMetadata.PersistentTask<?>> listener) { ActionListener<PersistentTasksCustomMetadata.PersistentTask<?>> listener) {
assertThat(taskState, instanceOf(RollupJobStatus.class)); assertThat(taskState, instanceOf(RollupJobStatus.class));
int c = counter.get(); int c = counter.get();
if (c == 0) { if (c == 0) {
assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STARTED)); assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STARTED));
} else if (c == 1) { } else if (c == 1) {
assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STOPPED)); assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STOPPED));
} else if (c == 2) { } else if (c == 2) {
assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STOPPED)); assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STOPPED));
} else { } else if (c == 3) {
fail("Should not have updated persistent statuses > 3 times"); assertThat(((RollupJobStatus) taskState).getIndexerState(), equalTo(IndexerState.STOPPED));
} else {
fail("Should not have updated persistent statuses > 4 times");
}
listener.onResponse(new PersistentTasksCustomMetadata.PersistentTask<>("foo", RollupField.TASK_NAME, job, 1,
new PersistentTasksCustomMetadata.Assignment("foo", "foo")));
counter.incrementAndGet();
} }
listener.onResponse(new PersistentTasksCustomMetadata.PersistentTask<>("foo", RollupField.TASK_NAME, job, 1, };
new PersistentTasksCustomMetadata.Assignment("foo", "foo"))); task.init(null, mock(TaskManager.class), taskId.toString(), 123);
counter.incrementAndGet(); assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED));
assertNull(((RollupJobStatus)task.getStatus()).getPosition());
} CountDownLatch latch = new CountDownLatch(1);
}; task.start(new ActionListener<StartRollupJobAction.Response>() {
task.init(null, mock(TaskManager.class), taskId.toString(), 123); @Override
assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPED)); public void onResponse(StartRollupJobAction.Response response) {
assertNull(((RollupJobStatus)task.getStatus()).getPosition()); assertTrue(response.isStarted());
assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STARTED));
latch.countDown();
}
CountDownLatch latch = new CountDownLatch(1); @Override
task.start(new ActionListener<StartRollupJobAction.Response>() { public void onFailure(Exception e) {
@Override fail("Should not have entered onFailure");
public void onResponse(StartRollupJobAction.Response response) { }
assertTrue(response.isStarted()); });
assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.STARTED)); latch.await(3, TimeUnit.SECONDS);
latch.countDown();
}
@Override task.triggered(new SchedulerEngine.Event(RollupJobTask.SCHEDULE_NAME + "_" + job.getConfig().getId(), 123, 123));
public void onFailure(Exception e) { assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.INDEXING));
fail("Should not have entered onFailure"); assertThat(task.getStats().getNumInvocations(), equalTo(1L));
}
});
latch.await(3, TimeUnit.SECONDS);
task.triggered(new SchedulerEngine.Event(RollupJobTask.SCHEDULE_NAME + "_" + job.getConfig().getId(), 123, 123)); // wait until the search request is send, this is unblocked in the client
assertThat(((RollupJobStatus)task.getStatus()).getIndexerState(), equalTo(IndexerState.INDEXING)); block.await(3, TimeUnit.SECONDS);
assertThat(task.getStats().getNumInvocations(), equalTo(1L));
task.stop(new ActionListener<StopRollupJobAction.Response>() { task.stop(new ActionListener<StopRollupJobAction.Response>() {
@Override @Override
public void onResponse(StopRollupJobAction.Response response) { public void onResponse(StopRollupJobAction.Response response) {
assertTrue(response.isStopped()); assertTrue(response.isStopped());
} }
@Override @Override
public void onFailure(Exception e) { public void onFailure(Exception e) {
fail("should not have entered onFailure"); fail("should not have entered onFailure");
} }
}); });
CountDownLatch latch2 = new CountDownLatch(1); // we issued stop but the indexer is waiting for the search response, therefore we should be in stopping state
task.stop(new ActionListener<StopRollupJobAction.Response>() { assertThat(((RollupJobStatus) task.getStatus()).getIndexerState(), equalTo(IndexerState.STOPPING));
@Override
public void onResponse(StopRollupJobAction.Response response) {
assertTrue(response.isStopped());
latch2.countDown();
}
@Override CountDownLatch latch2 = new CountDownLatch(1);
public void onFailure(Exception e) { task.stop(new ActionListener<StopRollupJobAction.Response>() {
fail("Should not have entered onFailure"); @Override
} public void onResponse(StopRollupJobAction.Response response) {
}); assertTrue(response.isStopped());
latch2.await(3, TimeUnit.SECONDS); latch2.countDown();
}
@Override
public void onFailure(Exception e) {
fail("Should not have entered onFailure");
}
});
latch2.await(3, TimeUnit.SECONDS);
unblock.countDown();
}
} }
public void testStopWhenAborting() throws InterruptedException { public void testStopWhenAborting() throws InterruptedException {
@ -820,4 +845,21 @@ public class RollupJobTaskTests extends ESTestCase {
}); });
latch.await(3, TimeUnit.SECONDS); latch.await(3, TimeUnit.SECONDS);
} }
private NoOpClient getEmptySearchResponseClient(CountDownLatch unblock, CountDownLatch block) {
return new NoOpClient(getTestName()) {
@SuppressWarnings("unchecked")
@Override
protected <Request extends ActionRequest, Response extends ActionResponse>
void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
try {
unblock.countDown();
block.await(3, TimeUnit.SECONDS);
} catch (InterruptedException e) {
fail("Should not have timed out");
}
listener.onResponse((Response) mock(SearchResponse.class));
}
};
}
} }