honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Persistent Tasks: remove listener from PersistentTasksExecutor#nodeOperation (#1032) 

Instead of having a separate listener for indicating that the current task is finished, this commit is switching to use allocated object itself.
This commit is contained in:
Igor Motov 2017-04-10 17:32:30 -04:00 committed by Martijn van Groningen
parent 95c6005f6f
commit 0a1abd430d
No known key found for this signature in database
GPG Key ID: AB236F4FCF2AF12A
6 changed files with 90 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java
View File

@ -18,11 +18,17 @@
*/
package org.elasticsearch.persistent;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.tasks.CancellableTask;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskCancelledException;
import org.elasticsearch.tasks.TaskId;
import org.elasticsearch.tasks.TaskManager;
import java.util.concurrent.atomic.AtomicReference;
@ -38,6 +44,8 @@ public class AllocatedPersistentTask extends CancellableTask {
private Exception failure;
private PersistentTasksService persistentTasksService;
private Logger logger;
private TaskManager taskManager;
public AllocatedPersistentTask(long id, String type, String action, String description, TaskId parentTask) {
@ -66,7 +74,7 @@ public class AllocatedPersistentTask extends CancellableTask {
/**
* Updates the persistent state for the corresponding persistent task.
*
* <p>
* This doesn't affect the status of this allocated task.
*/
public void updatePersistentStatus(Task.Status status, ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>> listener) {
@ -77,8 +85,11 @@ public class AllocatedPersistentTask extends CancellableTask {
return persistentTaskId;
}
void init(PersistentTasksService persistentTasksService, long persistentTaskId, long allocationId) {
void init(PersistentTasksService persistentTasksService, TaskManager taskManager, Logger logger, long persistentTaskId, long
allocationId) {
this.persistentTasksService = persistentTasksService;
this.logger = logger;
this.taskManager = taskManager;
this.persistentTaskId = persistentTaskId;
this.allocationId = allocationId;
}
@ -87,16 +98,8 @@ public class AllocatedPersistentTask extends CancellableTask {
return failure;
}
State markAsCompleted(Exception failure) {
State prevState = state.getAndSet(AllocatedPersistentTask.State.COMPLETED);
if (prevState == State.STARTED || prevState == State.CANCELLED) {
this.failure = failure;
}
return prevState;
}
boolean markAsCancelled() {
return state.compareAndSet(AllocatedPersistentTask.State.STARTED, AllocatedPersistentTask.State.CANCELLED);
return state.compareAndSet(AllocatedPersistentTask.State.STARTED, AllocatedPersistentTask.State.PENDING_CANCEL);
}
public State getState() {
@ -109,7 +112,53 @@ public class AllocatedPersistentTask extends CancellableTask {
public enum State {
STARTED, // the task is currently running
CANCELLED, // the task is cancelled
PENDING_CANCEL, // the task is cancelled on master, cancelling it locally
COMPLETED // the task is done running and trying to notify caller
}
public void markAsCompleted() {
completeAndNotifyIfNeeded(null);
}
public void markAsFailed(Exception e) {
if (CancelTasksRequest.DEFAULT_REASON.equals(getReasonCancelled())) {
completeAndNotifyIfNeeded(null);
} else {
completeAndNotifyIfNeeded(e);
}
}
private void completeAndNotifyIfNeeded(@Nullable Exception failure) {
State prevState = state.getAndSet(AllocatedPersistentTask.State.COMPLETED);
if (prevState == State.COMPLETED) {
logger.warn("attempt to complete task {} in the {} state", getPersistentTaskId(), prevState);
} else {
if (failure != null) {
logger.warn((Supplier<?>) () -> new ParameterizedMessage(
"task {} failed with an exception", getPersistentTaskId()), failure);
}
try {
this.failure = failure;
if (prevState == State.STARTED) {
logger.trace("sending notification for completed task {}", getPersistentTaskId());
persistentTasksService.sendCompletionNotification(getPersistentTaskId(), failure, new
ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>>() {
@Override
public void onResponse(PersistentTasksCustomMetaData.PersistentTask<?> persistentTask) {
logger.trace("notification for task {} was successful", getId());
}
@Override
public void onFailure(Exception e) {
logger.warn((Supplier<?>) () ->
new ParameterizedMessage("notification for task {} failed", getPersistentTaskId()), e);
}
});
}
} finally {
taskManager.unregister(this);
}
}
}
}

11
server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java
View File

@ -18,10 +18,8 @@
*/
package org.elasticsearch.persistent;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty;
/**
* This component is responsible for execution of persistent tasks.
@ -37,21 +35,20 @@ public class NodePersistentTasksExecutor {
public <Request extends PersistentTaskRequest> void executeTask(Request request,
AllocatedPersistentTask task,
PersistentTasksExecutor<Request> action,
ActionListener<Empty> listener) {
PersistentTasksExecutor<Request> action) {
threadPool.executor(action.getExecutor()).execute(new AbstractRunnable() {
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
task.markAsFailed(e);
}
@SuppressWarnings("unchecked")
@Override
protected void doRun() throws Exception {
try {
action.nodeOperation(task, request, listener);
action.nodeOperation(task, request);
} catch (Exception ex) {
listener.onFailure(ex);
task.markAsFailed(ex);
}
}

7
server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java
View File

@ -99,11 +99,10 @@ public abstract class PersistentTasksExecutor<Request extends PersistentTaskRequ
/**
* This operation will be executed on the executor node.
* <p>
* If nodeOperation throws an exception or triggers listener.onFailure() method, the task will be restarted,
* possibly on a different node. If listener.onResponse() is called, the task is considered to be successfully
* completed and will be removed from the cluster state and not restarted.
* NOTE: The nodeOperation has to throws an exception, trigger task.markAsCompleted() or task.completeAndNotifyIfNeeded() methods to
* indicate that the persistent task has finished.
*/
protected abstract void nodeOperation(AllocatedPersistentTask task, Request request, ActionListener<Empty> listener);
protected abstract void nodeOperation(AllocatedPersistentTask task, Request request);
public String getExecutor() {
return executor;

87
server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.persistent;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterStateListener;
@ -32,9 +31,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskCancelledException;
import org.elasticsearch.tasks.TaskManager;
import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException;
@ -80,7 +77,7 @@ public class PersistentTasksNodeService extends AbstractComponent implements Clu
// STARTED COMPLETED Noop - waiting for notification ack
// NULL NULL Noop - nothing to do
// NULL STARTED Remove locally, Mark as CANCELLED, Cancel
// NULL STARTED Remove locally, Mark as PENDING_CANCEL, Cancel
// NULL COMPLETED Remove locally
// Master states:
@ -90,10 +87,10 @@ public class PersistentTasksNodeService extends AbstractComponent implements Clu
// Local state:
// NULL - we don't have task registered locally in runningTasks
// STARTED - registered in TaskManager, requires master notification when finishes
// CANCELLED - registered in TaskManager, doesn't require master notification when finishes
// PENDING_CANCEL - registered in TaskManager, doesn't require master notification when finishes
// COMPLETED - not registered in TaskManager, notified, waiting for master to remove it from CS so we can remove locally
// When task finishes if it is marked as STARTED or CANCELLED it is marked as COMPLETED and unregistered,
// When task finishes if it is marked as STARTED or PENDING_CANCEL it is marked as COMPLETED and unregistered,
// If the task was STARTED, the master notification is also triggered (this is handled by unregisterTask() method, which is
// triggered by PersistentTaskListener
@ -121,7 +118,7 @@ public class PersistentTasksNodeService extends AbstractComponent implements Clu
AllocatedPersistentTask task = runningTasks.get(id);
if (task.getState() == AllocatedPersistentTask.State.COMPLETED) {
// Result was sent to the caller and the caller acknowledged acceptance of the result
finishTask(id);
runningTasks.remove(id);
} else {
// task is running locally, but master doesn't know about it - that means that the persistent task was removed
// cancel the task without notifying master
@ -140,14 +137,13 @@ public class PersistentTasksNodeService extends AbstractComponent implements Clu
taskInProgress.getRequest());
boolean processed = false;
try {
task.init(persistentTasksService, taskInProgress.getId(), taskInProgress.getAllocationId());
PersistentTaskListener listener = new PersistentTaskListener(task);
task.init(persistentTasksService, taskManager, logger, taskInProgress.getId(), taskInProgress.getAllocationId());
try {
runningTasks.put(new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()), task);
nodePersistentTasksExecutor.executeTask(taskInProgress.getRequest(), task, action, listener);
nodePersistentTasksExecutor.executeTask(taskInProgress.getRequest(), task, action);
} catch (Exception e) {
// Submit task failure
listener.onFailure(e);
task.markAsFailed(e);
}
processed = true;
} finally {
@ -158,16 +154,6 @@ public class PersistentTasksNodeService extends AbstractComponent implements Clu
}
}
/**
* Unregisters the locally running task. No notification to master will be send upon cancellation.
*/
private void finishTask(PersistentTaskId persistentTaskId) {
AllocatedPersistentTask task = runningTasks.remove(persistentTaskId);
if (task != null) {
taskManager.unregister(task);
}
}
/**
* Unregisters and then cancels the locally running task using the task manager. No notification to master will be send upon
* cancellation.
@ -194,65 +180,6 @@ public class PersistentTasksNodeService extends AbstractComponent implements Clu
}
}
private void unregisterTask(AllocatedPersistentTask task, Exception e) {
AllocatedPersistentTask.State prevState = task.markAsCompleted(e);
if (prevState == AllocatedPersistentTask.State.CANCELLED) {
// The task was cancelled by master - no need to send notifications
taskManager.unregister(task);
} else if (prevState == AllocatedPersistentTask.State.STARTED) {
// The task finished locally, but master doesn't know about it - we need notify the master before we can unregister it
logger.trace("sending notification for completed task {}", task.getPersistentTaskId());
persistentTasksService.sendCompletionNotification(task.getPersistentTaskId(), e, new ActionListener<PersistentTask<?>>() {
@Override
public void onResponse(PersistentTask<?> persistentTask) {
logger.trace("notification for task {} was successful", task.getId());
taskManager.unregister(task);
}
@Override
public void onFailure(Exception e) {
logger.warn((Supplier<?>) () ->
new ParameterizedMessage("notification for task {} failed", task.getPersistentTaskId()), e);
taskManager.unregister(task);
}
});
} else {
logger.warn("attempt to complete task {} in the {} state", task.getPersistentTaskId(), prevState);
}
}
private class PersistentTaskListener implements ActionListener<Empty> {
private final AllocatedPersistentTask task;
PersistentTaskListener(final AllocatedPersistentTask task) {
this.task = task;
}
@Override
public void onResponse(Empty response) {
unregisterTask(task, null);
}
@Override
public void onFailure(Exception e) {
if (task.isCancelled()) {
// The task was explicitly cancelled - no need to restart it, just log the exception if it's not TaskCancelledException
if (e instanceof TaskCancelledException == false) {
logger.warn((Supplier<?>) () -> new ParameterizedMessage(
"cancelled task {} failed with an exception, cancellation reason [{}]",
task.getPersistentTaskId(), task.getReasonCancelled()), e);
}
if (CancelTasksRequest.DEFAULT_REASON.equals(task.getReasonCancelled())) {
unregisterTask(task, null);
} else {
unregisterTask(task, e);
}
} else {
unregisterTask(task, e);
}
}
}
private static class PersistentTaskId {
private final long id;
private final long allocationId;

21
server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java
View File

@ -35,7 +35,6 @@ import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskManager;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest;
@ -131,8 +130,8 @@ public class PersistentTasksNodeServiceTests extends ESTestCase {
assertThat(executor.size(), equalTo(2));
// Finish both tasks
executor.get(0).listener.onFailure(new RuntimeException());
executor.get(1).listener.onResponse(Empty.INSTANCE);
executor.get(0).task.markAsFailed(new RuntimeException());
executor.get(1).task.markAsCompleted();
long failedTaskId = executor.get(0).task.getParentTaskId().getId();
long finishedTaskId = executor.get(1).task.getParentTaskId().getId();
executor.clear();
@ -217,7 +216,7 @@ public class PersistentTasksNodeServiceTests extends ESTestCase {
// Make sure it returns correct status
assertThat(taskManager.getTasks().size(), equalTo(1));
assertThat(taskManager.getTasks().values().iterator().next().getStatus().toString(), equalTo("{\"state\":\"CANCELLED\"}"));
assertThat(taskManager.getTasks().values().iterator().next().getStatus().toString(), equalTo("{\"state\":\"PENDING_CANCEL\"}"));
// That should trigger cancellation request
@ -227,9 +226,9 @@ public class PersistentTasksNodeServiceTests extends ESTestCase {
// finish or fail task
if (randomBoolean()) {
executor.get(0).listener.onResponse(Empty.INSTANCE);
executor.get(0).task.markAsCompleted();
} else {
executor.get(0).listener.onFailure(new IOException("test"));
executor.get(0).task.markAsFailed(new IOException("test"));
}
// Check the the task is now removed from task manager
@ -265,14 +264,11 @@ public class PersistentTasksNodeServiceTests extends ESTestCase {
private final PersistentTaskRequest request;
private final AllocatedPersistentTask task;
private final PersistentTasksExecutor<?> holder;
private final ActionListener<Empty> listener;
Execution(PersistentTaskRequest request, AllocatedPersistentTask task, PersistentTasksExecutor<?> holder,
ActionListener<Empty> listener) {
Execution(PersistentTaskRequest request, AllocatedPersistentTask task, PersistentTasksExecutor<?> holder) {
this.request = request;
this.task = task;
this.holder = holder;
this.listener = listener;
}
}
@ -285,9 +281,8 @@ public class PersistentTasksNodeServiceTests extends ESTestCase {
@Override
public <Request extends PersistentTaskRequest> void executeTask(Request request, AllocatedPersistentTask task,
PersistentTasksExecutor<Request> action,
ActionListener<Empty> listener) {
executions.add(new Execution(request, task, action, listener));
PersistentTasksExecutor<Request> action) {
executions.add(new Execution(request, task, action));
}
public Execution get(int i) {

15
server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
View File

@ -59,7 +59,6 @@ import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskCancelledException;
import org.elasticsearch.tasks.TaskId;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.watcher.ResourceWatcherService;
import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment;
@ -340,7 +339,7 @@ public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin {
}
@Override
protected void nodeOperation(AllocatedPersistentTask task, TestRequest request, ActionListener<Empty> listener) {
protected void nodeOperation(AllocatedPersistentTask task, TestRequest request) {
logger.info("started node operation for the task {}", task);
try {
TestTask testTask = (TestTask) task;
@ -355,10 +354,10 @@ public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin {
return;
}
if ("finish".equals(testTask.getOperation())) {
listener.onResponse(Empty.INSTANCE);
task.markAsCompleted();
return;
} else if ("fail".equals(testTask.getOperation())) {
listener.onFailure(new RuntimeException("Simulating failure"));
task.markAsFailed(new RuntimeException("Simulating failure"));
return;
} else if ("update_status".equals(testTask.getOperation())) {
testTask.setOperation(null);
@ -384,12 +383,12 @@ public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin {
// Cancellation make cause different ways for the task to finish
if (randomBoolean()) {
if (randomBoolean()) {
listener.onFailure(new TaskCancelledException(testTask.getReasonCancelled()));
task.markAsFailed(new TaskCancelledException(testTask.getReasonCancelled()));
} else {
listener.onResponse(Empty.INSTANCE);
task.markAsCompleted();
}
} else {
listener.onFailure(new RuntimeException(testTask.getReasonCancelled()));
task.markAsFailed(new RuntimeException(testTask.getReasonCancelled()));
}
return;
} else {
@ -397,7 +396,7 @@ public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin {
}
}
} catch (InterruptedException e) {
listener.onFailure(e);
task.markAsFailed(e);
}
}
}