honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make PersistentAction independent from TransportActions (elastic/x-pack-elasticsearch#742) 

Removes the transport layer dependency from PersistentActions, makes PersistentActionRegistry immutable and rename actions into tasks in class and variable names.

Original commit: elastic/x-pack-elasticsearch@e3e5b79c28
This commit is contained in:
Igor Motov 2017-03-22 14:02:30 -04:00 committed by GitHub
parent d779bf66a5
commit 5c4933f5ea
62 changed files with 1315 additions and 1165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
dev-tools/checkstyle_suppressions.xml
View File

@ -376,16 +376,17 @@
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]CompletionPersistentTaskAction.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]CreatePersistentTaskAction.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]NodePersistentTask.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionCoordinator.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionExecutor.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionRegistry.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionRequest.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionService.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTaskClusterService.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasks.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]NodePersistentTasksExecutor.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTaskRequest.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTaskResponse.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksClusterService.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksCustomMetaData.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksExecutor.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksExecutorRegistry.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksNodeService.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksService.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]RemovePersistentTaskAction.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]StartPersistentTaskAction.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]TransportPersistentAction.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]UpdatePersistentTaskStatusAction.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]package-info.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]rest[/\\]XPackRestHandler.java" checks="LineLength" />
@ -1089,16 +1090,15 @@
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]notification[/\\]slack[/\\]SlackAccountsTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]notification[/\\]slack[/\\]message[/\\]SlackMessageDefaultsTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]notification[/\\]slack[/\\]message[/\\]SlackMessageTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionCoordinatorStatusTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionCoordinatorTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionFullRestartIT.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionIT.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionRegistryTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionResponseTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTaskClusterServiceTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksClusterServiceTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksCustomMetaDataTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksExecutorFullRestartIT.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksExecutorIT.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksExecutorResponseTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksNodeServiceTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]StartPersistentActionRequestTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]TestPersistentActionPlugin.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]TestPersistentTasksPlugin.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]security[/\\]InternalClientIntegTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]security[/\\]InternalClientTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]security[/\\]SecurityContextTests.java" checks="LineLength" />

55
plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
View File

@ -123,12 +123,12 @@ import org.elasticsearch.xpack.ml.rest.validate.RestValidateDetectorAction;
import org.elasticsearch.xpack.ml.rest.validate.RestValidateJobConfigAction;
import org.elasticsearch.xpack.persistent.CompletionPersistentTaskAction;
import org.elasticsearch.xpack.persistent.CreatePersistentTaskAction;
import org.elasticsearch.xpack.persistent.PersistentActionCoordinator;
import org.elasticsearch.xpack.persistent.PersistentActionRegistry;
import org.elasticsearch.xpack.persistent.PersistentActionRequest;
import org.elasticsearch.xpack.persistent.PersistentActionService;
import org.elasticsearch.xpack.persistent.PersistentTaskClusterService;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasksClusterService;
import org.elasticsearch.xpack.persistent.PersistentTasksNodeService;
import org.elasticsearch.xpack.persistent.PersistentTasksExecutorRegistry;
import org.elasticsearch.xpack.persistent.PersistentTaskRequest;
import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.RemovePersistentTaskAction;
import org.elasticsearch.xpack.persistent.StartPersistentTaskAction;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction;
@ -208,16 +208,18 @@ public class MachineLearning implements ActionPlugin {
// Custom metadata
new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new),
new NamedWriteableRegistry.Entry(NamedDiff.class, "ml", MlMetadata.MlMetadataDiff::new),
new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new),
new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom),
new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE,
PersistentTasksCustomMetaData::new),
new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE,
PersistentTasksCustomMetaData::readDiffFrom),
// Persistent action requests
new NamedWriteableRegistry.Entry(PersistentActionRequest.class, StartDatafeedAction.NAME, StartDatafeedAction.Request::new),
new NamedWriteableRegistry.Entry(PersistentActionRequest.class, OpenJobAction.NAME, OpenJobAction.Request::new),
new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, StartDatafeedAction.NAME, StartDatafeedAction.Request::new),
new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, OpenJobAction.NAME, OpenJobAction.Request::new),
// Task statuses
new NamedWriteableRegistry.Entry(Task.Status.class, PersistentActionCoordinator.Status.NAME,
PersistentActionCoordinator.Status::new),
new NamedWriteableRegistry.Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME,
PersistentTasksNodeService.Status::new),
new NamedWriteableRegistry.Entry(Task.Status.class, JobState.NAME, JobState::fromStream),
new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream)
);
@ -228,13 +230,13 @@ public class MachineLearning implements ActionPlugin {
// Custom metadata
new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField("ml"),
parser -> MlMetadata.ML_METADATA_PARSER.parse(parser, null).build()),
new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasks.TYPE),
PersistentTasks::fromXContent),
new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksCustomMetaData.TYPE),
PersistentTasksCustomMetaData::fromXContent),
// Persistent action requests
new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(StartDatafeedAction.NAME),
new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(StartDatafeedAction.NAME),
StartDatafeedAction.Request::fromXContent),
new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(OpenJobAction.NAME),
new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(OpenJobAction.NAME),
OpenJobAction.Request::fromXContent),
// Task statuses
@ -290,17 +292,22 @@ public class MachineLearning implements ActionPlugin {
}
NormalizerFactory normalizerFactory = new NormalizerFactory(normalizerProcessFactory,
threadPool.executor(MachineLearning.THREAD_POOL_NAME));
PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, internalClient);
AutodetectProcessManager autodetectProcessManager = new AutodetectProcessManager(settings, internalClient, threadPool,
jobManager, jobProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory,
normalizerFactory);
normalizerFactory, persistentTasksService);
DatafeedJobRunner datafeedJobRunner = new DatafeedJobRunner(threadPool, internalClient, clusterService, jobProvider,
System::currentTimeMillis, auditor);
PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, threadPool, clusterService,
internalClient);
PersistentActionRegistry persistentActionRegistry = new PersistentActionRegistry(Settings.EMPTY);
System::currentTimeMillis, persistentTasksService, auditor);
InvalidLicenseEnforcer invalidLicenseEnforcer =
new InvalidLicenseEnforcer(settings, licenseState, threadPool, datafeedJobRunner, autodetectProcessManager);
PersistentTasksExecutorRegistry persistentTasksExecutorRegistry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Arrays.asList(
new OpenJobAction.OpenJobPersistentTasksExecutor(settings, threadPool, licenseState, persistentTasksService, clusterService,
autodetectProcessManager, auditor),
new StartDatafeedAction.StartDatafeedPersistentTasksExecutor(settings, threadPool, licenseState, persistentTasksService,
datafeedJobRunner, auditor)
));
return Arrays.asList(
mlLifeCycleService,
jobProvider,
@ -310,9 +317,9 @@ public class MachineLearning implements ActionPlugin {
new MlInitializationService(settings, threadPool, clusterService, internalClient),
jobDataCountsPersister,
datafeedJobRunner,
persistentActionService,
persistentActionRegistry,
new PersistentTaskClusterService(Settings.EMPTY, persistentActionRegistry, clusterService),
persistentTasksService,
persistentTasksExecutorRegistry,
new PersistentTasksClusterService(Settings.EMPTY, persistentTasksExecutorRegistry, clusterService),
auditor,
new CloseJobService(internalClient, threadPool, clusterService),
invalidLicenseEnforcer

22
plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java
View File

@ -32,8 +32,8 @@ import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.messages.Messages;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException;
import java.util.Collection;
@ -240,7 +240,7 @@ public class MlMetadata implements MetaData.Custom {
return this;
}
public Builder deleteJob(String jobId, PersistentTasks tasks) {
public Builder deleteJob(String jobId, PersistentTasksCustomMetaData tasks) {
Optional<DatafeedConfig> datafeed = getDatafeedByJobId(jobId);
if (datafeed.isPresent()) {
throw ExceptionsHelper.conflictStatusException("Cannot delete job [" + jobId + "] while datafeed ["
@ -285,7 +285,7 @@ public class MlMetadata implements MetaData.Custom {
}
}
public Builder updateDatafeed(DatafeedUpdate update, PersistentTasks persistentTasks) {
public Builder updateDatafeed(DatafeedUpdate update, PersistentTasksCustomMetaData persistentTasks) {
String datafeedId = update.getId();
DatafeedConfig oldDatafeedConfig = datafeeds.get(datafeedId);
if (oldDatafeedConfig == null) {
@ -303,7 +303,7 @@ public class MlMetadata implements MetaData.Custom {
return this;
}
public Builder removeDatafeed(String datafeedId, PersistentTasks persistentTasks) {
public Builder removeDatafeed(String datafeedId, PersistentTasksCustomMetaData persistentTasks) {
DatafeedConfig datafeed = datafeeds.get(datafeedId);
if (datafeed == null) {
throw ExceptionsHelper.missingDatafeedException(datafeedId);
@ -318,7 +318,7 @@ public class MlMetadata implements MetaData.Custom {
return datafeeds.values().stream().filter(s -> s.getJobId().equals(jobId)).findFirst();
}
private void checkDatafeedIsStopped(Supplier<String> msg, String datafeedId, PersistentTasks persistentTasks) {
private void checkDatafeedIsStopped(Supplier<String> msg, String datafeedId, PersistentTasksCustomMetaData persistentTasks) {
if (persistentTasks != null) {
Predicate<PersistentTask<?>> predicate = t -> {
StartDatafeedAction.Request storedRequest = (StartDatafeedAction.Request) t.getRequest();
@ -348,7 +348,7 @@ public class MlMetadata implements MetaData.Custom {
return new MlMetadata(jobs, datafeeds);
}
public void markJobAsDeleted(String jobId, PersistentTasks tasks) {
public void markJobAsDeleted(String jobId, PersistentTasksCustomMetaData tasks) {
Job job = jobs.get(jobId);
if (job == null) {
throw ExceptionsHelper.missingJobException(jobId);
@ -375,7 +375,7 @@ public class MlMetadata implements MetaData.Custom {
}
@Nullable
public static PersistentTask<?> getJobTask(String jobId, @Nullable PersistentTasks tasks) {
public static PersistentTask<?> getJobTask(String jobId, @Nullable PersistentTasksCustomMetaData tasks) {
if (tasks != null) {
Predicate<PersistentTask<?>> p = t -> {
OpenJobAction.Request storedRequest = (OpenJobAction.Request) t.getRequest();
@ -389,7 +389,7 @@ public class MlMetadata implements MetaData.Custom {
}
@Nullable
public static PersistentTask<?> getDatafeedTask(String datafeedId, @Nullable PersistentTasks tasks) {
public static PersistentTask<?> getDatafeedTask(String datafeedId, @Nullable PersistentTasksCustomMetaData tasks) {
if (tasks != null) {
Predicate<PersistentTask<?>> p = t -> {
StartDatafeedAction.Request storedRequest = (StartDatafeedAction.Request) t.getRequest();
@ -402,7 +402,7 @@ public class MlMetadata implements MetaData.Custom {
return null;
}
public static JobState getJobState(String jobId, @Nullable PersistentTasks tasks) {
public static JobState getJobState(String jobId, @Nullable PersistentTasksCustomMetaData tasks) {
PersistentTask<?> task = getJobTask(jobId, tasks);
if (task != null && task.getStatus() != null) {
JobState jobTaskState = (JobState) task.getStatus();
@ -414,7 +414,7 @@ public class MlMetadata implements MetaData.Custom {
return JobState.CLOSED;
}
public static DatafeedState getDatafeedState(String datafeedId, @Nullable PersistentTasks tasks) {
public static DatafeedState getDatafeedState(String datafeedId, @Nullable PersistentTasksCustomMetaData tasks) {
PersistentTask<?> task = getDatafeedTask(datafeedId, tasks);
if (task != null && task.getStatus() != null) {
return (DatafeedState) task.getStatus();

12
plugin/src/main/java/org/elasticsearch/xpack/ml/action/CloseJobAction.java
View File

@ -43,8 +43,8 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException;
import java.util.Date;
@ -286,7 +286,7 @@ public class CloseJobAction extends Action<CloseJobAction.Request, CloseJobActio
throw ExceptionsHelper.missingJobException(jobId);
}
PersistentTasks tasks = state.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
Optional<DatafeedConfig> datafeed = mlMetadata.getDatafeedByJobId(jobId);
if (datafeed.isPresent()) {
DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeed.get().getId(), tasks);
@ -311,12 +311,12 @@ public class CloseJobAction extends Action<CloseJobAction.Request, CloseJobActio
static ClusterState moveJobToClosingState(String jobId, ClusterState currentState) {
PersistentTask<?> task = validateAndFindTask(jobId, currentState);
PersistentTasks currentTasks = currentState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData currentTasks = currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
Map<Long, PersistentTask<?>> updatedTasks = new HashMap<>(currentTasks.taskMap());
PersistentTask<?> taskToUpdate = currentTasks.getTask(task.getId());
taskToUpdate = new PersistentTask<>(taskToUpdate, JobState.CLOSING);
updatedTasks.put(taskToUpdate.getId(), taskToUpdate);
PersistentTasks newTasks = new PersistentTasks(currentTasks.getCurrentId(), updatedTasks);
PersistentTasksCustomMetaData newTasks = new PersistentTasksCustomMetaData(currentTasks.getCurrentId(), updatedTasks);
MlMetadata mlMetadata = currentState.metaData().custom(MlMetadata.TYPE);
Job.Builder jobBuilder = new Job.Builder(mlMetadata.getJobs().get(jobId));
@ -328,7 +328,7 @@ public class CloseJobAction extends Action<CloseJobAction.Request, CloseJobActio
return builder
.metaData(new MetaData.Builder(currentState.metaData())
.putCustom(MlMetadata.TYPE, mlMetadataBuilder.build())
.putCustom(PersistentTasks.TYPE, newTasks))
.putCustom(PersistentTasksCustomMetaData.TYPE, newTasks))
.build();
}
}

6
plugin/src/main/java/org/elasticsearch/xpack/ml/action/DeleteDatafeedAction.java
View File

@ -32,7 +32,7 @@ import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.io.IOException;
import java.util.Objects;
@ -170,8 +170,8 @@ public class DeleteDatafeedAction extends Action<DeleteDatafeedAction.Request, D
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
MlMetadata currentMetadata = state.getMetaData().custom(MlMetadata.TYPE);
PersistentTasks persistentTasks =
state.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData persistentTasks =
state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata)
.removeDatafeed(request.getDatafeedId(), persistentTasks).build();
return ClusterState.builder(state).metaData(

8
plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetDatafeedsStatsAction.java
View File

@ -38,8 +38,8 @@ import org.elasticsearch.xpack.ml.action.util.QueryPage;
import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException;
import java.util.ArrayList;
@ -318,7 +318,7 @@ public class GetDatafeedsStatsAction extends Action<GetDatafeedsStatsAction.Requ
.map(d -> d.getId()).collect(Collectors.toList())
: Collections.singletonList(request.getDatafeedId());
PersistentTasks tasksInProgress = state.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasksInProgress = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
List<DatafeedStats> results = expandedDatafeedsIds.stream()
.map(datafeedId -> getDatafeedStats(datafeedId, state, tasksInProgress))
.collect(Collectors.toList());
@ -328,7 +328,7 @@ public class GetDatafeedsStatsAction extends Action<GetDatafeedsStatsAction.Requ
}
private static DatafeedStats getDatafeedStats(String datafeedId, ClusterState state,
PersistentTasks tasks) {
PersistentTasksCustomMetaData tasks) {
PersistentTask<?> task = MlMetadata.getDatafeedTask(datafeedId, tasks);
DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeedId, tasks);
DiscoveryNode node = null;

10
plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsStatsAction.java
View File

@ -45,7 +45,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManage
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.io.IOException;
import java.time.Duration;
@ -410,10 +410,10 @@ public class GetJobsStatsAction extends Action<GetJobsStatsAction.Request, GetJo
String jobId = task.getJobId();
logger.debug("Get stats for job [{}]", jobId);
ClusterState state = clusterService.state();
PersistentTasks tasks = state.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
Optional<Tuple<DataCounts, ModelSizeStats>> stats = processManager.getStatistics(jobId);
if (stats.isPresent()) {
PersistentTasks.PersistentTask<?> pTask = MlMetadata.getJobTask(jobId, tasks);
PersistentTasksCustomMetaData.PersistentTask<?> pTask = MlMetadata.getJobTask(jobId, tasks);
DiscoveryNode node = state.nodes().get(pTask.getExecutorNode());
JobState jobState = MlMetadata.getJobState(jobId, tasks);
String assignmentExplanation = pTask.getAssignment().getExplanation();
@ -437,13 +437,13 @@ public class GetJobsStatsAction extends Action<GetJobsStatsAction.Request, GetJo
AtomicInteger counter = new AtomicInteger(jobIds.size());
AtomicArray<Response.JobStats> jobStats = new AtomicArray<>(jobIds.size());
PersistentTasks tasks = clusterService.state().getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
for (int i = 0; i < jobIds.size(); i++) {
int slot = i;
String jobId = jobIds.get(i);
gatherDataCountsAndModelSizeStats(jobId, (dataCounts, modelSizeStats) -> {
JobState jobState = MlMetadata.getJobState(request.jobId, tasks);
PersistentTasks.PersistentTask<?> pTask = MlMetadata.getJobTask(jobId, tasks);
PersistentTasksCustomMetaData.PersistentTask<?> pTask = MlMetadata.getJobTask(jobId, tasks);
String assignmentExplanation = null;
if (pTask != null) {
assignmentExplanation = pTask.getAssignment().getExplanation();

164
plugin/src/main/java/org/elasticsearch/xpack/ml/action/OpenJobAction.java
View File

@ -12,6 +12,8 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@ -50,14 +52,13 @@ import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.ml.utils.JobStateObserver;
import org.elasticsearch.xpack.persistent.NodePersistentTask;
import org.elasticsearch.xpack.persistent.PersistentActionRegistry;
import org.elasticsearch.xpack.persistent.PersistentActionRequest;
import org.elasticsearch.xpack.persistent.PersistentActionResponse;
import org.elasticsearch.xpack.persistent.PersistentActionService;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.TransportPersistentAction;
import org.elasticsearch.xpack.persistent.PersistentTasksExecutor;
import org.elasticsearch.xpack.persistent.PersistentTaskRequest;
import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException;
import java.util.ArrayList;
@ -68,7 +69,7 @@ import java.util.Objects;
import static org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE;
public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActionResponse, OpenJobAction.RequestBuilder> {
public class OpenJobAction extends Action<OpenJobAction.Request, OpenJobAction.Response, OpenJobAction.RequestBuilder> {
public static final OpenJobAction INSTANCE = new OpenJobAction();
public static final String NAME = "cluster:admin/ml/job/open";
@ -83,11 +84,11 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
}
@Override
public PersistentActionResponse newResponse() {
return new PersistentActionResponse();
public Response newResponse() {
return new Response();
}
public static class Request extends PersistentActionRequest {
public static class Request extends PersistentTaskRequest {
public static final ParseField IGNORE_DOWNTIME = new ParseField("ignore_downtime");
public static final ParseField TIMEOUT = new ParseField("timeout");
@ -124,7 +125,8 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
readFrom(in);
}
Request() {}
Request() {
}
public String getJobId() {
return jobId;
@ -216,6 +218,40 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
}
}
public static class Response extends AcknowledgedResponse {
public Response() {
super();
}
public Response(boolean acknowledged) {
super(acknowledged);
}
@Override
public void readFrom(StreamInput in) throws IOException {
readAcknowledged(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
writeAcknowledged(out);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AcknowledgedResponse that = (AcknowledgedResponse) o;
return isAcknowledged() == that.isAcknowledged();
}
@Override
public int hashCode() {
return Objects.hash(isAcknowledged());
}
}
public static class JobTask extends NodePersistentTask {
private final String jobId;
@ -243,70 +279,81 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
}
static class RequestBuilder extends ActionRequestBuilder<Request, PersistentActionResponse, RequestBuilder> {
static class RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder> {
RequestBuilder(ElasticsearchClient client, OpenJobAction action) {
super(client, action, new Request());
}
}
public static class TransportAction extends TransportPersistentAction<Request> {
public static class TransportAction extends HandledTransportAction<Request, Response> {
private final JobStateObserver observer;
private final ClusterService clusterService;
private final AutodetectProcessManager autodetectProcessManager;
private final XPackLicenseState licenseState;
private final Auditor auditor;
private volatile int maxConcurrentJobAllocations;
private final PersistentTasksService persistentTasksService;
@Inject
public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, XPackLicenseState licenseState,
PersistentActionService persistentActionService, PersistentActionRegistry persistentActionRegistry,
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
ClusterService clusterService, AutodetectProcessManager autodetectProcessManager, Auditor auditor) {
super(settings, OpenJobAction.NAME, false, threadPool, transportService, persistentActionService,
persistentActionRegistry, actionFilters, indexNameExpressionResolver, Request::new, ThreadPool.Names.MANAGEMENT);
PersistentTasksService persistentTasksService, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService) {
super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new);
this.licenseState = licenseState;
this.clusterService = clusterService;
this.autodetectProcessManager = autodetectProcessManager;
this.auditor = auditor;
this.persistentTasksService = persistentTasksService;
this.observer = new JobStateObserver(threadPool, clusterService);
this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings);
clusterService.getClusterSettings()
.addSettingsUpdateConsumer(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, this::setMaxConcurrentJobAllocations);
}
@Override
protected void doExecute(Request request, ActionListener<PersistentActionResponse> listener) {
protected void doExecute(Request request, ActionListener<Response> listener) {
if (licenseState.isMachineLearningAllowed()) {
// If we already know that we can't find an ml node because all ml nodes are running at capacity or
// simply because there are no ml nodes in the cluster then we fail quickly here:
ClusterState clusterState = clusterService.state();
validate(request, clusterState);
Assignment assignment = selectLeastLoadedMlNode(request.getJobId(), clusterState, maxConcurrentJobAllocations, logger);
if (assignment.getExecutorNode() == null) {
throw new ElasticsearchStatusException("cannot open job [" + request.getJobId() + "], no suitable nodes found, " +
"allocation explanation [{}]", RestStatus.TOO_MANY_REQUESTS, assignment.getExplanation());
}
PersistentTaskOperationListener finalListener = new PersistentTaskOperationListener() {
@Override
public void onResponse(long taskId) {
waitForJobStarted(request, listener);
}
ActionListener<PersistentActionResponse> finalListener =
ActionListener.wrap(response -> waitForJobStarted(request, response, listener), listener::onFailure);
super.doExecute(request, finalListener);
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
};
persistentTasksService.createPersistentActionTask(NAME, request, finalListener);
} else {
listener.onFailure(LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING));
}
}
void waitForJobStarted(Request request, PersistentActionResponse response, ActionListener<PersistentActionResponse> listener) {
void waitForJobStarted(Request request, ActionListener<Response> listener) {
observer.waitForState(request.getJobId(), request.timeout, JobState.OPENED, e -> {
if (e != null) {
listener.onFailure(e);
} else {
listener.onResponse(response);
listener.onResponse(new Response(true));
}
});
}
}
public static class OpenJobPersistentTasksExecutor extends PersistentTasksExecutor<Request> {
private final AutodetectProcessManager autodetectProcessManager;
private final XPackLicenseState licenseState;
private final Auditor auditor;
private final ThreadPool threadPool;
private volatile int maxConcurrentJobAllocations;
public OpenJobPersistentTasksExecutor(Settings settings, ThreadPool threadPool, XPackLicenseState licenseState,
PersistentTasksService persistentTasksService, ClusterService clusterService,
AutodetectProcessManager autodetectProcessManager, Auditor auditor) {
super(settings, NAME, persistentTasksService, ThreadPool.Names.MANAGEMENT);
this.licenseState = licenseState;
this.autodetectProcessManager = autodetectProcessManager;
this.auditor = auditor;
this.threadPool = threadPool;
this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings);
clusterService.getClusterSettings()
.addSettingsUpdateConsumer(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, this::setMaxConcurrentJobAllocations);
}
@Override
public Assignment getAssignment(Request request, ClusterState clusterState) {
@ -317,9 +364,20 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
@Override
public void validate(Request request, ClusterState clusterState) {
MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE);
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE);
OpenJobAction.validate(request.getJobId(), mlMetadata, tasks, clusterState.nodes());
if (licenseState.isMachineLearningAllowed()) {
// If we already know that we can't find an ml node because all ml nodes are running at capacity or
// simply because there are no ml nodes in the cluster then we fail quickly here:
MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE);
PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
OpenJobAction.validate(request.getJobId(), mlMetadata, tasks, clusterState.nodes());
Assignment assignment = selectLeastLoadedMlNode(request.getJobId(), clusterState, maxConcurrentJobAllocations, logger);
if (assignment.getExecutorNode() == null) {
throw new ElasticsearchStatusException("cannot open job [" + request.getJobId() + "], no suitable nodes found, " +
"allocation explanation [{}]", RestStatus.TOO_MANY_REQUESTS, assignment.getExplanation());
}
} else {
throw LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING);
}
}
@Override
@ -385,7 +443,7 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
* Fail fast before trying to update the job state on master node if the job doesn't exist or its state
* is not what it should be.
*/
static void validate(String jobId, MlMetadata mlMetadata, @Nullable PersistentTasks tasks, DiscoveryNodes nodes) {
static void validate(String jobId, MlMetadata mlMetadata, @Nullable PersistentTasksCustomMetaData tasks, DiscoveryNodes nodes) {
Job job = mlMetadata.getJobs().get(jobId);
if (job == null) {
throw ExceptionsHelper.missingJobException(jobId);
@ -410,7 +468,7 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
}
if (jobState.isAnyOf(JobState.CLOSED, JobState.FAILED) == false) {
throw new ElasticsearchStatusException("[" + jobId + "] expected state [" + JobState.CLOSED
+ "] or [" + JobState.FAILED + "], but got [" + jobState +"]", RestStatus.CONFLICT);
+ "] or [" + JobState.FAILED + "], but got [" + jobState + "]", RestStatus.CONFLICT);
}
}
@ -427,7 +485,7 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
long maxAvailable = Long.MIN_VALUE;
List<String> reasons = new LinkedList<>();
DiscoveryNode minLoadedNode = null;
PersistentTasks persistentTasks = clusterState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData persistentTasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
for (DiscoveryNode node : clusterState.getNodes()) {
Map<String, String> nodeAttributes = node.getAttributes();
String maxNumberOfOpenJobsStr = nodeAttributes.get(AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE.getKey());
@ -450,7 +508,7 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
return jobTaskState == null || // executor node didn't have the chance to set job status to OPENING
jobTaskState == JobState.OPENING || // executor node is busy starting the cpp process
task.isCurrentStatus() == false; // previous executor node failed and
// current executor node didn't have the chance to set job status to OPENING
// current executor node didn't have the chance to set job status to OPENING
}).size();
} else {
numberOfAssignedJobs = 0;

133
plugin/src/main/java/org/elasticsearch/xpack/ml/action/StartDatafeedAction.java
View File

@ -14,6 +14,8 @@ import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.ValidateActions;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@ -55,21 +57,20 @@ import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.ml.utils.DatafeedStateObserver;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.NodePersistentTask;
import org.elasticsearch.xpack.persistent.PersistentActionRegistry;
import org.elasticsearch.xpack.persistent.PersistentActionRequest;
import org.elasticsearch.xpack.persistent.PersistentActionResponse;
import org.elasticsearch.xpack.persistent.PersistentActionService;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.TransportPersistentAction;
import org.elasticsearch.xpack.persistent.PersistentTaskRequest;
import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasksExecutor;
import java.io.IOException;
import java.util.Objects;
import java.util.function.LongSupplier;
public class StartDatafeedAction
extends Action<StartDatafeedAction.Request, PersistentActionResponse, StartDatafeedAction.RequestBuilder> {
extends Action<StartDatafeedAction.Request, StartDatafeedAction.Response, StartDatafeedAction.RequestBuilder> {
public static final ParseField START_TIME = new ParseField("start");
public static final ParseField END_TIME = new ParseField("end");
@ -88,11 +89,11 @@ public class StartDatafeedAction
}
@Override
public PersistentActionResponse newResponse() {
return new PersistentActionResponse();
public Response newResponse() {
return new Response();
}
public static class Request extends PersistentActionRequest implements ToXContent {
public static class Request extends PersistentTaskRequest implements ToXContent {
public static ObjectParser<Request, Void> PARSER = new ObjectParser<>(NAME, Request::new);
@ -250,7 +251,41 @@ public class StartDatafeedAction
}
}
static class RequestBuilder extends ActionRequestBuilder<Request, PersistentActionResponse, RequestBuilder> {
public static class Response extends AcknowledgedResponse {
public Response() {
super();
}
public Response(boolean acknowledged) {
super(acknowledged);
}
@Override
public void readFrom(StreamInput in) throws IOException {
readAcknowledged(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
writeAcknowledged(out);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AcknowledgedResponse that = (AcknowledgedResponse) o;
return isAcknowledged() == that.isAcknowledged();
}
@Override
public int hashCode() {
return Objects.hash(isAcknowledged());
}
}
static class RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder> {
RequestBuilder(ElasticsearchClient client, StartDatafeedAction action) {
super(client, action, new Request());
@ -303,48 +338,68 @@ public class StartDatafeedAction
}
}
public static class TransportAction extends TransportPersistentAction<Request> {
public static class TransportAction extends HandledTransportAction<Request, Response> {
private final DatafeedStateObserver observer;
private final DatafeedJobRunner datafeedJobRunner;
private final XPackLicenseState licenseState;
private final Auditor auditor;
private final PersistentTasksService persistentTasksService;
@Inject
public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, XPackLicenseState licenseState,
PersistentActionService persistentActionService, PersistentActionRegistry persistentActionRegistry,
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
ClusterService clusterService, DatafeedJobRunner datafeedJobRunner, Auditor auditor) {
super(settings, NAME, false, threadPool, transportService, persistentActionService, persistentActionRegistry,
actionFilters, indexNameExpressionResolver, Request::new, ThreadPool.Names.MANAGEMENT);
PersistentTasksService persistentTasksService, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService) {
super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new);
this.licenseState = licenseState;
this.datafeedJobRunner = datafeedJobRunner;
this.auditor = auditor;
this.persistentTasksService = persistentTasksService;
this.observer = new DatafeedStateObserver(threadPool, clusterService);
}
@Override
protected void doExecute(Request request, ActionListener<PersistentActionResponse> listener) {
protected void doExecute(Request request, ActionListener<Response> listener) {
if (licenseState.isMachineLearningAllowed()) {
ActionListener<PersistentActionResponse> finalListener = ActionListener
.wrap(response -> waitForDatafeedStarted(request, response, listener), listener::onFailure);
super.doExecute(request, finalListener);
PersistentTaskOperationListener finalListener = new PersistentTaskOperationListener() {
@Override
public void onResponse(long taskId) {
waitForDatafeedStarted(request, listener);
}
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
};
persistentTasksService.createPersistentActionTask(NAME, request, finalListener);
} else {
listener.onFailure(LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING));
}
}
void waitForDatafeedStarted(Request request,
PersistentActionResponse response,
ActionListener<PersistentActionResponse> listener) {
void waitForDatafeedStarted(Request request, ActionListener<Response> listener) {
observer.waitForState(request.getDatafeedId(), request.timeout, DatafeedState.STARTED, e -> {
if (e != null) {
listener.onFailure(e);
} else {
listener.onResponse(response);
listener.onResponse(new Response(true));
}
});
}
}
public static class StartDatafeedPersistentTasksExecutor extends PersistentTasksExecutor<Request> {
private final DatafeedJobRunner datafeedJobRunner;
private final XPackLicenseState licenseState;
private final Auditor auditor;
private final ThreadPool threadPool;
public StartDatafeedPersistentTasksExecutor(Settings settings, ThreadPool threadPool, XPackLicenseState licenseState,
PersistentTasksService persistentTasksService, DatafeedJobRunner datafeedJobRunner,
Auditor auditor) {
super(settings, NAME, persistentTasksService, ThreadPool.Names.MANAGEMENT);
this.licenseState = licenseState;
this.datafeedJobRunner = datafeedJobRunner;
this.auditor = auditor;
this.threadPool = threadPool;
}
@Override
public Assignment getAssignment(Request request, ClusterState clusterState) {
@ -355,10 +410,14 @@ public class StartDatafeedAction
@Override
public void validate(Request request, ClusterState clusterState) {
MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE);
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE);
DiscoveryNodes nodes = clusterState.getNodes();
StartDatafeedAction.validate(request.getDatafeedId(), mlMetadata, tasks, nodes);
if (licenseState.isMachineLearningAllowed()) {
MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE);
PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
DiscoveryNodes nodes = clusterState.getNodes();
StartDatafeedAction.validate(request.getDatafeedId(), mlMetadata, tasks, nodes);
} else {
throw LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING);
}
}
@Override
@ -403,7 +462,7 @@ public class StartDatafeedAction
}
static void validate(String datafeedId, MlMetadata mlMetadata, PersistentTasks tasks, DiscoveryNodes nodes) {
static void validate(String datafeedId, MlMetadata mlMetadata, PersistentTasksCustomMetaData tasks, DiscoveryNodes nodes) {
DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId);
if (datafeed == null) {
throw ExceptionsHelper.missingDatafeedException(datafeedId);
@ -441,7 +500,7 @@ public class StartDatafeedAction
public static Assignment selectNode(Logger logger, String datafeedId, ClusterState clusterState) {
MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE);
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId);
DiscoveryNodes nodes = clusterState.getNodes();

8
plugin/src/main/java/org/elasticsearch/xpack/ml/action/StopDatafeedAction.java
View File

@ -44,8 +44,8 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.messages.Messages;
import org.elasticsearch.xpack.ml.utils.DatafeedStateObserver;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException;
import java.util.List;
@ -217,7 +217,7 @@ public class StopDatafeedAction
ClusterState state = clusterService.state();
MetaData metaData = state.metaData();
MlMetadata mlMetadata = metaData.custom(MlMetadata.TYPE);
PersistentTasks tasks = metaData.custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = metaData.custom(PersistentTasksCustomMetaData.TYPE);
String nodeId = validateAndReturnNodeId(request.getDatafeedId(), mlMetadata, tasks);
request.setNodes(nodeId);
ActionListener<Response> finalListener =
@ -262,7 +262,7 @@ public class StopDatafeedAction
}
}
static String validateAndReturnNodeId(String datafeedId, MlMetadata mlMetadata, PersistentTasks tasks) {
static String validateAndReturnNodeId(String datafeedId, MlMetadata mlMetadata, PersistentTasksCustomMetaData tasks) {
DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId);
if (datafeed == null) {
throw new ResourceNotFoundException(Messages.getMessage(Messages.DATAFEED_NOT_FOUND, datafeedId));

8
plugin/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java
View File

@ -30,7 +30,7 @@ import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.io.IOException;
import java.util.List;
@ -62,8 +62,8 @@ public abstract class TransportJobTaskAction<OperationTask extends Task, Request
// node running the job task.
ClusterState state = clusterService.state();
JobManager.getJobOrThrowIfUnknown(state, jobId);
PersistentTasks tasks = clusterService.state().getMetaData().custom(PersistentTasks.TYPE);
PersistentTasks.PersistentTask<?> jobTask = MlMetadata.getJobTask(jobId, tasks);
PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlMetadata.getJobTask(jobId, tasks);
if (jobTask == null || jobTask.isAssigned() == false) {
listener.onFailure( new ElasticsearchStatusException("job [" + jobId + "] state is [" + JobState.CLOSED +
"], but must be [" + JobState.OPENED + "] to perform requested action", RestStatus.CONFLICT));
@ -76,7 +76,7 @@ public abstract class TransportJobTaskAction<OperationTask extends Task, Request
@Override
protected final void taskOperation(Request request, OperationTask task, ActionListener<Response> listener) {
ClusterState state = clusterService.state();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
JobState jobState = MlMetadata.getJobState(request.getJobId(), tasks);
if (jobState == JobState.OPENED) {
innerTaskOperation(request, task, listener);

6
plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateDatafeedAction.java
View File

@ -32,7 +32,7 @@ import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.ml.datafeed.DatafeedUpdate;
import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.io.IOException;
import java.util.Objects;
@ -161,8 +161,8 @@ public class UpdateDatafeedAction extends Action<UpdateDatafeedAction.Request, P
public ClusterState execute(ClusterState currentState) throws Exception {
DatafeedUpdate update = request.getUpdate();
MlMetadata currentMetadata = state.getMetaData().custom(MlMetadata.TYPE);
PersistentTasks persistentTasks =
state.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData persistentTasks =
state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata)
.updateDatafeed(update, persistentTasks).build();
updatedDatafeed = newMetadata.getDatafeed(update.getId());

5
plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateJobAction.java
View File

@ -35,7 +35,8 @@ import org.elasticsearch.xpack.ml.job.JobManager;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.config.JobUpdate;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.io.IOException;
import java.util.Objects;
@ -172,7 +173,7 @@ public class UpdateJobAction extends Action<UpdateJobAction.Request, PutJobActio
throw new IllegalArgumentException("Job Id " + Job.ALL + " cannot be for update");
}
PersistentTasks tasks = clusterService.state().getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
boolean jobIsOpen = MlMetadata.getJobState(request.getJobId(), tasks) == JobState.OPENED;
semaphoreByJob.computeIfAbsent(request.getJobId(), id -> new Semaphore(1)).acquire();

5
plugin/src/main/java/org/elasticsearch/xpack/ml/client/MachineLearningClient.java
View File

@ -34,7 +34,6 @@ import org.elasticsearch.xpack.ml.action.StopDatafeedAction;
import org.elasticsearch.xpack.ml.action.UpdateDatafeedAction;
import org.elasticsearch.xpack.ml.action.UpdateJobAction;
import org.elasticsearch.xpack.ml.action.UpdateModelSnapshotAction;
import org.elasticsearch.xpack.persistent.PersistentActionResponse;
public class MachineLearningClient {
@ -109,7 +108,7 @@ public class MachineLearningClient {
client.execute(GetRecordsAction.INSTANCE, request, listener);
}
public void openJob(OpenJobAction.Request request, ActionListener<PersistentActionResponse> listener) {
public void openJob(OpenJobAction.Request request, ActionListener<OpenJobAction.Response> listener) {
client.execute(OpenJobAction.INSTANCE, request, listener);
}
@ -134,7 +133,7 @@ public class MachineLearningClient {
client.execute(RevertModelSnapshotAction.INSTANCE, request, listener);
}
public void startDatafeed(StartDatafeedAction.Request request, ActionListener<PersistentActionResponse> listener) {
public void startDatafeed(StartDatafeedAction.Request request, ActionListener<StartDatafeedAction.Response> listener) {
client.execute(StartDatafeedAction.INSTANCE, request, listener);
}

24
plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunner.java
View File

@ -34,9 +34,10 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.job.results.Bucket;
import org.elasticsearch.xpack.ml.job.results.Result;
import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.ml.utils.DatafeedStateObserver;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction;
import java.time.Duration;
import java.util.Collections;
@ -60,17 +61,19 @@ public class DatafeedJobRunner extends AbstractComponent {
private final JobProvider jobProvider;
private final ThreadPool threadPool;
private final Supplier<Long> currentTimeSupplier;
private final PersistentTasksService persistentTasksService;
private final Auditor auditor;
private final ConcurrentMap<String, Holder> runningDatafeeds = new ConcurrentHashMap<>();
public DatafeedJobRunner(ThreadPool threadPool, Client client, ClusterService clusterService, JobProvider jobProvider,
Supplier<Long> currentTimeSupplier, Auditor auditor) {
Supplier<Long> currentTimeSupplier, PersistentTasksService persistentTasksService, Auditor auditor) {
super(Settings.EMPTY);
this.client = Objects.requireNonNull(client);
this.clusterService = Objects.requireNonNull(clusterService);
this.jobProvider = Objects.requireNonNull(jobProvider);
this.threadPool = threadPool;
this.currentTimeSupplier = Objects.requireNonNull(currentTimeSupplier);
this.persistentTasksService = persistentTasksService;
this.auditor = auditor;
}
@ -268,10 +271,17 @@ public class DatafeedJobRunner extends AbstractComponent {
}
private void updateDatafeedState(long persistentTaskId, DatafeedState datafeedState, Consumer<Exception> handler) {
UpdatePersistentTaskStatusAction.Request request = new UpdatePersistentTaskStatusAction.Request(persistentTaskId, datafeedState);
client.execute(UpdatePersistentTaskStatusAction.INSTANCE, request, ActionListener.wrap(r -> {
handler.accept(null);
}, handler));
persistentTasksService.updateStatus(persistentTaskId, datafeedState, new PersistentTaskOperationListener() {
@Override
public void onResponse(long taskId) {
handler.accept(null);
}
@Override
public void onFailure(Exception e) {
handler.accept(e);
}
});
}
private static Duration getFrequencyOrDefault(DatafeedConfig datafeed, Job job) {

9
plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
View File

@ -19,7 +19,6 @@ import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.MachineLearningTemplateRegistry;
import org.elasticsearch.xpack.ml.action.DeleteJobAction;
import org.elasticsearch.xpack.ml.action.PutJobAction;
import org.elasticsearch.xpack.ml.action.RevertModelSnapshotAction;
@ -34,7 +33,7 @@ import org.elasticsearch.xpack.ml.job.persistence.JobStorageDeletionTask;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.util.Collections;
import java.util.List;
@ -129,7 +128,7 @@ public class JobManager extends AbstractComponent {
}
public JobState getJobState(String jobId) {
PersistentTasks tasks = clusterService.state().getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
return MlMetadata.getJobState(jobId, tasks);
}
@ -281,7 +280,7 @@ public class JobManager extends AbstractComponent {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
MlMetadata.Builder builder = createMlMetadataBuilder(currentState);
builder.deleteJob(jobId, currentState.getMetaData().custom(PersistentTasks.TYPE));
builder.deleteJob(jobId, currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE));
return buildNewClusterState(currentState, builder);
}
});
@ -307,7 +306,7 @@ public class JobManager extends AbstractComponent {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
MlMetadata currentMlMetadata = currentState.metaData().custom(MlMetadata.TYPE);
PersistentTasks tasks = currentState.metaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = currentState.metaData().custom(PersistentTasksCustomMetaData.TYPE);
MlMetadata.Builder builder = new MlMetadata.Builder(currentMlMetadata);
builder.markJobAsDeleted(jobId, tasks);
return buildNewClusterState(currentState, builder);

75
plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java
View File

@ -7,7 +7,6 @@ package org.elasticsearch.xpack.ml.job.process.autodetect;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.collect.Tuple;
@ -41,7 +40,8 @@ import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer;
import org.elasticsearch.xpack.ml.job.process.normalizer.ScoresUpdater;
import org.elasticsearch.xpack.ml.job.process.normalizer.ShortCircuitingRenormalizer;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction;
import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import java.io.IOException;
import java.io.InputStream;
@ -74,6 +74,7 @@ public class AutodetectProcessManager extends AbstractComponent {
private final JobResultsPersister jobResultsPersister;
private final JobDataCountsPersister jobDataCountsPersister;
private final PersistentTasksService persistentTasksService;
private final ConcurrentMap<String, AutodetectCommunicator> autoDetectCommunicatorByJob;
@ -82,7 +83,8 @@ public class AutodetectProcessManager extends AbstractComponent {
public AutodetectProcessManager(Settings settings, Client client, ThreadPool threadPool, JobManager jobManager,
JobProvider jobProvider, JobResultsPersister jobResultsPersister,
JobDataCountsPersister jobDataCountsPersister,
AutodetectProcessFactory autodetectProcessFactory, NormalizerFactory normalizerFactory) {
AutodetectProcessFactory autodetectProcessFactory, NormalizerFactory normalizerFactory,
PersistentTasksService persistentTasksService) {
super(settings);
this.client = client;
this.threadPool = threadPool;
@ -94,6 +96,7 @@ public class AutodetectProcessManager extends AbstractComponent {
this.jobResultsPersister = jobResultsPersister;
this.jobDataCountsPersister = jobDataCountsPersister;
this.persistentTasksService = persistentTasksService;
this.autoDetectCommunicatorByJob = new ConcurrentHashMap<>();
}
@ -113,18 +116,18 @@ public class AutodetectProcessManager extends AbstractComponent {
* Passes data to the native process.
* This is a blocking call that won't return until all the data has been
* written to the process.
*
* <p>
* An ElasticsearchStatusException will be thrown is any of these error conditions occur:
* <ol>
* <li>If a configured field is missing from the CSV header</li>
* <li>If JSON data is malformed and we cannot recover parsing</li>
* <li>If a high proportion of the records the timestamp field that cannot be parsed</li>
* <li>If a high proportion of the records chronologically out of order</li>
* <li>If a configured field is missing from the CSV header</li>
* <li>If JSON data is malformed and we cannot recover parsing</li>
* <li>If a high proportion of the records the timestamp field that cannot be parsed</li>
* <li>If a high proportion of the records chronologically out of order</li>
* </ol>
*
* @param jobId the jobId
* @param input Data input stream
* @param params Data processing parameters
* @param jobId the jobId
* @param input Data input stream
* @param params Data processing parameters
* @return Count of records, fields, bytes, etc written
*/
public DataCounts processData(String jobId, InputStream input, DataLoadParams params) {
@ -152,9 +155,9 @@ public class AutodetectProcessManager extends AbstractComponent {
* opportunity to process all data previously sent to it with none left
* sitting in buffers.
*
* @param jobId The job to flush
* @param jobId The job to flush
* @param params Parameters about whether interim results calculation
* should occur and for which period of time
* should occur and for which period of time
*/
public void flushJob(String jobId, InterimResultsParams params) {
logger.debug("Flushing job {}", jobId);
@ -278,9 +281,10 @@ public class AutodetectProcessManager extends AbstractComponent {
/**
* Stop the running job and mark it as finished.<br>
* @param jobId The job to stop
* @param restart Whether the job should be restarted by persistent tasks
* @param reason The reason for closing the job
*
* @param jobId The job to stop
* @param restart Whether the job should be restarted by persistent tasks
* @param reason The reason for closing the job
*/
public void closeJob(String jobId, boolean restart, String reason) {
logger.debug("Attempting to close job [{}], because [{}]", jobId, reason);
@ -321,34 +325,39 @@ public class AutodetectProcessManager extends AbstractComponent {
}
private void setJobState(long taskId, String jobId, JobState state) {
UpdatePersistentTaskStatusAction.Request request = new UpdatePersistentTaskStatusAction.Request(taskId, state);
client.execute(UpdatePersistentTaskStatusAction.INSTANCE, request, new ActionListener<UpdatePersistentTaskStatusAction.Response>() {
persistentTasksService.updateStatus(taskId, state, new PersistentTaskOperationListener() {
@Override
public void onResponse(UpdatePersistentTaskStatusAction.Response response) {
if (response.isAcknowledged()) {
logger.info("Successfully set job state to [{}] for job [{}]", state, jobId);
} else {
logger.info("Changing job state to [{}] for job [{}] wasn't acked", state, jobId);
}
public void onResponse(long taskId) {
logger.info("Successfully set job state to [{}] for job [{}]", state, jobId);
}
@Override
public void onFailure(Exception e) {
logger.error("Could not set job state to [" + state + "] for job [" + jobId +"]", e);
logger.error("Could not set job state to [" + state + "] for job [" + jobId + "]", e);
}
});
}
public void setJobState(long taskId, JobState state, CheckedConsumer<Exception, IOException> handler) {
UpdatePersistentTaskStatusAction.Request request = new UpdatePersistentTaskStatusAction.Request(taskId, state);
client.execute(UpdatePersistentTaskStatusAction.INSTANCE, request,
ActionListener.wrap(r -> handler.accept(null), e -> {
try {
handler.accept(e);
} catch (IOException e1) {
logger.warn("Error while delagating exception [" + e.getMessage() + "]", e1);
persistentTasksService.updateStatus(taskId, state, new PersistentTaskOperationListener() {
@Override
public void onResponse(long taskId) {
try {
handler.accept(null);
} catch (IOException e1) {
logger.warn("Error while delegating response", e1);
}
}
}));
@Override
public void onFailure(Exception e) {
try {
handler.accept(e);
} catch (IOException e1) {
logger.warn("Error while delegating exception [" + e.getMessage() + "]", e1);
}
}
});
}
public Optional<Tuple<DataCounts, ModelSizeStats>> getStatistics(String jobId) {

7
plugin/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java
View File

@ -20,7 +20,6 @@ import org.elasticsearch.rest.action.RestBuilderListener;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.action.StartDatafeedAction;
import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.persistent.PersistentActionResponse;
import java.io.IOException;
@ -55,12 +54,12 @@ public class RestStartDatafeedAction extends BaseRestHandler {
}
return channel -> {
client.execute(StartDatafeedAction.INSTANCE, jobDatafeedRequest,
new RestBuilderListener<PersistentActionResponse>(channel) {
new RestBuilderListener<StartDatafeedAction.Response>(channel) {
@Override
public RestResponse buildResponse(PersistentActionResponse r, XContentBuilder builder) throws Exception {
public RestResponse buildResponse(StartDatafeedAction.Response r, XContentBuilder builder) throws Exception {
builder.startObject();
builder.field("started", true);
builder.field("started", r.isAcknowledged());
builder.endObject();
return new BytesRestResponse(RestStatus.OK, builder);
}

8
plugin/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java
View File

@ -19,7 +19,6 @@ import org.elasticsearch.rest.action.RestBuilderListener;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.action.OpenJobAction;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.persistent.PersistentActionResponse;
import java.io.IOException;
@ -45,12 +44,11 @@ public class RestOpenJobAction extends BaseRestHandler {
}
}
return channel -> {
client.execute(OpenJobAction.INSTANCE, request, new RestBuilderListener<PersistentActionResponse>(channel) {
client.execute(OpenJobAction.INSTANCE, request, new RestBuilderListener<OpenJobAction.Response>(channel) {
@Override
public RestResponse buildResponse(PersistentActionResponse r, XContentBuilder builder) throws Exception {
public RestResponse buildResponse(OpenJobAction.Response r, XContentBuilder builder) throws Exception {
builder.startObject();
builder.field("opened", true);
builder.field("opened", r.isAcknowledged());
builder.endObject();
return new BytesRestResponse(RestStatus.OK, builder);
}

4
plugin/src/main/java/org/elasticsearch/xpack/ml/utils/DatafeedStateObserver.java
View File

@ -14,7 +14,7 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.util.function.Consumer;
import java.util.function.Predicate;
@ -35,7 +35,7 @@ public class DatafeedStateObserver {
ClusterStateObserver observer =
new ClusterStateObserver(clusterService, LOGGER, threadPool.getThreadContext());
Predicate<ClusterState> predicate = (newState) -> {
PersistentTasks tasks = newState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeedId, tasks);
return datafeedState == expectedState;
};

6
plugin/src/main/java/org/elasticsearch/xpack/ml/utils/JobStateObserver.java
View File

@ -16,7 +16,7 @@ import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.util.function.Consumer;
import java.util.function.Predicate;
@ -66,7 +66,7 @@ public class JobStateObserver {
handler.accept(null);
}
} else {
PersistentTasks tasks = state.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
JobState actual = MlMetadata.getJobState(jobId, tasks);
Exception e = new IllegalArgumentException("Timeout expired while waiting for job state [" + actual +
"] to change to [" + expectedState + "]");
@ -90,7 +90,7 @@ public class JobStateObserver {
@Override
public boolean test(ClusterState newState) {
PersistentTasks tasks = newState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
JobState jobState = MlMetadata.getJobState(jobId, tasks);
if (jobState == JobState.FAILED) {
failed = true;

38
plugin/src/main/java/org/elasticsearch/xpack/persistent/CompletionPersistentTaskAction.java
View File

@ -105,6 +105,36 @@ public class CompletionPersistentTaskAction extends Action<CompletionPersistentT
}
public static class Response extends AcknowledgedResponse {
public Response() {
super();
}
public Response(boolean acknowledged) {
super(acknowledged);
}
@Override
public void readFrom(StreamInput in) throws IOException {
readAcknowledged(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
writeAcknowledged(out);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AcknowledgedResponse that = (AcknowledgedResponse) o;
return isAcknowledged() == that.isAcknowledged();
}
@Override
public int hashCode() {
return Objects.hash(isAcknowledged());
}
}
@ -118,16 +148,16 @@ public class CompletionPersistentTaskAction extends Action<CompletionPersistentT
public static class TransportAction extends TransportMasterNodeAction<Request, Response> {
private final PersistentTaskClusterService persistentTaskClusterService;
private final PersistentTasksClusterService persistentTasksClusterService;
@Inject
public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters,
PersistentTaskClusterService persistentTaskClusterService,
PersistentTasksClusterService persistentTasksClusterService,
IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, CompletionPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, Request::new);
this.persistentTaskClusterService = persistentTaskClusterService;
this.persistentTasksClusterService = persistentTasksClusterService;
}
@Override
@ -148,7 +178,7 @@ public class CompletionPersistentTaskAction extends Action<CompletionPersistentT
@Override
protected final void masterOperation(final Request request, ClusterState state, final ActionListener<Response> listener) {
persistentTaskClusterService.completeOrRestartPersistentTask(request.taskId, request.exception, new ActionListener<Empty>() {
persistentTasksClusterService.completeOrRestartPersistentTask(request.taskId, request.exception, new ActionListener<Empty>() {
@Override
public void onResponse(Empty empty) {
listener.onResponse(newResponse());

50
plugin/src/main/java/org/elasticsearch/xpack/persistent/CreatePersistentTaskAction.java
View File

@ -34,7 +34,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
* This action can be used to add the record for the persistent action to the cluster state.
*/
public class CreatePersistentTaskAction extends Action<CreatePersistentTaskAction.Request,
PersistentActionResponse,
PersistentTaskResponse,
CreatePersistentTaskAction.RequestBuilder> {
public static final CreatePersistentTaskAction INSTANCE = new CreatePersistentTaskAction();
@ -50,15 +50,15 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
}
@Override
public PersistentActionResponse newResponse() {
return new PersistentActionResponse();
public PersistentTaskResponse newResponse() {
return new PersistentTaskResponse();
}
public static class Request extends MasterNodeRequest<Request> {
private String action;
private PersistentActionRequest request;
private PersistentTaskRequest request;
private boolean stopped;
@ -68,7 +68,7 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
}
public Request(String action, PersistentActionRequest request) {
public Request(String action, PersistentTaskRequest request) {
this.action = action;
this.request = request;
this.stopped = false;
@ -79,7 +79,7 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
action = in.readString();
request = in.readNamedWriteable(PersistentActionRequest.class);
request = in.readNamedWriteable(PersistentTaskRequest.class);
stopped = in.readBoolean();
removeOnCompletion = in.readBoolean();
}
@ -129,11 +129,11 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
this.action = action;
}
public PersistentActionRequest getRequest() {
public PersistentTaskRequest getRequest() {
return request;
}
public void setRequest(PersistentActionRequest request) {
public void setRequest(PersistentTaskRequest request) {
this.request = request;
}
@ -155,7 +155,7 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
}
public static class RequestBuilder extends MasterNodeOperationRequestBuilder<CreatePersistentTaskAction.Request,
PersistentActionResponse, CreatePersistentTaskAction.RequestBuilder> {
PersistentTaskResponse, CreatePersistentTaskAction.RequestBuilder> {
protected RequestBuilder(ElasticsearchClient client, CreatePersistentTaskAction action) {
super(client, action, new Request());
@ -166,8 +166,8 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
return this;
}
public RequestBuilder setRequest(PersistentActionRequest persistentActionRequest) {
request.setRequest(persistentActionRequest);
public RequestBuilder setRequest(PersistentTaskRequest persistentTaskRequest) {
request.setRequest(persistentTaskRequest);
return this;
}
@ -188,23 +188,23 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
}
}
public static class TransportAction extends TransportMasterNodeAction<Request, PersistentActionResponse> {
public static class TransportAction extends TransportMasterNodeAction<Request, PersistentTaskResponse> {
private final PersistentTaskClusterService persistentTaskClusterService;
private final PersistentTasksClusterService persistentTasksClusterService;
@Inject
public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters,
PersistentTaskClusterService persistentTaskClusterService,
PersistentActionRegistry persistentActionRegistry,
PersistentActionService persistentActionService,
PersistentTasksClusterService persistentTasksClusterService,
PersistentTasksExecutorRegistry persistentTasksExecutorRegistry,
PersistentTasksService persistentTasksService,
IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, CreatePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, Request::new);
this.persistentTaskClusterService = persistentTaskClusterService;
PersistentActionExecutor executor = new PersistentActionExecutor(threadPool);
clusterService.addListener(new PersistentActionCoordinator(settings, persistentActionService, persistentActionRegistry,
transportService.getTaskManager(), executor));
this.persistentTasksClusterService = persistentTasksClusterService;
NodePersistentTasksExecutor executor = new NodePersistentTasksExecutor(threadPool);
clusterService.addListener(new PersistentTasksNodeService(settings, persistentTasksService, persistentTasksExecutorRegistry,
transportService.getTaskManager(), threadPool, executor));
}
@Override
@ -213,8 +213,8 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
}
@Override
protected PersistentActionResponse newResponse() {
return new PersistentActionResponse();
protected PersistentTaskResponse newResponse() {
return new PersistentTaskResponse();
}
@Override
@ -225,12 +225,12 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
@Override
protected final void masterOperation(final Request request, ClusterState state,
final ActionListener<PersistentActionResponse> listener) {
persistentTaskClusterService.createPersistentTask(request.action, request.request, request.stopped, request.removeOnCompletion,
final ActionListener<PersistentTaskResponse> listener) {
persistentTasksClusterService.createPersistentTask(request.action, request.request, request.stopped, request.removeOnCompletion,
new ActionListener<Long>() {
@Override
public void onResponse(Long newTaskId) {
listener.onResponse(new PersistentActionResponse(newTaskId));
listener.onResponse(new PersistentTaskResponse(newTaskId));
}
@Override

18
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionExecutor.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/NodePersistentTasksExecutor.java
View File

@ -11,20 +11,20 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty;
/**
* This component is responsible for execution of persistent actions.
* This component is responsible for execution of persistent tasks.
*/
public class PersistentActionExecutor {
public class NodePersistentTasksExecutor {
private final ThreadPool threadPool;
public PersistentActionExecutor(ThreadPool threadPool) {
public NodePersistentTasksExecutor(ThreadPool threadPool) {
this.threadPool = threadPool;
}
public <Request extends PersistentActionRequest> void executeAction(Request request,
NodePersistentTask task,
PersistentActionRegistry.PersistentActionHolder<Request> holder,
ActionListener<Empty> listener) {
threadPool.executor(holder.getExecutor()).execute(new AbstractRunnable() {
public <Request extends PersistentTaskRequest> void executeTask(Request request,
NodePersistentTask task,
PersistentTasksExecutor<Request> action,
ActionListener<Empty> listener) {
threadPool.executor(action.getExecutor()).execute(new AbstractRunnable() {
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
@ -34,7 +34,7 @@ public class PersistentActionExecutor {
@Override
protected void doRun() throws Exception {
try {
holder.getPersistentAction().nodeOperation(task, request, listener);
action.nodeOperation(task, request, listener);
} catch (Exception ex) {
listener.onFailure(ex);
}

91
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRegistry.java
View File

@ -1,91 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import java.util.Collections;
import java.util.Map;
/**
* Components that registers all persistent actions
*/
public class PersistentActionRegistry extends AbstractComponent {
private volatile Map<String, PersistentActionHolder<?>> actions = Collections.emptyMap();
private final Object actionHandlerMutex = new Object();
public PersistentActionRegistry(Settings settings) {
super(settings);
}
public <Request extends PersistentActionRequest> void registerPersistentAction(String action,
TransportPersistentAction<Request> persistentAction) {
registerPersistentAction(new PersistentActionHolder<>(action, persistentAction, persistentAction.getExecutor()));
}
private <Request extends PersistentActionRequest> void registerPersistentAction(
PersistentActionHolder<Request> reg) {
synchronized (actionHandlerMutex) {
PersistentActionHolder<?> replaced = actions.get(reg.getAction());
actions = MapBuilder.newMapBuilder(actions).put(reg.getAction(), reg).immutableMap();
if (replaced != null) {
logger.warn("registered two handlers for persistent action {}, handlers: {}, {}", reg.getAction(), reg, replaced);
}
}
}
public void removeHandler(String action) {
synchronized (actionHandlerMutex) {
actions = MapBuilder.newMapBuilder(actions).remove(action).immutableMap();
}
}
@SuppressWarnings("unchecked")
public <Request extends PersistentActionRequest> PersistentActionHolder<Request> getPersistentActionHolderSafe(String action) {
PersistentActionHolder<Request> holder = (PersistentActionHolder<Request>) actions.get(action);
if (holder == null) {
throw new IllegalStateException("Unknown persistent action [" + action + "]");
}
return holder;
}
public <Request extends PersistentActionRequest>
TransportPersistentAction<Request> getPersistentActionSafe(String action) {
PersistentActionHolder<Request> holder = getPersistentActionHolderSafe(action);
return holder.getPersistentAction();
}
public static final class PersistentActionHolder<Request extends PersistentActionRequest> {
private final String action;
private final TransportPersistentAction<Request> persistentAction;
private final String executor;
public PersistentActionHolder(String action, TransportPersistentAction<Request> persistentAction, String executor) {
this.action = action;
this.persistentAction = persistentAction;
this.executor = executor;
}
public String getAction() {
return action;
}
public TransportPersistentAction<Request> getPersistentAction() {
return persistentAction;
}
public String getExecutor() {
return executor;
}
}
}

90
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionService.java
View File

@ -1,90 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.persistent;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskId;
import org.elasticsearch.threadpool.ThreadPool;
/**
* Service responsible for executing restartable actions that can survive disappearance of a coordinating and executor nodes.
*/
public class PersistentActionService extends AbstractComponent {
private final Client client;
private final ThreadPool threadPool;
private final ClusterService clusterService;
public PersistentActionService(Settings settings, ThreadPool threadPool, ClusterService clusterService, Client client) {
super(settings);
this.client = client;
this.threadPool = threadPool;
this.clusterService = clusterService;
}
public <Request extends PersistentActionRequest> void sendRequest(String action, Request request,
ActionListener<PersistentActionResponse> listener) {
CreatePersistentTaskAction.Request startRequest = new CreatePersistentTaskAction.Request(action, request);
try {
client.execute(CreatePersistentTaskAction.INSTANCE, startRequest, listener);
} catch (Exception e) {
listener.onFailure(e);
}
}
public void sendCompletionNotification(long taskId, Exception failure,
ActionListener<CompletionPersistentTaskAction.Response> listener) {
CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure);
// Need to fork otherwise: java.lang.AssertionError: should not be called by a cluster state applier.
// reason [the applied cluster state is not yet available])
try {
threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() {
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
@Override
protected void doRun() throws Exception {
client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, listener);
}
});
} catch (Exception e) {
listener.onFailure(e);
}
}
public void sendCancellation(long taskId, ActionListener<CancelTasksResponse> listener) {
DiscoveryNode localNode = clusterService.localNode();
CancelTasksRequest cancelTasksRequest = new CancelTasksRequest();
cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId));
cancelTasksRequest.setReason("persistent action was removed");
try {
client.admin().cluster().cancelTasks(cancelTasksRequest, listener);
} catch (Exception e) {
listener.onFailure(e);
}
}
public void updateStatus(long taskId, Task.Status status, ActionListener<UpdatePersistentTaskStatusAction.Response> listener) {
UpdatePersistentTaskStatusAction.Request updateStatusRequest = new UpdatePersistentTaskStatusAction.Request(taskId, status);
try {
client.execute(UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, listener);
} catch (Exception e) {
listener.onFailure(e);
}
}
}

4
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRequest.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskRequest.java
View File

@ -12,9 +12,9 @@ import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskId;
/**
* Base class for a request for a persistent action
* Base class for a request for a persistent task
*/
public abstract class PersistentActionRequest extends ActionRequest implements NamedWriteable, ToXContent {
public abstract class PersistentTaskRequest extends ActionRequest implements NamedWriteable, ToXContent {
@Override
public Task createTask(long id, String type, String action, TaskId parentTaskId) {
return new NodePersistentTask(id, type, action, getDescription(), parentTaskId);

10
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionResponse.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskResponse.java
View File

@ -13,16 +13,16 @@ import java.io.IOException;
import java.util.Objects;
/**
* Response upon a successful start or an persistent action
* Response upon a successful start or an persistent task
*/
public class PersistentActionResponse extends ActionResponse {
public class PersistentTaskResponse extends ActionResponse {
private long taskId;
public PersistentActionResponse() {
public PersistentTaskResponse() {
super();
}
public PersistentActionResponse(long taskId) {
public PersistentTaskResponse(long taskId) {
this.taskId = taskId;
}
@ -46,7 +46,7 @@ public class PersistentActionResponse extends ActionResponse {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
PersistentActionResponse that = (PersistentActionResponse) o;
PersistentTaskResponse that = (PersistentTaskResponse) o;
return taskId == that.taskId;
}

64
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterService.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksClusterService.java
View File

@ -19,20 +19,20 @@ import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.util.Objects;
/**
* Component that runs only on the master node and is responsible for assigning running tasks to nodes
*/
public class PersistentTaskClusterService extends AbstractComponent implements ClusterStateListener {
public class PersistentTasksClusterService extends AbstractComponent implements ClusterStateListener {
private final ClusterService clusterService;
private final PersistentActionRegistry registry;
private final PersistentTasksExecutorRegistry registry;
public PersistentTaskClusterService(Settings settings, PersistentActionRegistry registry, ClusterService clusterService) {
public PersistentTasksClusterService(Settings settings, PersistentTasksExecutorRegistry registry, ClusterService clusterService) {
super(settings);
this.clusterService = clusterService;
clusterService.addListener(this);
@ -47,15 +47,17 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
* @param request request
* @param listener the listener that will be called when task is started
*/
public <Request extends PersistentActionRequest> void createPersistentTask(String action, Request request, boolean stopped,
boolean removeOnCompletion,
ActionListener<Long> listener) {
public <Request extends PersistentTaskRequest> void createPersistentTask(String action, Request request, boolean stopped,
boolean removeOnCompletion,
ActionListener<Long> listener) {
clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
validate(action, clusterService.state(), request);
final Assignment assignment;
if (stopped) {
assignment = PersistentTasks.FINISHED_TASK_ASSIGNMENT; // the task is stopped no need to assign it anywhere
// the task is stopped no need to assign it anywhere
assignment = PersistentTasksCustomMetaData.FINISHED_TASK_ASSIGNMENT;
} else {
assignment = getAssignement(action, currentState, request);
}
@ -70,7 +72,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
listener.onResponse(
((PersistentTasks) newState.getMetaData().custom(PersistentTasks.TYPE)).getCurrentId());
((PersistentTasksCustomMetaData) newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)).getCurrentId());
}
});
}
@ -94,7 +96,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
PersistentTasks.Builder tasksInProgress = builder(currentState);
PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState);
if (tasksInProgress.hasTask(id)) {
if (failure != null) {
// If the task failed - we need to restart it on another node, otherwise we just remove it
@ -132,7 +134,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
clusterService.submitStateUpdateTask("start persistent task", new ClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
PersistentTasks.Builder tasksInProgress = builder(currentState);
PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState);
if (tasksInProgress.hasTask(id)) {
return update(currentState, tasksInProgress
.assignTask(id, (action, request) -> getAssignement(action, currentState, request)));
@ -163,7 +165,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
clusterService.submitStateUpdateTask("remove persistent task", new ClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
PersistentTasks.Builder tasksInProgress = builder(currentState);
PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState);
if (tasksInProgress.hasTask(id)) {
return update(currentState, tasksInProgress.removeTask(id));
} else {
@ -194,7 +196,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
PersistentTasks.Builder tasksInProgress = builder(currentState);
PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState);
if (tasksInProgress.hasTask(id)) {
return update(currentState, tasksInProgress.updateTaskStatus(id, status));
} else {
@ -214,10 +216,14 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
});
}
private <Request extends PersistentActionRequest> Assignment getAssignement(String action, ClusterState currentState, Request request) {
TransportPersistentAction<Request> persistentAction = registry.getPersistentActionSafe(action);
persistentAction.validate(request, currentState);
return persistentAction.getAssignment(request, currentState);
private <Request extends PersistentTaskRequest> Assignment getAssignement(String taskName, ClusterState currentState, Request request) {
PersistentTasksExecutor<Request> persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName);
return persistentTasksExecutor.getAssignment(request, currentState);
}
private <Request extends PersistentTaskRequest> void validate(String taskName, ClusterState currentState, Request request) {
PersistentTasksExecutor<Request> persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName);
persistentTasksExecutor.validate(request, currentState);
}
@Override
@ -234,12 +240,12 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
}
interface ExecutorNodeDecider {
<Request extends PersistentActionRequest> Assignment getAssignment(String action, ClusterState currentState, Request request);
<Request extends PersistentTaskRequest> Assignment getAssignment(String action, ClusterState currentState, Request request);
}
static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecider decider) {
PersistentTasks tasks = event.state().getMetaData().custom(PersistentTasks.TYPE);
PersistentTasks prevTasks = event.previousState().getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
PersistentTasksCustomMetaData prevTasks = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
if (tasks != null && (Objects.equals(tasks, prevTasks) == false ||
event.nodesChanged() ||
event.routingTableChanged() ||
@ -250,7 +256,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
if (taskInProgress.needsReassignment(event.state().nodes())) {
// there is an unassigned task or task with a disappeared node - we need to try assigning it
if (Objects.equals(taskInProgress.getAssignment(),
decider.getAssignment(taskInProgress.getAction(), event.state(), taskInProgress.getRequest())) == false) {
decider.getAssignment(taskInProgress.getTaskName(), event.state(), taskInProgress.getRequest())) == false) {
// it looks like a assignment for at least one task is possible - let's trigger reassignment
reassignmentRequired = true;
break;
@ -270,7 +276,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
clusterService.submitStateUpdateTask("reassign persistent tasks", new ClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
return reassignTasks(currentState, logger, PersistentTaskClusterService.this::getAssignement);
return reassignTasks(currentState, logger, PersistentTasksClusterService.this::getAssignement);
}
@Override
@ -286,7 +292,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
}
static ClusterState reassignTasks(ClusterState currentState, Logger logger, ExecutorNodeDecider decider) {
PersistentTasks tasks = currentState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
ClusterState clusterState = currentState;
DiscoveryNodes nodes = currentState.nodes();
if (tasks != null) {
@ -295,7 +301,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
for (PersistentTask<?> task : tasks.tasks()) {
if (task.needsReassignment(nodes)) {
// there is an unassigned task - we need to try assigning it
Assignment assignment = decider.getAssignment(task.getAction(), clusterState, task.getRequest());
Assignment assignment = decider.getAssignment(task.getTaskName(), clusterState, task.getRequest());
if (Objects.equals(assignment, task.getAssignment()) == false) {
logger.trace("reassigning task {} from node {} to node {}", task.getId(),
task.getAssignment().getExecutorNode(), assignment.getExecutorNode());
@ -315,14 +321,14 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
return clusterState;
}
private static PersistentTasks.Builder builder(ClusterState currentState) {
return PersistentTasks.builder(currentState.getMetaData().custom(PersistentTasks.TYPE));
private static PersistentTasksCustomMetaData.Builder builder(ClusterState currentState) {
return PersistentTasksCustomMetaData.builder(currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE));
}
private static ClusterState update(ClusterState currentState, PersistentTasks.Builder tasksInProgress) {
private static ClusterState update(ClusterState currentState, PersistentTasksCustomMetaData.Builder tasksInProgress) {
if (tasksInProgress.isChanged()) {
return ClusterState.builder(currentState).metaData(
MetaData.builder(currentState.metaData()).putCustom(PersistentTasks.TYPE, tasksInProgress.build())
MetaData.builder(currentState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasksInProgress.build())
).build();
} else {
return currentState;

107
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasks.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksCustomMetaData.java
View File

@ -43,7 +43,7 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constru
/**
* A cluster state record that contains a list of all running persistent tasks
*/
public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom> implements MetaData.Custom {
public final class PersistentTasksCustomMetaData extends AbstractNamedDiffable<MetaData.Custom> implements MetaData.Custom {
public static final String TYPE = "persistent_tasks";
private static final String API_CONTEXT = MetaData.XContentContext.API.toString();
@ -53,21 +53,21 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
private final long currentId;
public PersistentTasks(long currentId, Map<Long, PersistentTask<?>> tasks) {
public PersistentTasksCustomMetaData(long currentId, Map<Long, PersistentTask<?>> tasks) {
this.currentId = currentId;
this.tasks = tasks;
}
private static final ObjectParser<Builder, Void> PERSISTENT_TASKS_PARSER = new ObjectParser<>(TYPE, Builder::new);
private static final ObjectParser<TaskBuilder<PersistentActionRequest>, Void> PERSISTENT_TASK_PARSER =
private static final ObjectParser<TaskBuilder<PersistentTaskRequest>, Void> PERSISTENT_TASK_PARSER =
new ObjectParser<>("tasks", TaskBuilder::new);
public static final ConstructingObjectParser<Assignment, Void> ASSIGNMENT_PARSER =
new ConstructingObjectParser<>("assignment", objects -> new Assignment((String) objects[0], (String) objects[1]));
private static final NamedObjectParser<PersistentActionRequest, Void> REQUEST_PARSER =
(XContentParser p, Void c, String name) -> p.namedObject(PersistentActionRequest.class, name, null);
private static final NamedObjectParser<PersistentTaskRequest, Void> REQUEST_PARSER =
(XContentParser p, Void c, String name) -> p.namedObject(PersistentTaskRequest.class, name, null);
private static final NamedObjectParser<Status, Void> STATUS_PARSER =
(XContentParser p, Void c, String name) -> p.namedObject(Status.class, name, null);
@ -82,20 +82,20 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
// Task parser initialization
PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setId, new ParseField("id"));
PERSISTENT_TASK_PARSER.declareString(TaskBuilder::setAction, new ParseField("action"));
PERSISTENT_TASK_PARSER.declareString(TaskBuilder::setTaskName, new ParseField("name"));
PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setAllocationId, new ParseField("allocation_id"));
PERSISTENT_TASK_PARSER.declareBoolean(TaskBuilder::setRemoveOnCompletion, new ParseField("remove_on_completion"));
PERSISTENT_TASK_PARSER.declareBoolean(TaskBuilder::setStopped, new ParseField("stopped"));
PERSISTENT_TASK_PARSER.declareNamedObjects(
(TaskBuilder<PersistentActionRequest> taskBuilder, List<PersistentActionRequest> objects) -> {
(TaskBuilder<PersistentTaskRequest> taskBuilder, List<PersistentTaskRequest> objects) -> {
if (objects.size() != 1) {
throw new IllegalArgumentException("only one action request per task is allowed");
throw new IllegalArgumentException("only one request per task is allowed");
}
taskBuilder.setRequest(objects.get(0));
}, REQUEST_PARSER, new ParseField("request"));
PERSISTENT_TASK_PARSER.declareNamedObjects(
(TaskBuilder<PersistentActionRequest> taskBuilder, List<Status> objects) -> {
(TaskBuilder<PersistentTaskRequest> taskBuilder, List<Status> objects) -> {
if (objects.size() != 1) {
throw new IllegalArgumentException("only one status per task is allowed");
}
@ -120,16 +120,16 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return this.tasks.get(id);
}
public Collection<PersistentTask<?>> findTasks(String actionName, Predicate<PersistentTask<?>> predicate) {
public Collection<PersistentTask<?>> findTasks(String taskName, Predicate<PersistentTask<?>> predicate) {
return this.tasks().stream()
.filter(p -> actionName.equals(p.getAction()))
.filter(p -> taskName.equals(p.getTaskName()))
.filter(predicate)
.collect(Collectors.toList());
}
public boolean tasksExist(String actionName, Predicate<PersistentTask<?>> predicate) {
public boolean tasksExist(String taskName, Predicate<PersistentTask<?>> predicate) {
return this.tasks().stream()
.filter(p -> actionName.equals(p.getAction()))
.filter(p -> taskName.equals(p.getTaskName()))
.anyMatch(predicate);
}
@ -137,7 +137,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
PersistentTasks that = (PersistentTasks) o;
PersistentTasksCustomMetaData that = (PersistentTasksCustomMetaData) o;
return currentId == that.currentId &&
Objects.equals(tasks, that.tasks);
}
@ -152,8 +152,9 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return Strings.toString(this);
}
public long getNumberOfTasksOnNode(String nodeId, String action) {
return tasks.values().stream().filter(task -> action.equals(task.action) && nodeId.equals(task.assignment.executorNode)).count();
public long getNumberOfTasksOnNode(String nodeId, String taskName) {
return tasks.values().stream().filter(
task -> taskName.equals(task.taskName) && nodeId.equals(task.assignment.executorNode)).count();
}
@Override
@ -166,7 +167,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return ALL_CONTEXTS;
}
public static PersistentTasks fromXContent(XContentParser parser) throws IOException {
public static PersistentTasksCustomMetaData fromXContent(XContentParser parser) throws IOException {
return PERSISTENT_TASKS_PARSER.parse(parser, null).build();
}
@ -221,10 +222,10 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
/**
* A record that represents a single running persistent task
*/
public static class PersistentTask<Request extends PersistentActionRequest> implements Writeable, ToXContent {
public static class PersistentTask<Request extends PersistentTaskRequest> implements Writeable, ToXContent {
private final long id;
private final long allocationId;
private final String action;
private final String taskName;
private final Request request;
private final boolean stopped;
private final boolean removeOnCompletion;
@ -235,25 +236,25 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
private final Long allocationIdOnLastStatusUpdate;
public PersistentTask(long id, String action, Request request, boolean stopped, boolean removeOnCompletion, Assignment assignment) {
this(id, 0L, action, request, stopped, removeOnCompletion, null, assignment, null);
public PersistentTask(long id, String taskName, Request request, boolean stopped, boolean removeOnCompletion, Assignment assignment) {
this(id, 0L, taskName, request, stopped, removeOnCompletion, null, assignment, null);
}
public PersistentTask(PersistentTask<Request> task, boolean stopped, Assignment assignment) {
this(task.id, task.allocationId + 1L, task.action, task.request, stopped, task.removeOnCompletion, task.status,
this(task.id, task.allocationId + 1L, task.taskName, task.request, stopped, task.removeOnCompletion, task.status,
assignment, task.allocationId);
}
public PersistentTask(PersistentTask<Request> task, Status status) {
this(task.id, task.allocationId, task.action, task.request, task.stopped, task.removeOnCompletion, status,
this(task.id, task.allocationId, task.taskName, task.request, task.stopped, task.removeOnCompletion, status,
task.assignment, task.allocationId);
}
private PersistentTask(long id, long allocationId, String action, Request request, boolean stopped, boolean removeOnCompletion,
private PersistentTask(long id, long allocationId, String taskName, Request request, boolean stopped, boolean removeOnCompletion,
Status status, Assignment assignment, Long allocationIdOnLastStatusUpdate) {
this.id = id;
this.allocationId = allocationId;
this.action = action;
this.taskName = taskName;
this.request = request;
this.status = status;
this.stopped = stopped;
@ -268,8 +269,8 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
private PersistentTask(StreamInput in) throws IOException {
id = in.readLong();
allocationId = in.readLong();
action = in.readString();
request = (Request) in.readNamedWriteable(PersistentActionRequest.class);
taskName = in.readString();
request = (Request) in.readNamedWriteable(PersistentTaskRequest.class);
stopped = in.readBoolean();
removeOnCompletion = in.readBoolean();
status = in.readOptionalNamedWriteable(Task.Status.class);
@ -281,7 +282,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
public void writeTo(StreamOutput out) throws IOException {
out.writeLong(id);
out.writeLong(allocationId);
out.writeString(action);
out.writeString(taskName);
out.writeNamedWriteable(request);
out.writeBoolean(stopped);
out.writeBoolean(removeOnCompletion);
@ -298,7 +299,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
PersistentTask<?> that = (PersistentTask<?>) o;
return id == that.id &&
allocationId == that.allocationId &&
Objects.equals(action, that.action) &&
Objects.equals(taskName, that.taskName) &&
Objects.equals(request, that.request) &&
stopped == that.stopped &&
removeOnCompletion == that.removeOnCompletion &&
@ -309,7 +310,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
@Override
public int hashCode() {
return Objects.hash(id, allocationId, action, request, stopped, removeOnCompletion, status, assignment,
return Objects.hash(id, allocationId, taskName, request, stopped, removeOnCompletion, status, assignment,
allocationIdOnLastStatusUpdate);
}
@ -326,8 +327,8 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return allocationId;
}
public String getAction() {
return action;
public String getTaskName() {
return taskName;
}
public Request getRequest() {
@ -380,7 +381,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
builder.startObject();
{
builder.field("id", id);
builder.field("action", action);
builder.field("name", taskName);
builder.startObject("request");
{
builder.field(request.getWriteableName(), request, params);
@ -420,10 +421,10 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
}
}
private static class TaskBuilder<Request extends PersistentActionRequest> {
private static class TaskBuilder<Request extends PersistentTaskRequest> {
private long id;
private long allocationId;
private String action;
private String taskName;
private Request request;
private boolean stopped = true;
private boolean removeOnCompletion;
@ -441,8 +442,8 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return this;
}
public TaskBuilder<Request> setAction(String action) {
this.action = action;
public TaskBuilder<Request> setTaskName(String taskName) {
this.taskName = taskName;
return this;
}
@ -478,7 +479,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
}
public PersistentTask<Request> build() {
return new PersistentTask<>(id, allocationId, action, request, stopped, removeOnCompletion, status,
return new PersistentTask<>(id, allocationId, taskName, request, stopped, removeOnCompletion, status,
assignment, allocationIdOnLastStatusUpdate);
}
}
@ -488,7 +489,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return TYPE;
}
public PersistentTasks(StreamInput in) throws IOException {
public PersistentTasksCustomMetaData(StreamInput in) throws IOException {
currentId = in.readLong();
tasks = in.readMap(StreamInput::readLong, PersistentTask::new);
}
@ -525,7 +526,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return new Builder();
}
public static Builder builder(PersistentTasks tasks) {
public static Builder builder(PersistentTasksCustomMetaData tasks) {
return new Builder(tasks);
}
@ -537,7 +538,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
public Builder() {
}
public Builder(PersistentTasks tasksInProgress) {
public Builder(PersistentTasksCustomMetaData tasksInProgress) {
if (tasksInProgress != null) {
tasks.putAll(tasksInProgress.tasks);
currentId = tasksInProgress.currentId;
@ -551,7 +552,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return this;
}
private <Request extends PersistentActionRequest> Builder setTasks(List<TaskBuilder<Request>> tasks) {
private <Request extends PersistentTaskRequest> Builder setTasks(List<TaskBuilder<Request>> tasks) {
for (TaskBuilder builder : tasks) {
PersistentTask<?> task = builder.build();
this.tasks.put(task.getId(), task);
@ -564,11 +565,11 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
* <p>
* After the task is added its id can be found by calling {{@link #getCurrentId()}} method.
*/
public <Request extends PersistentActionRequest> Builder addTask(String action, Request request, boolean stopped,
boolean removeOnCompletion, Assignment assignment) {
public <Request extends PersistentTaskRequest> Builder addTask(String taskName, Request request, boolean stopped,
boolean removeOnCompletion, Assignment assignment) {
changed = true;
currentId++;
tasks.put(currentId, new PersistentTask<>(currentId, action, request, stopped, removeOnCompletion, assignment));
tasks.put(currentId, new PersistentTask<>(currentId, taskName, request, stopped, removeOnCompletion, assignment));
return this;
}
@ -591,11 +592,11 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
* {@link #reassignTask(long, BiFunction)} instead
*/
@SuppressWarnings("unchecked")
public <Request extends PersistentActionRequest> Builder assignTask(long taskId,
BiFunction<String, Request, Assignment> executorNodeFunc) {
public <Request extends PersistentTaskRequest> Builder assignTask(long taskId,
BiFunction<String, Request, Assignment> executorNodeFunc) {
PersistentTask<Request> taskInProgress = (PersistentTask<Request>) tasks.get(taskId);
if (taskInProgress != null && taskInProgress.assignment.isAssigned() == false) { // only assign unassigned tasks
Assignment assignment = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request);
Assignment assignment = executorNodeFunc.apply(taskInProgress.taskName, taskInProgress.request);
if (assignment.isAssigned() || taskInProgress.isStopped()) {
changed = true;
tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment));
@ -608,12 +609,12 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
* Reassigns the task to another node if the task exist
*/
@SuppressWarnings("unchecked")
public <Request extends PersistentActionRequest> Builder reassignTask(long taskId,
BiFunction<String, Request, Assignment> executorNodeFunc) {
public <Request extends PersistentTaskRequest> Builder reassignTask(long taskId,
BiFunction<String, Request, Assignment> executorNodeFunc) {
PersistentTask<Request> taskInProgress = (PersistentTask<Request>) tasks.get(taskId);
if (taskInProgress != null) {
changed = true;
Assignment assignment = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request);
Assignment assignment = executorNodeFunc.apply(taskInProgress.taskName, taskInProgress.request);
tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment));
}
return this;
@ -680,8 +681,8 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return changed;
}
public PersistentTasks build() {
return new PersistentTasks(currentId, Collections.unmodifiableMap(tasks));
public PersistentTasksCustomMetaData build() {
return new PersistentTasksCustomMetaData(currentId, Collections.unmodifiableMap(tasks));
}
}
}

54
plugin/src/main/java/org/elasticsearch/xpack/persistent/TransportPersistentAction.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutor.java
View File

@ -6,41 +6,37 @@
package org.elasticsearch.xpack.persistent;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import java.util.function.Predicate;
import java.util.function.Supplier;
/**
* An action that can survive restart of requesting or executing node.
* These actions are using cluster state rather than only transport service to send requests and responses.
* An executor of tasks that can survive restart of requesting or executing node.
* These tasks are using cluster state rather than only transport service to send requests and responses.
*/
public abstract class TransportPersistentAction<Request extends PersistentActionRequest>
extends HandledTransportAction<Request, PersistentActionResponse> {
public abstract class PersistentTasksExecutor<Request extends PersistentTaskRequest> extends AbstractComponent {
private final String executor;
private final PersistentActionService persistentActionService;
private final String taskName;
private final PersistentTasksService persistentTasksService;
protected TransportPersistentAction(Settings settings, String actionName, boolean canTripCircuitBreaker, ThreadPool threadPool,
TransportService transportService, PersistentActionService persistentActionService,
PersistentActionRegistry persistentActionRegistry,
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
Supplier<Request> requestSupplier, String executor) {
super(settings, actionName, canTripCircuitBreaker, threadPool, transportService, actionFilters, indexNameExpressionResolver,
requestSupplier);
protected PersistentTasksExecutor(Settings settings, String taskName, PersistentTasksService persistentTasksService,
String executor) {
super(settings);
this.taskName = taskName;
this.executor = executor;
this.persistentActionService = persistentActionService;
persistentActionRegistry.registerPersistentAction(actionName, this);
this.persistentTasksService = persistentTasksService;
}
public String getTaskName() {
return taskName;
}
public static final Assignment NO_NODE_FOUND = new Assignment(null, "no appropriate nodes found for the assignment");
@ -59,21 +55,20 @@ public abstract class TransportPersistentAction<Request extends PersistentAction
}
}
/**
* Finds the least loaded node that satisfies the selector criteria
*/
protected DiscoveryNode selectLeastLoadedNode(ClusterState clusterState, Predicate<DiscoveryNode> selector) {
long minLoad = Long.MAX_VALUE;
DiscoveryNode minLoadedNode = null;
PersistentTasks persistentTasks = clusterState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData persistentTasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
for (DiscoveryNode node : clusterState.getNodes()) {
if (selector.test(node)) {
if (persistentTasks == null) {
// We don't have any task running yet, pick the first available node
return node;
}
long numberOfTasks = persistentTasks.getNumberOfTasksOnNode(node.getId(), actionName);
long numberOfTasks = persistentTasks.getNumberOfTasksOnNode(node.getId(), taskName);
if (minLoad > numberOfTasks) {
minLoad = numberOfTasks;
minLoadedNode = node;
@ -92,11 +87,6 @@ public abstract class TransportPersistentAction<Request extends PersistentAction
}
@Override
protected void doExecute(Request request, ActionListener<PersistentActionResponse> listener) {
persistentActionService.sendRequest(actionName, request, listener);
}
/**
* Updates the persistent task status in the cluster state.
* <p>
@ -104,10 +94,10 @@ public abstract class TransportPersistentAction<Request extends PersistentAction
* task allocator about the state of the currently running tasks.
*/
protected void updatePersistentTaskStatus(NodePersistentTask task, Task.Status status, ActionListener<Empty> listener) {
persistentActionService.updateStatus(task.getPersistentTaskId(), status,
new ActionListener<UpdatePersistentTaskStatusAction.Response>() {
persistentTasksService.updateStatus(task.getPersistentTaskId(), status,
new PersistentTaskOperationListener() {
@Override
public void onResponse(UpdatePersistentTaskStatusAction.Response response) {
public void onResponse(long taskId) {
listener.onResponse(Empty.INSTANCE);
}

41
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorRegistry.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
/**
* Components that registers all persistent task executors
*/
public class PersistentTasksExecutorRegistry extends AbstractComponent {
private final Map<String, PersistentTasksExecutor<?>> taskExecutors;
@SuppressWarnings("unchecked")
public PersistentTasksExecutorRegistry(Settings settings, Collection<PersistentTasksExecutor<?>> taskExecutors) {
super(settings);
Map<String, PersistentTasksExecutor<?>> map = new HashMap<>();
for (PersistentTasksExecutor<?> executor : taskExecutors) {
map.put(executor.getTaskName(), executor);
}
this.taskExecutors = Collections.unmodifiableMap(map);
}
@SuppressWarnings("unchecked")
public <Request extends PersistentTaskRequest> PersistentTasksExecutor<Request> getPersistentTaskExecutorSafe(String taskName) {
PersistentTasksExecutor<Request> executor = (PersistentTasksExecutor<Request>) taskExecutors.get(taskName);
if (executor == null) {
throw new IllegalStateException("Unknown persistent executor [" + taskName + "]");
}
return executor;
}
}

78
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinator.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeService.java
View File

@ -9,7 +9,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.common.Nullable;
@ -19,12 +18,15 @@ import org.elasticsearch.common.inject.Provider;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskCancelledException;
import org.elasticsearch.tasks.TaskManager;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException;
import java.util.HashMap;
@ -37,33 +39,35 @@ import java.util.concurrent.atomic.AtomicReference;
import static java.util.Objects.requireNonNull;
/**
* This component is responsible for coordination of execution of persistent actions on individual nodes. It runs on all
* This component is responsible for coordination of execution of persistent tasks on individual nodes. It runs on all
* non-transport client nodes in the cluster and monitors cluster state changes to detect started commands.
*/
public class PersistentActionCoordinator extends AbstractComponent implements ClusterStateListener {
public class PersistentTasksNodeService extends AbstractComponent implements ClusterStateListener {
private final Map<PersistentTaskId, RunningPersistentTask> runningTasks = new HashMap<>();
private final PersistentActionService persistentActionService;
private final PersistentActionRegistry persistentActionRegistry;
private final PersistentTasksService persistentTasksService;
private final PersistentTasksExecutorRegistry persistentTasksExecutorRegistry;
private final TaskManager taskManager;
private final PersistentActionExecutor persistentActionExecutor;
private final ThreadPool threadPool;
private final NodePersistentTasksExecutor nodePersistentTasksExecutor;
public PersistentActionCoordinator(Settings settings,
PersistentActionService persistentActionService,
PersistentActionRegistry persistentActionRegistry,
TaskManager taskManager,
PersistentActionExecutor persistentActionExecutor) {
public PersistentTasksNodeService(Settings settings,
PersistentTasksService persistentTasksService,
PersistentTasksExecutorRegistry persistentTasksExecutorRegistry,
TaskManager taskManager, ThreadPool threadPool,
NodePersistentTasksExecutor nodePersistentTasksExecutor) {
super(settings);
this.persistentActionService = persistentActionService;
this.persistentActionRegistry = persistentActionRegistry;
this.persistentTasksService = persistentTasksService;
this.persistentTasksExecutorRegistry = persistentTasksExecutorRegistry;
this.taskManager = taskManager;
this.persistentActionExecutor = persistentActionExecutor;
this.threadPool = threadPool;
this.nodePersistentTasksExecutor = nodePersistentTasksExecutor;
}
@Override
public void clusterChanged(ClusterChangedEvent event) {
PersistentTasks tasks = event.state().getMetaData().custom(PersistentTasks.TYPE);
PersistentTasks previousTasks = event.previousState().getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
PersistentTasksCustomMetaData previousTasks = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) {
// We have some changes let's check if they are related to our node
@ -111,10 +115,9 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
}
private <Request extends PersistentActionRequest> void startTask(PersistentTask<Request> taskInProgress) {
PersistentActionRegistry.PersistentActionHolder<Request> holder =
persistentActionRegistry.getPersistentActionHolderSafe(taskInProgress.getAction());
NodePersistentTask task = (NodePersistentTask) taskManager.register("persistent", taskInProgress.getAction() + "[c]",
private <Request extends PersistentTaskRequest> void startTask(PersistentTask<Request> taskInProgress) {
PersistentTasksExecutor<Request> action = persistentTasksExecutorRegistry.getPersistentTaskExecutorSafe(taskInProgress.getTaskName());
NodePersistentTask task = (NodePersistentTask) taskManager.register("persistent", taskInProgress.getTaskName() + "[c]",
taskInProgress.getRequest());
boolean processed = false;
try {
@ -124,7 +127,7 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
PersistentTaskListener listener = new PersistentTaskListener(runningPersistentTask);
try {
runningTasks.put(new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()), runningPersistentTask);
persistentActionExecutor.executeAction(taskInProgress.getRequest(), task, holder, listener);
nodePersistentTasksExecutor.executeTask(taskInProgress.getRequest(), task, action, listener);
} catch (Exception e) {
// Submit task failure
listener.onFailure(e);
@ -149,9 +152,11 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
RunningPersistentTask task = runningTasks.remove(persistentTaskId);
if (task != null && task.getTask() != null) {
if (task.markAsCancelled()) {
persistentActionService.sendCancellation(task.getTask().getId(), new ActionListener<CancelTasksResponse>() {
persistentTasksService.sendCancellation(task.getTask().getId(), new PersistentTaskOperationListener() {
@Override
public void onResponse(CancelTasksResponse cancelTasksResponse) {
public void onResponse(long taskId) {
logger.trace("Persistent task with id {} was cancelled", taskId);
}
@Override
@ -171,7 +176,24 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
taskManager.unregister(task.getTask());
} else {
if (task.restartCompletionNotification()) {
persistentActionService.sendCompletionNotification(task.getId(), task.getFailure(), new PublishedResponseListener(task));
// Need to fork otherwise: java.lang.AssertionError: should not be called by a cluster state applier.
// reason [the applied cluster state is not yet available])
PublishedResponseListener listener = new PublishedResponseListener(task);
try {
threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() {
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
@Override
protected void doRun() throws Exception {
persistentTasksService.sendCompletionNotification(task.getId(), task.getFailure(), listener);
}
});
} catch (Exception e) {
listener.onFailure(e);
}
} else {
logger.warn("attempt to resend notification for task {} in the {} state", task.getId(), task.getState());
}
@ -184,7 +206,7 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
} else {
logger.trace("sending notification for failed task {}", task.getId());
if (task.startNotification(e)) {
persistentActionService.sendCompletionNotification(task.getId(), e, new PublishedResponseListener(task));
persistentTasksService.sendCompletionNotification(task.getId(), e, new PublishedResponseListener(task));
} else {
logger.warn("attempt to send notification for task {} in the {} state", task.getId(), task.getState());
}
@ -223,7 +245,7 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
}
}
private class PublishedResponseListener implements ActionListener<CompletionPersistentTaskAction.Response> {
private class PublishedResponseListener implements PersistentTaskOperationListener {
private final RunningPersistentTask task;
PublishedResponseListener(final RunningPersistentTask task) {
@ -232,7 +254,7 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
@Override
public void onResponse(CompletionPersistentTaskAction.Response response) {
public void onResponse(long taskId) {
logger.trace("notification for task {} was successful", task.getId());
if (task.markAsNotified() == false) {
logger.warn("attempt to mark task {} in the {} state as NOTIFIED", task.getId(), task.getState());

135
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksService.java Normal file
View File

@ -0,0 +1,135 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.persistent;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskId;
/**
* This service is used by persistent actions to propagate changes in the action state and notify about completion
*/
public class PersistentTasksService extends AbstractComponent {
private final Client client;
private final ClusterService clusterService;
public PersistentTasksService(Settings settings, ClusterService clusterService, Client client) {
super(settings);
this.client = client;
this.clusterService = clusterService;
}
/**
* Creates the specified persistent action and tries to start it immediately, upon completion the task is
* removed from the cluster state
*/
public <Request extends PersistentTaskRequest> void createPersistentActionTask(String action, Request request,
PersistentTaskOperationListener listener) {
createPersistentActionTask(action, request, false, true, listener);
}
/**
* Creates the specified persistent action. The action is started unless the stopped parameter is equal to true.
* If removeOnCompletion parameter is equal to true, the task is removed from the cluster state upon completion.
* Otherwise it will remain there in the stopped state.
*/
public <Request extends PersistentTaskRequest> void createPersistentActionTask(String action, Request request,
boolean stopped,
boolean removeOnCompletion,
PersistentTaskOperationListener listener) {
CreatePersistentTaskAction.Request createPersistentActionRequest = new CreatePersistentTaskAction.Request(action, request);
createPersistentActionRequest.setStopped(stopped);
createPersistentActionRequest.setRemoveOnCompletion(removeOnCompletion);
try {
client.execute(CreatePersistentTaskAction.INSTANCE, createPersistentActionRequest, ActionListener.wrap(
o -> listener.onResponse(o.getTaskId()), listener::onFailure));
} catch (Exception e) {
listener.onFailure(e);
}
}
/**
* Notifies the PersistentTasksClusterService about successful (failure == null) completion of a task or its failure
*
*/
public void sendCompletionNotification(long taskId, Exception failure, PersistentTaskOperationListener listener) {
CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure);
try {
client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, ActionListener.wrap(o -> listener.onResponse(taskId),
listener::onFailure));
} catch (Exception e) {
listener.onFailure(e);
}
}
/**
* Cancels the persistent task.
*/
public void sendCancellation(long taskId, PersistentTaskOperationListener listener) {
DiscoveryNode localNode = clusterService.localNode();
CancelTasksRequest cancelTasksRequest = new CancelTasksRequest();
cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId));
cancelTasksRequest.setReason("persistent action was removed");
try {
client.admin().cluster().cancelTasks(cancelTasksRequest, ActionListener.wrap(o -> listener.onResponse(taskId),
listener::onFailure));
} catch (Exception e) {
listener.onFailure(e);
}
}
/**
* Updates status of the persistent task
*/
public void updateStatus(long taskId, Task.Status status, PersistentTaskOperationListener listener) {
UpdatePersistentTaskStatusAction.Request updateStatusRequest = new UpdatePersistentTaskStatusAction.Request(taskId, status);
try {
client.execute(UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, ActionListener.wrap(
o -> listener.onResponse(taskId), listener::onFailure));
} catch (Exception e) {
listener.onFailure(e);
}
}
/**
* Removes a persistent task
*/
public void removeTask(long taskId, PersistentTaskOperationListener listener) {
RemovePersistentTaskAction.Request removeRequest = new RemovePersistentTaskAction.Request(taskId);
try {
client.execute(RemovePersistentTaskAction.INSTANCE, removeRequest, ActionListener.wrap(o -> listener.onResponse(taskId),
listener::onFailure));
} catch (Exception e) {
listener.onFailure(e);
}
}
/**
* Starts a persistent task
*/
public void startTask(long taskId, PersistentTaskOperationListener listener) {
StartPersistentTaskAction.Request startRequest = new StartPersistentTaskAction.Request(taskId);
try {
client.execute(StartPersistentTaskAction.INSTANCE, startRequest, ActionListener.wrap(o -> listener.onResponse(taskId),
listener::onFailure));
} catch (Exception e) {
listener.onFailure(e);
}
}
public interface PersistentTaskOperationListener {
void onResponse(long taskId);
void onFailure(Exception e);
}
}

8
plugin/src/main/java/org/elasticsearch/xpack/persistent/RemovePersistentTaskAction.java
View File

@ -148,16 +148,16 @@ public class RemovePersistentTaskAction extends Action<RemovePersistentTaskActio
public static class TransportAction extends TransportMasterNodeAction<Request, Response> {
private final PersistentTaskClusterService persistentTaskClusterService;
private final PersistentTasksClusterService persistentTasksClusterService;
@Inject
public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters,
PersistentTaskClusterService persistentTaskClusterService,
PersistentTasksClusterService persistentTasksClusterService,
IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, RemovePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, Request::new);
this.persistentTaskClusterService = persistentTaskClusterService;
this.persistentTasksClusterService = persistentTasksClusterService;
}
@Override
@ -178,7 +178,7 @@ public class RemovePersistentTaskAction extends Action<RemovePersistentTaskActio
@Override
protected final void masterOperation(final Request request, ClusterState state, final ActionListener<Response> listener) {
persistentTaskClusterService.removePersistentTask(request.taskId, new ActionListener<Empty>() {
persistentTasksClusterService.removePersistentTask(request.taskId, new ActionListener<Empty>() {
@Override
public void onResponse(Empty empty) {
listener.onResponse(new Response(true));

10
plugin/src/main/java/org/elasticsearch/xpack/persistent/StartPersistentTaskAction.java
View File

@ -31,7 +31,7 @@ import java.io.IOException;
import java.util.Objects;
/**
* This action can be used to start persistent action previously created using {@link CreatePersistentTaskAction}
* This action can be used to start a persistent task previously created using {@link CreatePersistentTaskAction}
*/
public class StartPersistentTaskAction extends Action<StartPersistentTaskAction.Request,
StartPersistentTaskAction.Response,
@ -151,16 +151,16 @@ public class StartPersistentTaskAction extends Action<StartPersistentTaskAction.
public static class TransportAction extends TransportMasterNodeAction<Request, Response> {
private final PersistentTaskClusterService persistentTaskClusterService;
private final PersistentTasksClusterService persistentTasksClusterService;
@Inject
public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters,
PersistentTaskClusterService persistentTaskClusterService,
PersistentTasksClusterService persistentTasksClusterService,
IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, StartPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, Request::new);
this.persistentTaskClusterService = persistentTaskClusterService;
this.persistentTasksClusterService = persistentTasksClusterService;
}
@Override
@ -181,7 +181,7 @@ public class StartPersistentTaskAction extends Action<StartPersistentTaskAction.
@Override
protected final void masterOperation(final Request request, ClusterState state, final ActionListener<Response> listener) {
persistentTaskClusterService.startPersistentTask(request.taskId, new ActionListener<Empty>() {
persistentTasksClusterService.startPersistentTask(request.taskId, new ActionListener<Empty>() {
@Override
public void onResponse(Empty empty) {
listener.onResponse(new Response(true));

8
plugin/src/main/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskStatusAction.java
View File

@ -164,16 +164,16 @@ public class UpdatePersistentTaskStatusAction extends Action<UpdatePersistentTas
public static class TransportAction extends TransportMasterNodeAction<Request, Response> {
private final PersistentTaskClusterService persistentTaskClusterService;
private final PersistentTasksClusterService persistentTasksClusterService;
@Inject
public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters,
PersistentTaskClusterService persistentTaskClusterService,
PersistentTasksClusterService persistentTasksClusterService,
IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, UpdatePersistentTaskStatusAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, Request::new);
this.persistentTaskClusterService = persistentTaskClusterService;
this.persistentTasksClusterService = persistentTasksClusterService;
}
@Override
@ -194,7 +194,7 @@ public class UpdatePersistentTaskStatusAction extends Action<UpdatePersistentTas
@Override
protected final void masterOperation(final Request request, ClusterState state, final ActionListener<Response> listener) {
persistentTaskClusterService.updatePersistentTaskStatus(request.taskId, request.status, new ActionListener<Empty>() {
persistentTasksClusterService.updatePersistentTaskStatus(request.taskId, request.status, new ActionListener<Empty>() {
@Override
public void onResponse(Empty empty) {
listener.onResponse(new Response(true));

31
plugin/src/main/java/org/elasticsearch/xpack/persistent/package-info.java
View File

@ -5,31 +5,30 @@
*/
/**
* The Persistent Actions are actions responsible for executing restartable actions that can survive disappearance of a
* The Persistent Tasks Executors are responsible for executing restartable tasks that can survive disappearance of a
* coordinating and executor nodes.
* <p>
* In order to be resilient to node restarts, the persistent actions are using the cluster state instead of a transport service to send
* In order to be resilient to node restarts, the persistent tasks are using the cluster state instead of a transport service to send
* requests and responses. The execution is done in six phases:
* <p>
* 1. The coordinating node sends an ordinary transport request to the master node to start a new persistent action. This action is handled
* by the {@link org.elasticsearch.xpack.persistent.PersistentActionService}, which is using
* {@link org.elasticsearch.xpack.persistent.PersistentTaskClusterService} to update cluster state with the record about running persistent
* 1. The coordinating node sends an ordinary transport request to the master node to start a new persistent task. This task is handled
* by the {@link org.elasticsearch.xpack.persistent.PersistentTasksService}, which is using
* {@link org.elasticsearch.xpack.persistent.PersistentTasksClusterService} to update cluster state with the record about running persistent
* task.
* <p>
* 2. The master node updates the {@link org.elasticsearch.xpack.persistent.PersistentTasks} in the cluster state to indicate that
* there is a new persistent action
* running in the system.
* 2. The master node updates the {@link org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData} in the cluster state to indicate
* that there is a new persistent task is running in the system.
* <p>
* 3. The {@link org.elasticsearch.xpack.persistent.PersistentActionCoordinator} running on every node in the cluster monitors changes in
* the cluster state and starts execution of all new actions assigned to the node it is running on.
* 3. The {@link org.elasticsearch.xpack.persistent.PersistentTasksNodeService} running on every node in the cluster monitors changes in
* the cluster state and starts execution of all new tasks assigned to the node it is running on.
* <p>
* 4. If the action fails to start on the node, the {@link org.elasticsearch.xpack.persistent.PersistentActionCoordinator} uses the
* {@link org.elasticsearch.xpack.persistent.PersistentTasks} to notify the
* {@link org.elasticsearch.xpack.persistent.PersistentActionService}, which reassigns the action to another node in the cluster.
* 4. If the task fails to start on the node, the {@link org.elasticsearch.xpack.persistent.PersistentTasksNodeService} uses the
* {@link org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData} to notify the
* {@link org.elasticsearch.xpack.persistent.PersistentTasksService}, which reassigns the action to another node in the cluster.
* <p>
* 5. If action finishes successfully on the node and calls listener.onResponse(), the corresponding persistent action is removed from the
* cluster state unless .
* 5. If a task finishes successfully on the node and calls listener.onResponse(), the corresponding persistent action is removed from the
* cluster state unless removeOnCompletion flag for this task is set to false.
* <p>
* 6. The {@link org.elasticsearch.xpack.persistent.RemovePersistentTaskAction} action can be also used to remove the persistent action.
* 6. The {@link org.elasticsearch.xpack.persistent.RemovePersistentTaskAction} action can be also used to remove the persistent task.
*/
package org.elasticsearch.xpack.persistent;

55
plugin/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
View File

@ -28,8 +28,7 @@ import org.elasticsearch.xpack.ml.client.MachineLearningClient;
import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentActionResponse;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.junit.Before;
import java.util.Collections;
@ -99,7 +98,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
// test that license restricted apis do not work
try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) {
client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress());
PlainListenableActionFuture<PersistentActionResponse> listener = new PlainListenableActionFuture<>(client.threadPool());
PlainListenableActionFuture<OpenJobAction.Response> listener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), listener);
listener.actionGet();
fail("open job action should not be enabled!");
@ -123,9 +122,9 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
// test that license restricted apis do now work
try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) {
client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress());
PlainListenableActionFuture<PersistentActionResponse> listener = new PlainListenableActionFuture<>(client.threadPool());
PlainListenableActionFuture<OpenJobAction.Response> listener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), listener);
PersistentActionResponse response = listener.actionGet();
OpenJobAction.Response response = listener.actionGet();
assertNotNull(response);
}
}
@ -192,12 +191,12 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet();
assertNotNull(putDatafeedResponse);
// open job
PlainListenableActionFuture<PersistentActionResponse> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
PlainListenableActionFuture<OpenJobAction.Response> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener);
PersistentActionResponse openJobResponse = openJobListener.actionGet();
OpenJobAction.Response openJobResponse = openJobListener.actionGet();
assertNotNull(openJobResponse);
// start datafeed
PlainListenableActionFuture<PersistentActionResponse> listener = new PlainListenableActionFuture<>(client.threadPool());
PlainListenableActionFuture<StartDatafeedAction.Response> listener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener);
listener.actionGet();
}
@ -218,7 +217,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
assertEquals(DatafeedState.STOPPED, datafeedState);
ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(0, tasks.taskMap().size());
});
@ -228,12 +227,12 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) {
client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress());
// open job
PlainListenableActionFuture<PersistentActionResponse> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
PlainListenableActionFuture<OpenJobAction.Response> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener);
PersistentActionResponse openJobResponse = openJobListener.actionGet();
OpenJobAction.Response openJobResponse = openJobListener.actionGet();
assertNotNull(openJobResponse);
// start datafeed
PlainListenableActionFuture<PersistentActionResponse> listener = new PlainListenableActionFuture<>(client.threadPool());
PlainListenableActionFuture<StartDatafeedAction.Response> listener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener);
listener.actionGet();
}
@ -246,7 +245,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
assertEquals(DatafeedState.STARTED, datafeedState);
ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(2, tasks.taskMap().size());
});
@ -266,7 +265,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
assertEquals(DatafeedState.STOPPED, datafeedState);
ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(0, tasks.taskMap().size());
});
}
@ -287,9 +286,9 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
new PutDatafeedAction.Request(createDatafeed("foobar", "foo", Collections.singletonList("foo"))), putDatafeedListener);
PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet();
assertNotNull(putDatafeedResponse);
PlainListenableActionFuture<PersistentActionResponse> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
PlainListenableActionFuture<OpenJobAction.Response> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener);
PersistentActionResponse openJobResponse = openJobListener.actionGet();
OpenJobAction.Response openJobResponse = openJobListener.actionGet();
assertNotNull(openJobResponse);
}
@ -303,14 +302,14 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
JobState jobState = getJobStats("foo").getState();
assertEquals(JobState.CLOSED, jobState);
ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(0, tasks.taskMap().size());
});
// test that license restricted apis do not work
try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) {
client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress());
PlainListenableActionFuture<PersistentActionResponse> listener = new PlainListenableActionFuture<>(client.threadPool());
PlainListenableActionFuture<StartDatafeedAction.Response> listener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener);
listener.actionGet();
fail("start datafeed action should not be enabled!");
@ -328,14 +327,14 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) {
client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress());
// re-open job now that the license is valid again
PlainListenableActionFuture<PersistentActionResponse> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
PlainListenableActionFuture<OpenJobAction.Response> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener);
PersistentActionResponse openJobResponse = openJobListener.actionGet();
OpenJobAction.Response openJobResponse = openJobListener.actionGet();
assertNotNull(openJobResponse);
PlainListenableActionFuture<PersistentActionResponse> listener = new PlainListenableActionFuture<>(client.threadPool());
PlainListenableActionFuture<StartDatafeedAction.Response> listener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener);
PersistentActionResponse response = listener.actionGet();
StartDatafeedAction.Response response = listener.actionGet();
assertNotNull(response);
}
}
@ -356,14 +355,14 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
new PutDatafeedAction.Request(createDatafeed("foobar", "foo", Collections.singletonList("foo"))), putDatafeedListener);
PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet();
assertNotNull(putDatafeedResponse);
PlainListenableActionFuture<PersistentActionResponse> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
PlainListenableActionFuture<OpenJobAction.Response> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener);
PersistentActionResponse openJobResponse = openJobListener.actionGet();
OpenJobAction.Response openJobResponse = openJobListener.actionGet();
assertNotNull(openJobResponse);
PlainListenableActionFuture<PersistentActionResponse> startDatafeedListener = new PlainListenableActionFuture<>(
PlainListenableActionFuture<StartDatafeedAction.Response> startDatafeedListener = new PlainListenableActionFuture<>(
client.threadPool());
new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), startDatafeedListener);
PersistentActionResponse startDatafeedResponse = startDatafeedListener.actionGet();
StartDatafeedAction.Response startDatafeedResponse = startDatafeedListener.actionGet();
assertNotNull(startDatafeedResponse);
}
@ -400,9 +399,9 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
new MachineLearningClient(client).putJob(new PutJobAction.Request(createJob("foo").build()), putJobListener);
PutJobAction.Response putJobResponse = putJobListener.actionGet();
assertNotNull(putJobResponse);
PlainListenableActionFuture<PersistentActionResponse> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
PlainListenableActionFuture<OpenJobAction.Response> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener);
PersistentActionResponse openJobResponse = openJobListener.actionGet();
OpenJobAction.Response openJobResponse = openJobListener.actionGet();
assertNotNull(openJobResponse);
}

24
plugin/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
View File

@ -26,8 +26,8 @@ import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.config.JobTests;
import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException;
import java.util.Collections;
@ -36,7 +36,7 @@ import static org.elasticsearch.xpack.ml.action.OpenJobActionTests.createJobTask
import static org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests.createDatafeedConfig;
import static org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests.createDatafeedJob;
import static org.elasticsearch.xpack.ml.job.config.JobTests.buildJobBuilder;
import static org.elasticsearch.xpack.persistent.PersistentTasks.INITIAL_ASSIGNMENT;
import static org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.sameInstance;
@ -133,7 +133,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
assertThat(result.getJobs().get("1"), sameInstance(job1));
assertThat(result.getDatafeeds().get("1"), nullValue());
builder.deleteJob("1", new PersistentTasks(0L, Collections.emptyMap()));
builder.deleteJob("1", new PersistentTasksCustomMetaData(0L, Collections.emptyMap()));
result = builder.build();
assertThat(result.getJobs().get("1"), nullValue());
assertThat(result.getDatafeeds().get("1"), nullValue());
@ -151,7 +151,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
PersistentTask<OpenJobAction.Request> task = createJobTask(0L, "1", null, JobState.CLOSED);
MlMetadata.Builder builder2 = new MlMetadata.Builder(result);
ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
() -> builder2.deleteJob("1", new PersistentTasks(0L, Collections.singletonMap(0L, task))));
() -> builder2.deleteJob("1", new PersistentTasksCustomMetaData(0L, Collections.singletonMap(0L, task))));
assertThat(e.status(), equalTo(RestStatus.CONFLICT));
}
@ -163,7 +163,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
builder.putDatafeed(datafeedConfig1);
ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
() -> builder.deleteJob(job1.getId(), new PersistentTasks(0L, Collections.emptyMap())));
() -> builder.deleteJob(job1.getId(), new PersistentTasksCustomMetaData(0L, Collections.emptyMap())));
assertThat(e.status(), equalTo(RestStatus.CONFLICT));
String expectedMsg = "Cannot delete job [" + job1.getId() + "] while datafeed [" + datafeedConfig1.getId() + "] refers to it";
assertThat(e.getMessage(), equalTo(expectedMsg));
@ -172,7 +172,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
public void testRemoveJob_failBecauseJobDoesNotExist() {
MlMetadata.Builder builder1 = new MlMetadata.Builder();
expectThrows(ResourceNotFoundException.class,
() -> builder1.deleteJob("1", new PersistentTasks(0L, Collections.emptyMap())));
() -> builder1.deleteJob("1", new PersistentTasksCustomMetaData(0L, Collections.emptyMap())));
}
public void testCrudDatafeed() {
@ -187,7 +187,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
assertThat(result.getDatafeeds().get("datafeed1"), sameInstance(datafeedConfig1));
builder = new MlMetadata.Builder(result);
builder.removeDatafeed("datafeed1", new PersistentTasks(0, Collections.emptyMap()));
builder.removeDatafeed("datafeed1", new PersistentTasksCustomMetaData(0, Collections.emptyMap()));
result = builder.build();
assertThat(result.getJobs().get("job_id"), sameInstance(job1));
assertThat(result.getDatafeeds().get("datafeed1"), nullValue());
@ -271,8 +271,8 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
StartDatafeedAction.Request request = new StartDatafeedAction.Request(datafeedConfig1.getId(), 0L);
PersistentTask<StartDatafeedAction.Request> taskInProgress =
new PersistentTask<>(0, StartDatafeedAction.NAME, request, false, true, INITIAL_ASSIGNMENT);
PersistentTasks tasksInProgress =
new PersistentTasks(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress));
PersistentTasksCustomMetaData tasksInProgress =
new PersistentTasksCustomMetaData(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress));
DatafeedUpdate.Builder update = new DatafeedUpdate.Builder(datafeedConfig1.getId());
update.setScrollSize(5000);
@ -333,8 +333,8 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
StartDatafeedAction.Request request = new StartDatafeedAction.Request("datafeed1", 0L);
PersistentTask<StartDatafeedAction.Request> taskInProgress =
new PersistentTask<>(0, StartDatafeedAction.NAME, request, false, true, INITIAL_ASSIGNMENT);
PersistentTasks tasksInProgress =
new PersistentTasks(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress));
PersistentTasksCustomMetaData tasksInProgress =
new PersistentTasksCustomMetaData(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress));
MlMetadata.Builder builder2 = new MlMetadata.Builder(result);
ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,

22
plugin/src/test/java/org/elasticsearch/xpack/ml/action/CloseJobActionTests.java
View File

@ -13,11 +13,11 @@ import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import java.util.Collections;
import java.util.HashMap;
@ -35,10 +35,11 @@ public class CloseJobActionTests extends ESTestCase {
createJobTask(1L, "job_id", null, randomFrom(JobState.OPENED, JobState.FAILED));
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
.putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, Collections.singletonMap(1L, task))));
.putCustom(PersistentTasksCustomMetaData.TYPE,
new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task))));
ClusterState result = CloseJobAction.moveJobToClosingState("job_id", csBuilder.build());
PersistentTasks actualTasks = result.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData actualTasks = result.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(JobState.CLOSING, actualTasks.getTask(1L).getStatus());
MlMetadata actualMetadata = result.metaData().custom(MlMetadata.TYPE);
@ -49,7 +50,7 @@ public class CloseJobActionTests extends ESTestCase {
MlMetadata.Builder mlBuilder = new MlMetadata.Builder();
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
.putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, Collections.emptyMap())));
.putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(1L, Collections.emptyMap())));
expectThrows(ResourceNotFoundException.class, () -> CloseJobAction.moveJobToClosingState("job_id", csBuilder.build()));
}
@ -59,14 +60,15 @@ public class CloseJobActionTests extends ESTestCase {
PersistentTask<OpenJobAction.Request> task = createJobTask(1L, "job_id", null, JobState.OPENING);
ClusterState.Builder csBuilder1 = ClusterState.builder(new ClusterName("_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
.putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, Collections.singletonMap(1L, task))));
.putCustom(PersistentTasksCustomMetaData.TYPE,
new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task))));
ElasticsearchStatusException result =
expectThrows(ElasticsearchStatusException.class, () -> CloseJobAction.moveJobToClosingState("job_id", csBuilder1.build()));
assertEquals("cannot close job [job_id], expected job state [opened], but got [opening]", result.getMessage());
ClusterState.Builder csBuilder2 = ClusterState.builder(new ClusterName("_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
.putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, Collections.emptyMap())));
.putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(1L, Collections.emptyMap())));
result = expectThrows(ElasticsearchStatusException.class, () -> CloseJobAction.moveJobToClosingState("job_id", csBuilder2.build()));
assertEquals("cannot close job [job_id], expected job state [opened], but got [closed]", result.getMessage());
}
@ -81,7 +83,7 @@ public class CloseJobActionTests extends ESTestCase {
tasks.put(2L, createDatafeedTask(2L, "datafeed_id", 0L, null, DatafeedState.STARTED));
ClusterState cs1 = ClusterState.builder(new ClusterName("_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
.putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, tasks))).build();
.putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(1L, tasks))).build();
ElasticsearchStatusException e =
expectThrows(ElasticsearchStatusException.class, () -> CloseJobAction.validateAndFindTask("job_id", cs1));
@ -95,7 +97,7 @@ public class CloseJobActionTests extends ESTestCase {
}
ClusterState cs2 = ClusterState.builder(new ClusterName("_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
.putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, tasks))).build();
.putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(1L, tasks))).build();
assertEquals(jobTask, CloseJobAction.validateAndFindTask("job_id", cs2));
}
@ -103,7 +105,7 @@ public class CloseJobActionTests extends ESTestCase {
String nodeId, DatafeedState datafeedState) {
PersistentTask<StartDatafeedAction.Request> task =
new PersistentTask<>(id, StartDatafeedAction.NAME, new StartDatafeedAction.Request(datafeedId, startTime), false, true,
new PersistentTasks.Assignment(nodeId, "test assignment"));
new PersistentTasksCustomMetaData.Assignment(nodeId, "test assignment"));
task = new PersistentTask<>(task, datafeedState);
return task;
}

21
plugin/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java
View File

@ -24,10 +24,9 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.persistent.PersistentActionCoordinator;
import org.elasticsearch.xpack.persistent.PersistentActionRequest;
import org.elasticsearch.xpack.persistent.PersistentActionResponse;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasksNodeService;
import org.elasticsearch.xpack.persistent.PersistentTaskRequest;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.security.Security;
import org.junit.After;
@ -70,12 +69,13 @@ public class DatafeedJobsIT extends SecurityIntegTestCase {
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedWriteables());
entries.addAll(new SearchModule(Settings.EMPTY, true, Collections.emptyList()).getNamedWriteables());
entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new));
entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new));
entries.add(new NamedWriteableRegistry.Entry(PersistentActionRequest.class, StartDatafeedAction.NAME,
entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE,
PersistentTasksCustomMetaData::new));
entries.add(new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, StartDatafeedAction.NAME,
StartDatafeedAction.Request::new));
entries.add(new NamedWriteableRegistry.Entry(PersistentActionRequest.class, OpenJobAction.NAME, OpenJobAction.Request::new));
entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, PersistentActionCoordinator.Status.NAME,
PersistentActionCoordinator.Status::new));
entries.add(new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, OpenJobAction.NAME, OpenJobAction.Request::new));
entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME,
PersistentTasksNodeService.Status::new));
entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, JobState.NAME, JobState::fromStream));
entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream));
final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
@ -187,8 +187,7 @@ public class DatafeedJobsIT extends SecurityIntegTestCase {
assertTrue(putDatafeedResponse.isAcknowledged());
StartDatafeedAction.Request startDatafeedRequest = new StartDatafeedAction.Request(datafeedConfig.getId(), 0L);
PersistentActionResponse startDatafeedResponse =
client().execute(StartDatafeedAction.INSTANCE, startDatafeedRequest).get();
StartDatafeedAction.Response startDatafeedResponse = client().execute(StartDatafeedAction.INSTANCE, startDatafeedRequest).get();
assertBusy(() -> {
DataCounts dataCounts = getDataCounts(job.getId());
assertThat(dataCounts.getProcessedRecordCount(), equalTo(numDocs1));

45
plugin/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionTests.java
View File

@ -29,12 +29,11 @@ import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.net.InetAddress;
import java.util.ArrayList;
@ -58,14 +57,14 @@ public class OpenJobActionTests extends ESTestCase {
PersistentTask<OpenJobAction.Request> task =
createJobTask(1L, "job_id", "_node_id", randomFrom(JobState.CLOSED, JobState.FAILED));
PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task));
PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
OpenJobAction.validate("job_id", mlBuilder.build(), tasks, nodes);
OpenJobAction.validate("job_id", mlBuilder.build(), new PersistentTasks(1L, Collections.emptyMap()), nodes);
OpenJobAction.validate("job_id", mlBuilder.build(), new PersistentTasksCustomMetaData(1L, Collections.emptyMap()), nodes);
OpenJobAction.validate("job_id", mlBuilder.build(), null, nodes);
task = createJobTask(1L, "job_id", "_other_node_id", JobState.OPENED);
tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task));
tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
OpenJobAction.validate("job_id", mlBuilder.build(), tasks, nodes);
}
@ -95,7 +94,7 @@ public class OpenJobActionTests extends ESTestCase {
JobState jobState = randomFrom(JobState.OPENING, JobState.OPENED, JobState.CLOSING);
PersistentTask<OpenJobAction.Request> task = createJobTask(1L, "job_id", "_node_id", jobState);
PersistentTasks tasks1 = new PersistentTasks(1L, Collections.singletonMap(1L, task));
PersistentTasksCustomMetaData tasks1 = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
Exception e = expectThrows(ElasticsearchStatusException.class,
() -> OpenJobAction.validate("job_id", mlBuilder.build(), tasks1, nodes));
@ -103,7 +102,7 @@ public class OpenJobActionTests extends ESTestCase {
jobState = randomFrom(JobState.OPENING, JobState.CLOSING);
task = createJobTask(1L, "job_id", "_other_node_id", jobState);
PersistentTasks tasks2 = new PersistentTasks(1L, Collections.singletonMap(1L, task));
PersistentTasksCustomMetaData tasks2 = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
e = expectThrows(ElasticsearchStatusException.class,
() -> OpenJobAction.validate("job_id", mlBuilder.build(), tasks2, nodes));
@ -129,14 +128,14 @@ public class OpenJobActionTests extends ESTestCase {
new Assignment("_node_id1", "test assignment")));
taskMap.put(2L, new PersistentTask<>(2L, OpenJobAction.NAME, new OpenJobAction.Request("job_id3"), false, true,
new Assignment("_node_id2", "test assignment")));
PersistentTasks tasks = new PersistentTasks(3L, taskMap);
PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(3L, taskMap);
ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
MetaData.Builder metaData = MetaData.builder();
RoutingTable.Builder routingTable = RoutingTable.builder();
addJobAndIndices(metaData, routingTable, "job_id1", "job_id2", "job_id3", "job_id4");
cs.nodes(nodes);
metaData.putCustom(PersistentTasks.TYPE, tasks);
metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
cs.metaData(metaData);
cs.routingTable(routingTable.build());
Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id4", cs.build(), 2, logger);
@ -160,14 +159,14 @@ public class OpenJobActionTests extends ESTestCase {
taskMap.put(id, createJobTask(id, "job_id" + id, nodeId, JobState.OPENED));
}
}
PersistentTasks tasks = new PersistentTasks(numNodes * maxRunningJobsPerNode, taskMap);
PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(numNodes * maxRunningJobsPerNode, taskMap);
ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
MetaData.Builder metaData = MetaData.builder();
RoutingTable.Builder routingTable = RoutingTable.builder();
addJobAndIndices(metaData, routingTable, "job_id1", "job_id2");
cs.nodes(nodes);
metaData.putCustom(PersistentTasks.TYPE, tasks);
metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
cs.metaData(metaData);
cs.routingTable(routingTable.build());
Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id2", cs.build(), 2, logger);
@ -187,14 +186,14 @@ public class OpenJobActionTests extends ESTestCase {
PersistentTask<OpenJobAction.Request> task =
new PersistentTask<>(1L, OpenJobAction.NAME, new OpenJobAction.Request("job_id1"), false, true,
new Assignment("_node_id1", "test assignment"));
PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task));
PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
MetaData.Builder metaData = MetaData.builder();
RoutingTable.Builder routingTable = RoutingTable.builder();
addJobAndIndices(metaData, routingTable, "job_id1", "job_id2");
cs.nodes(nodes);
metaData.putCustom(PersistentTasks.TYPE, tasks);
metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
cs.metaData(metaData);
cs.routingTable(routingTable.build());
Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id2", cs.build(), 2, logger);
@ -220,7 +219,7 @@ public class OpenJobActionTests extends ESTestCase {
taskMap.put(2L, createJobTask(2L, "job_id3", "_node_id2", JobState.OPENING));
taskMap.put(3L, createJobTask(3L, "job_id4", "_node_id2", JobState.OPENING));
taskMap.put(4L, createJobTask(4L, "job_id5", "_node_id3", JobState.OPENING));
PersistentTasks tasks = new PersistentTasks(5L, taskMap);
PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(5L, taskMap);
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
csBuilder.nodes(nodes);
@ -228,7 +227,7 @@ public class OpenJobActionTests extends ESTestCase {
RoutingTable.Builder routingTable = RoutingTable.builder();
addJobAndIndices(metaData, routingTable, "job_id1", "job_id2", "job_id3", "job_id4", "job_id5", "job_id6", "job_id7");
csBuilder.routingTable(routingTable.build());
metaData.putCustom(PersistentTasks.TYPE, tasks);
metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
csBuilder.metaData(metaData);
ClusterState cs = csBuilder.build();
@ -237,30 +236,30 @@ public class OpenJobActionTests extends ESTestCase {
PersistentTask<OpenJobAction.Request> lastTask = createJobTask(5L, "job_id6", "_node_id3", JobState.OPENING);
taskMap.put(5L, lastTask);
tasks = new PersistentTasks(6L, taskMap);
tasks = new PersistentTasksCustomMetaData(6L, taskMap);
csBuilder = ClusterState.builder(cs);
csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasks.TYPE, tasks));
csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
cs = csBuilder.build();
result = OpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, logger);
assertNull("no node selected, because OPENING state", result.getExecutorNode());
assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
taskMap.put(5L, new PersistentTask<>(lastTask, false, new Assignment("_node_id3", "test assignment")));
tasks = new PersistentTasks(6L, taskMap);
tasks = new PersistentTasksCustomMetaData(6L, taskMap);
csBuilder = ClusterState.builder(cs);
csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasks.TYPE, tasks));
csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
cs = csBuilder.build();
result = OpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, logger);
assertNull("no node selected, because stale task", result.getExecutorNode());
assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
taskMap.put(5L, new PersistentTask<>(lastTask, null));
tasks = new PersistentTasks(6L, taskMap);
tasks = new PersistentTasksCustomMetaData(6L, taskMap);
csBuilder = ClusterState.builder(cs);
csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasks.TYPE, tasks));
csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
cs = csBuilder.build();
result = OpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, logger);
assertNull("no node selected, because null state", result.getExecutorNode());

30
plugin/src/test/java/org/elasticsearch/xpack/ml/action/StartDatafeedActionTests.java
View File

@ -23,9 +23,9 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests;
import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.net.InetAddress;
import java.util.Collections;
@ -35,7 +35,7 @@ import java.util.Map;
import static org.elasticsearch.xpack.ml.action.OpenJobActionTests.createJobTask;
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createDatafeed;
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createScheduledJob;
import static org.elasticsearch.xpack.persistent.PersistentTasks.INITIAL_ASSIGNMENT;
import static org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT;
import static org.hamcrest.Matchers.equalTo;
public class StartDatafeedActionTests extends ESTestCase {
@ -48,7 +48,7 @@ public class StartDatafeedActionTests extends ESTestCase {
JobState jobState = randomFrom(JobState.FAILED, JobState.CLOSED, JobState.CLOSING, JobState.OPENING);
PersistentTask<OpenJobAction.Request> task = createJobTask(0L, job.getId(), "node_id", jobState);
PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task));
PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task));
DiscoveryNodes nodes = DiscoveryNodes.builder()
.add(new DiscoveryNode("node_name", "node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
@ -57,7 +57,7 @@ public class StartDatafeedActionTests extends ESTestCase {
ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build())
.putCustom(PersistentTasks.TYPE, tasks))
.putCustom(PersistentTasksCustomMetaData.TYPE, tasks))
.nodes(nodes);
Assignment result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build());
@ -66,10 +66,10 @@ public class StartDatafeedActionTests extends ESTestCase {
"] while state [opened] is required", result.getExplanation());
task = createJobTask(0L, job.getId(), "node_id", JobState.OPENED);
tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task));
tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task));
cs = ClusterState.builder(new ClusterName("cluster_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build())
.putCustom(PersistentTasks.TYPE, tasks))
.putCustom(PersistentTasksCustomMetaData.TYPE, tasks))
.nodes(nodes);
result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build());
assertEquals("node_id", result.getExecutorNode());
@ -83,7 +83,7 @@ public class StartDatafeedActionTests extends ESTestCase {
String nodeId = randomBoolean() ? "node_id2" : null;
PersistentTask<OpenJobAction.Request> task = createJobTask(0L, job.getId(), nodeId, JobState.OPENED);
PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task));
PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task));
DiscoveryNodes nodes = DiscoveryNodes.builder()
.add(new DiscoveryNode("node_name", "node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
@ -92,7 +92,7 @@ public class StartDatafeedActionTests extends ESTestCase {
ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build())
.putCustom(PersistentTasks.TYPE, tasks))
.putCustom(PersistentTasksCustomMetaData.TYPE, tasks))
.nodes(nodes);
Assignment result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build());
@ -101,10 +101,10 @@ public class StartDatafeedActionTests extends ESTestCase {
result.getExplanation());
task = createJobTask(0L, job.getId(), "node_id1", JobState.OPENED);
tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task));
tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task));
cs = ClusterState.builder(new ClusterName("cluster_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build())
.putCustom(PersistentTasks.TYPE, tasks))
.putCustom(PersistentTasksCustomMetaData.TYPE, tasks))
.nodes(nodes);
result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build());
assertEquals("node_id1", result.getExecutorNode());
@ -128,7 +128,7 @@ public class StartDatafeedActionTests extends ESTestCase {
PersistentTask<OpenJobAction.Request> task =
new PersistentTask<>(0L, OpenJobAction.NAME, new OpenJobAction.Request("job_id"), false, true,
INITIAL_ASSIGNMENT);
PersistentTasks tasks = new PersistentTasks(0L, Collections.singletonMap(0L, task));
PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(0L, Collections.singletonMap(0L, task));
DatafeedConfig datafeedConfig1 = DatafeedJobRunnerTests.createDatafeedConfig("foo-datafeed", "job_id").build();
MlMetadata mlMetadata2 = new MlMetadata.Builder(mlMetadata1)
.putDatafeed(datafeedConfig1)
@ -158,7 +158,7 @@ public class StartDatafeedActionTests extends ESTestCase {
Map<Long, PersistentTask<?>> taskMap = new HashMap<>();
taskMap.put(0L, jobTask);
taskMap.put(1L, datafeedTask);
PersistentTasks tasks = new PersistentTasks(2L, taskMap);
PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(2L, taskMap);
Exception e = expectThrows(ElasticsearchStatusException.class,
() -> StartDatafeedAction.validate("datafeed_id", mlMetadata1, tasks, nodes));
@ -186,7 +186,7 @@ public class StartDatafeedActionTests extends ESTestCase {
Map<Long, PersistentTask<?>> taskMap = new HashMap<>();
taskMap.put(0L, jobTask);
taskMap.put(1L, datafeedTask);
PersistentTasks tasks = new PersistentTasks(2L, taskMap);
PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(2L, taskMap);
StartDatafeedAction.validate("datafeed_id", mlMetadata1, tasks, nodes);
datafeedTask = new PersistentTask<>(0L, StartDatafeedAction.NAME, new StartDatafeedAction.Request("datafeed_id", 0L),

22
plugin/src/test/java/org/elasticsearch/xpack/ml/action/StopDatafeedActionRequestTests.java
View File

@ -15,9 +15,9 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase;
import org.elasticsearch.xpack.persistent.PersistentActionRequest;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTaskRequest;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.util.Collections;
@ -47,10 +47,10 @@ public class StopDatafeedActionRequestTests extends AbstractStreamableXContentTe
}
public void testValidate() {
PersistentTask<?> task = new PersistentTask<PersistentActionRequest>(1L, StartDatafeedAction.NAME,
new StartDatafeedAction.Request("foo", 0L), false, false, new PersistentTasks.Assignment("node_id", ""));
PersistentTask<?> task = new PersistentTask<PersistentTaskRequest>(1L, StartDatafeedAction.NAME,
new StartDatafeedAction.Request("foo", 0L), false, false, new PersistentTasksCustomMetaData.Assignment("node_id", ""));
task = new PersistentTask<>(task, DatafeedState.STARTED);
PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task));
PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
Job job = createDatafeedJob().build();
MlMetadata mlMetadata1 = new MlMetadata.Builder().putJob(job, false).build();
@ -66,14 +66,14 @@ public class StopDatafeedActionRequestTests extends AbstractStreamableXContentTe
}
public void testValidate_alreadyStopped() {
PersistentTasks tasks;
PersistentTasksCustomMetaData tasks;
if (randomBoolean()) {
PersistentTask<?> task = new PersistentTask<PersistentActionRequest>(1L, StartDatafeedAction.NAME,
new StartDatafeedAction.Request("foo", 0L), false, false, new PersistentTasks.Assignment("node_id", ""));
PersistentTask<?> task = new PersistentTask<PersistentTaskRequest>(1L, StartDatafeedAction.NAME,
new StartDatafeedAction.Request("foo", 0L), false, false, new PersistentTasksCustomMetaData.Assignment("node_id", ""));
task = new PersistentTask<>(task, DatafeedState.STOPPED);
tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task));
tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
} else {
tasks = randomBoolean() ? null : new PersistentTasks(0L, Collections.emptyMap());
tasks = randomBoolean() ? null : new PersistentTasksCustomMetaData(0L, Collections.emptyMap());
}
Job job = createDatafeedJob().build();

25
plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunnerTests.java
View File

@ -8,7 +8,6 @@ package org.elasticsearch.xpack.ml.datafeed;
import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
@ -41,10 +40,10 @@ import org.elasticsearch.xpack.ml.job.persistence.MockClientBuilder;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.notifications.AuditMessage;
import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction.Response;
import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.junit.Before;
import org.mockito.ArgumentCaptor;
@ -84,6 +83,7 @@ public class DatafeedJobRunnerTests extends ESTestCase {
private DatafeedJobRunner datafeedJobRunner;
private long currentTime = 120000;
private Auditor auditor;
private PersistentTasksService persistentTasksService;
@Before
@SuppressWarnings("unchecked")
@ -93,14 +93,14 @@ public class DatafeedJobRunnerTests extends ESTestCase {
mlMetadata.putJob(job, false);
mlMetadata.putDatafeed(createDatafeedConfig("datafeed_id", job.getId()).build());
PersistentTask<OpenJobAction.Request> task = createJobTask(0L, job.getId(), "node_id", JobState.OPENED);
PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task));
PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task));
DiscoveryNodes nodes = DiscoveryNodes.builder()
.add(new DiscoveryNode("node_name", "node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
Collections.emptyMap(), Collections.emptySet(), Version.CURRENT))
.build();
ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build())
.putCustom(PersistentTasks.TYPE, tasks))
.putCustom(PersistentTasksCustomMetaData.TYPE, tasks))
.nodes(nodes);
clusterService = mock(ClusterService.class);
@ -140,7 +140,9 @@ public class DatafeedJobRunnerTests extends ESTestCase {
when(client.execute(same(PostDataAction.INSTANCE), any())).thenReturn(jobDataFuture);
when(client.execute(same(FlushJobAction.INSTANCE), any())).thenReturn(flushJobFuture);
datafeedJobRunner = new DatafeedJobRunner(threadPool, client, clusterService, jobProvider, () -> currentTime, auditor) {
persistentTasksService = mock(PersistentTasksService.class);
datafeedJobRunner = new DatafeedJobRunner(threadPool, client, clusterService, jobProvider, () -> currentTime,
persistentTasksService, auditor) {
@Override
DataExtractorFactory createDataExtractorFactory(DatafeedConfig datafeedConfig, Job job) {
return dataExtractorFactory;
@ -153,13 +155,12 @@ public class DatafeedJobRunnerTests extends ESTestCase {
consumer.accept(new ResourceNotFoundException("dummy"));
return null;
}).when(jobProvider).bucketsViaInternalClient(any(), any(), any(), any());
doAnswer(invocationOnMock -> {
@SuppressWarnings("rawtypes")
ActionListener<Response> listener = (ActionListener<Response>) invocationOnMock.getArguments()[2];
listener.onResponse(new Response(true));
PersistentTaskOperationListener listener = (PersistentTaskOperationListener) invocationOnMock.getArguments()[2];
listener.onResponse(0L);
return null;
}).when(client).execute(same(UpdatePersistentTaskStatusAction.INSTANCE), any(), any());
}).when(persistentTasksService).updateStatus(anyLong(), any(), any());
}
public void testLookbackOnly_WarnsWhenNoDataIsRetrieved() throws Exception {

18
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
View File

@ -26,8 +26,8 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException;
import java.util.Collection;
@ -165,7 +165,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet();
assertBusy(() -> {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
PersistentTask task = tasks.taskMap().values().iterator().next();
DiscoveryNode node = clusterState.nodes().resolveNode(task.getExecutorNode());
@ -216,7 +216,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
// Sample each cs update and keep track each time a node holds more than `maxConcurrentJobAllocations` opening jobs.
List<String> violations = new CopyOnWriteArrayList<>();
internalCluster().clusterService(nonMlNode).addListener(event -> {
PersistentTasks tasks = event.state().metaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = event.state().metaData().custom(PersistentTasksCustomMetaData.TYPE);
if (tasks == null) {
return;
}
@ -247,7 +247,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
assertBusy(() -> {
ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(numJobs, tasks.taskMap().size());
for (PersistentTask<?> task : tasks.taskMap().values()) {
assertNotNull(task.getExecutorNode());
@ -271,7 +271,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
ensureStableCluster(1, nonMlNode);
assertBusy(() -> {
ClusterState state = client(nonMlNode).admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(numJobs, tasks.taskMap().size());
for (PersistentTask<?> task : tasks.taskMap().values()) {
assertNull(task.getExecutorNode());
@ -287,7 +287,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
ensureStableCluster(1 + numMlNodes);
assertBusy(() -> {
ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(numJobs, tasks.taskMap().size());
for (PersistentTask<?> task : tasks.taskMap().values()) {
assertNotNull(task.getExecutorNode());
@ -332,7 +332,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
client().execute(CloseJobAction.INSTANCE, closeJobRequest);
assertBusy(() -> {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(0, tasks.taskMap().size());
});
logger.info("Stop data node");
@ -357,7 +357,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
private void assertJobTask(String jobId, JobState expectedState, boolean hasExecutorNode) {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(1, tasks.taskMap().size());
PersistentTask<?> task = tasks.findTasks(OpenJobAction.NAME, p -> {
return p.getRequest() instanceof OpenJobAction.Request &&

6
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java
View File

@ -24,8 +24,8 @@ import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
@ -125,7 +125,7 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase {
disrupt.run();
assertBusy(() -> {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = clusterState.metaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertNotNull(tasks);
assertEquals(2, tasks.taskMap().size());
for (PersistentTask<?> task : tasks.tasks()) {

6
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java
View File

@ -17,7 +17,7 @@ import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
public class TooManyJobsIT extends BaseMlIntegTestCase {
@ -49,10 +49,10 @@ public class TooManyJobsIT extends BaseMlIntegTestCase {
client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request("2")).actionGet();
assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.CLOSED);
ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(1, tasks.taskMap().size());
// now just double check that the first job is still opened:
PersistentTasks.PersistentTask task = tasks.taskMap().values().iterator().next();
PersistentTasksCustomMetaData.PersistentTask task = tasks.taskMap().values().iterator().next();
assertEquals(JobState.OPENED, task.getStatus());
OpenJobAction.Request openJobRequest = (OpenJobAction.Request) task.getRequest();
assertEquals("1", openJobRequest.getJobId());

24
plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
View File

@ -34,7 +34,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles;
import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction;
import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.junit.Before;
import org.mockito.Mockito;
@ -122,16 +122,15 @@ public class AutodetectProcessManagerTests extends ESTestCase {
public void testOpenJob() {
Client client = mock(Client.class);
PersistentTasksService persistentTasksService = mock(PersistentTasksService.class);
AutodetectCommunicator communicator = mock(AutodetectCommunicator.class);
when(jobManager.getJobOrThrowIfUnknown("foo")).thenReturn(createJobDetails("foo"));
AutodetectProcessManager manager = createManager(communicator, client);
AutodetectProcessManager manager = createManager(communicator, client, persistentTasksService);
manager.openJob("foo", 1L, false, e -> {});
assertEquals(1, manager.numberOfOpenJobs());
assertTrue(manager.jobHasActiveAutodetectProcess("foo"));
UpdatePersistentTaskStatusAction.Request expectedRequest =
new UpdatePersistentTaskStatusAction.Request(1L, JobState.OPENED);
verify(client).execute(eq(UpdatePersistentTaskStatusAction.INSTANCE), eq(expectedRequest), any());
verify(persistentTasksService).updateStatus(eq(1L), eq(JobState.OPENED), any());
}
public void testOpenJob_exceedMaxNumJobs() {
@ -149,6 +148,7 @@ public class AutodetectProcessManagerTests extends ESTestCase {
when(jobManager.getJobOrThrowIfUnknown("foobar")).thenReturn(createJobDetails("foobar"));
Client client = mock(Client.class);
PersistentTasksService persistentTasksService = mock(PersistentTasksService.class);
ThreadPool threadPool = mock(ThreadPool.class);
ThreadPool.Cancellable cancellable = mock(ThreadPool.Cancellable.class);
when(threadPool.scheduleWithFixedDelay(any(), any(), any())).thenReturn(cancellable);
@ -161,7 +161,7 @@ public class AutodetectProcessManagerTests extends ESTestCase {
Settings.Builder settings = Settings.builder();
settings.put(AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE.getKey(), 3);
AutodetectProcessManager manager = spy(new AutodetectProcessManager(settings.build(), client, threadPool, jobManager, jobProvider,
jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory));
jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory, persistentTasksService));
DataCounts dataCounts = new DataCounts("foo");
ModelSnapshot modelSnapshot = new ModelSnapshot.Builder("foo").build();
@ -319,11 +319,11 @@ public class AutodetectProcessManagerTests extends ESTestCase {
handler.accept(new DataCounts(jobId));
return null;
}).when(jobProvider).dataCounts(eq("my_id"), any(), any());
PersistentTasksService persistentTasksService = mock(PersistentTasksService.class);
AutodetectProcess autodetectProcess = mock(AutodetectProcess.class);
AutodetectProcessFactory autodetectProcessFactory = (j, modelSnapshot, quantiles, filters, i, e) -> autodetectProcess;
AutodetectProcessManager manager = new AutodetectProcessManager(Settings.EMPTY, client, threadPool, jobManager, jobProvider,
jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory);
jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory, persistentTasksService);
expectThrows(EsRejectedExecutionException.class,
() -> manager.create("my_id", 1L, dataCounts, modelSnapshot, quantiles, filters, false, e -> {}));
@ -332,14 +332,16 @@ public class AutodetectProcessManagerTests extends ESTestCase {
private AutodetectProcessManager createManager(AutodetectCommunicator communicator) {
Client client = mock(Client.class);
return createManager(communicator, client);
PersistentTasksService persistentTasksService = mock(PersistentTasksService.class);
return createManager(communicator, client, persistentTasksService);
}
private AutodetectProcessManager createManager(AutodetectCommunicator communicator, Client client) {
private AutodetectProcessManager createManager(AutodetectCommunicator communicator, Client client,
PersistentTasksService persistentTasksService) {
ThreadPool threadPool = mock(ThreadPool.class);
AutodetectProcessFactory autodetectProcessFactory = mock(AutodetectProcessFactory.class);
AutodetectProcessManager manager = new AutodetectProcessManager(Settings.EMPTY, client, threadPool, jobManager, jobProvider,
jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory);
jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory, persistentTasksService);
manager = spy(manager);
doReturn(communicator).when(manager)
.create(any(), anyLong(), eq(dataCounts), eq(modelSnapshot), eq(quantiles), eq(filters), anyBoolean(), any());

50
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionRegistryTests.java
View File

@ -1,50 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.ThreadPool;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class PersistentActionRegistryTests extends ESTestCase {
public void testActionLookup() {
PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY);
TransportPersistentAction<?> action1 = mock(TransportPersistentAction.class);
when(action1.getExecutor()).thenReturn(ThreadPool.Names.MANAGEMENT);
TransportPersistentAction<?> action2 = mock(TransportPersistentAction.class);
when(action2.getExecutor()).thenReturn(ThreadPool.Names.GENERIC);
registry.registerPersistentAction("test1", action1);
registry.registerPersistentAction("test2", action2);
assertEquals(registry.getPersistentActionHolderSafe("test1").getAction(), "test1");
assertEquals(registry.getPersistentActionHolderSafe("test1").getExecutor(), ThreadPool.Names.MANAGEMENT);
assertEquals(registry.getPersistentActionHolderSafe("test1").getPersistentAction(), action1);
assertEquals(registry.getPersistentActionSafe("test1"), action1);
assertEquals(registry.getPersistentActionHolderSafe("test2").getAction(), "test2");
assertEquals(registry.getPersistentActionHolderSafe("test2").getExecutor(), ThreadPool.Names.GENERIC);
assertEquals(registry.getPersistentActionHolderSafe("test2").getPersistentAction(), action2);
assertEquals(registry.getPersistentActionSafe("test2"), action2);
try {
registry.getPersistentActionHolderSafe("test3");
fail("Should have failed");
} catch (IllegalStateException ex) {
assertEquals(ex.getMessage(), "Unknown persistent action [test3]");
}
try {
registry.getPersistentActionSafe("test3");
fail("Should have failed");
} catch (IllegalStateException ex) {
assertEquals(ex.getMessage(), "Unknown persistent action [test3]");
}
}
}

89
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterServiceTests.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksClusterServiceTests.java
View File

@ -18,9 +18,9 @@ import org.elasticsearch.cluster.routing.RoutingTable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest;
import java.util.ArrayList;
import java.util.Arrays;
@ -29,13 +29,13 @@ import java.util.HashSet;
import java.util.List;
import static java.util.Collections.emptyMap;
import static org.elasticsearch.xpack.persistent.TransportPersistentAction.NO_NODE_FOUND;
import static org.elasticsearch.xpack.persistent.PersistentTasksExecutor.NO_NODE_FOUND;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
public class PersistentTaskClusterServiceTests extends ESTestCase {
public class PersistentTasksClusterServiceTests extends ESTestCase {
public void testReassignmentRequired() {
int numberOfIterations = randomIntBetween(1, 30);
@ -50,10 +50,10 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
clusterState = insignificantChange(clusterState);
}
ClusterChangedEvent event = new ClusterChangedEvent("test", clusterState, previousState);
assertThat(dumpEvent(event), PersistentTaskClusterService.reassignmentRequired(event,
new PersistentTaskClusterService.ExecutorNodeDecider() {
assertThat(dumpEvent(event), PersistentTasksClusterService.reassignmentRequired(event,
new PersistentTasksClusterService.ExecutorNodeDecider() {
@Override
public <Request extends PersistentActionRequest> Assignment getAssignment(
public <Request extends PersistentTaskRequest> Assignment getAssignment(
String action, ClusterState currentState, Request request) {
if ("never_assign".equals(((TestRequest) request).getTestParam())) {
return NO_NODE_FOUND;
@ -66,14 +66,14 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
public void testReassignTasksWithNoTasks() {
ClusterState clusterState = initialState();
assertThat(reassign(clusterState).metaData().custom(PersistentTasks.TYPE), nullValue());
assertThat(reassign(clusterState).metaData().custom(PersistentTasksCustomMetaData.TYPE), nullValue());
}
public void testReassignConsidersClusterStateUpdates() {
ClusterState clusterState = initialState();
ClusterState.Builder builder = ClusterState.builder(clusterState);
PersistentTasks.Builder tasks = PersistentTasks.builder(
clusterState.metaData().custom(PersistentTasks.TYPE));
PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(
clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE));
DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes());
addTestNodes(nodes, randomIntBetween(1, 10));
int numberOfTasks = randomIntBetween(2, 40);
@ -81,11 +81,11 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
addTask(tasks, "should_assign", "assign_one", randomBoolean() ? null : "no_longer_exits", false);
}
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, tasks.build());
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build());
clusterState = builder.metaData(metaData).nodes(nodes).build();
ClusterState newClusterState = reassign(clusterState);
PersistentTasks tasksInProgress = newClusterState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasksInProgress, notNullValue());
}
@ -93,8 +93,8 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
public void testReassignTasks() {
ClusterState clusterState = initialState();
ClusterState.Builder builder = ClusterState.builder(clusterState);
PersistentTasks.Builder tasks = PersistentTasks.builder(
clusterState.metaData().custom(PersistentTasks.TYPE));
PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(
clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE));
DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes());
addTestNodes(nodes, randomIntBetween(1, 10));
int numberOfTasks = randomIntBetween(0, 40);
@ -118,11 +118,11 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
}
}
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, tasks.build());
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build());
clusterState = builder.metaData(metaData).nodes(nodes).build();
ClusterState newClusterState = reassign(clusterState);
PersistentTasks tasksInProgress = newClusterState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasksInProgress, notNullValue());
assertThat("number of tasks shouldn't change as a result or reassignment",
@ -133,15 +133,15 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
for (PersistentTask<?> task : tasksInProgress.tasks()) {
if (task.isStopped()) {
assertThat("stopped tasks should be never assigned", task.getExecutorNode(), nullValue());
assertThat(task.getAssignment().getExplanation(), equalTo("explanation: " + task.getAction()));
assertThat(task.getAssignment().getExplanation(), equalTo("explanation: " + task.getTaskName()));
} else {
// explanation should correspond to the action name
switch (task.getAction()) {
switch (task.getTaskName()) {
case "should_assign":
assertThat(task.getExecutorNode(), notNullValue());
assertThat(task.isAssigned(), equalTo(true));
if (clusterState.nodes().nodeExists(task.getExecutorNode()) == false) {
logger.info(clusterState.metaData().custom(PersistentTasks.TYPE).toString());
logger.info(clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE).toString());
}
assertThat("task should be assigned to a node that is in the cluster, was assigned to " + task.getExecutorNode(),
clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true));
@ -162,7 +162,7 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
}
break;
default:
fail("Unknown action " + task.getAction());
fail("Unknown action " + task.getTaskName());
}
}
}
@ -176,10 +176,10 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
}
private ClusterState reassign(ClusterState clusterState) {
return PersistentTaskClusterService.reassignTasks(clusterState, logger,
new PersistentTaskClusterService.ExecutorNodeDecider() {
return PersistentTasksClusterService.reassignTasks(clusterState, logger,
new PersistentTasksClusterService.ExecutorNodeDecider() {
@Override
public <Request extends PersistentActionRequest> Assignment getAssignment(
public <Request extends PersistentTaskRequest> Assignment getAssignment(
String action, ClusterState currentState, Request request) {
TestRequest testRequest = (TestRequest) request;
switch (testRequest.getTestParam()) {
@ -203,7 +203,7 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
private Assignment assignOnlyOneTaskAtATime(ClusterState clusterState) {
DiscoveryNodes nodes = clusterState.nodes();
PersistentTasks tasksInProgress = clusterState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasksInProgress = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
if (tasksInProgress.findTasks("assign_one",
task -> task.isStopped() == false && nodes.nodeExists(task.getExecutorNode())).isEmpty()) {
return randomNodeAssignment(clusterState.nodes());
@ -232,12 +232,12 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
return "nodes_changed: " + event.nodesChanged() +
" nodes_removed:" + event.nodesRemoved() +
" routing_table_changed:" + event.routingTableChanged() +
" tasks: " + event.state().metaData().custom(PersistentTasks.TYPE);
" tasks: " + event.state().metaData().custom(PersistentTasksCustomMetaData.TYPE);
}
private ClusterState significantChange(ClusterState clusterState) {
ClusterState.Builder builder = ClusterState.builder(clusterState);
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
if (tasks != null) {
if (randomBoolean()) {
for (PersistentTask<?> task : tasks.tasks()) {
@ -255,11 +255,12 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
// we don't have any unassigned tasks - add some
if (randomBoolean()) {
logger.info("added random task");
addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasks.builder(tasks), null, false);
addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks), null,
false);
tasksOrNodesChanged = true;
} else {
logger.info("added unassignable task with custom assignment message");
addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasks.builder(tasks),
addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks),
new Assignment(null, "change me"), "never_assign", false);
tasksOrNodesChanged = true;
}
@ -282,10 +283,10 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
return builder.build();
}
private PersistentTasks removeTasksWithChangingAssignment(PersistentTasks tasks) {
private PersistentTasksCustomMetaData removeTasksWithChangingAssignment(PersistentTasksCustomMetaData tasks) {
if (tasks != null) {
boolean changed = false;
PersistentTasks.Builder tasksBuilder = PersistentTasks.builder(tasks);
PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
for (PersistentTask<?> task : tasks.tasks()) {
// Remove all unassigned tasks that cause changing assignments they might trigger a significant change
if ("never_assign".equals(((TestRequest) task.getRequest()).getTestParam()) &&
@ -304,9 +305,9 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
private ClusterState insignificantChange(ClusterState clusterState) {
ClusterState.Builder builder = ClusterState.builder(clusterState);
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
tasks = removeTasksWithChangingAssignment(tasks);
PersistentTasks.Builder tasksBuilder = PersistentTasks.builder(tasks);
PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
if (randomBoolean()) {
if (hasAssignableTasks(tasks, clusterState.nodes()) == false) {
@ -328,7 +329,7 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
} else {
logger.info("changed routing table");
MetaData.Builder metaData = MetaData.builder(clusterState.metaData());
metaData.putCustom(PersistentTasks.TYPE, tasksBuilder.build());
metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build());
RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable());
changeRoutingTable(metaData, routingTable);
builder.metaData(metaData).routingTable(routingTable.build());
@ -350,12 +351,12 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
// clear the task
if (randomBoolean()) {
logger.info("removed all tasks");
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE,
PersistentTasks.builder().build());
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE,
PersistentTasksCustomMetaData.builder().build());
return builder.metaData(metaData).build();
} else {
logger.info("set task custom to null");
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).removeCustom(PersistentTasks.TYPE);
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).removeCustom(PersistentTasksCustomMetaData.TYPE);
return builder.metaData(metaData).build();
}
}
@ -374,11 +375,11 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
.numberOfReplicas(1)
.build();
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).put(indexMetaData, false)
.putCustom(PersistentTasks.TYPE, tasksBuilder.build());
.putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build());
return builder.metaData(metaData).build();
}
private boolean hasAssignableTasks(PersistentTasks tasks, DiscoveryNodes discoveryNodes) {
private boolean hasAssignableTasks(PersistentTasksCustomMetaData tasks, DiscoveryNodes discoveryNodes) {
if (tasks == null || tasks.tasks().isEmpty()) {
return false;
}
@ -393,26 +394,26 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
});
}
private boolean hasTasksAssignedTo(PersistentTasks tasks, String nodeId) {
private boolean hasTasksAssignedTo(PersistentTasksCustomMetaData tasks, String nodeId) {
return tasks != null && tasks.tasks().stream().anyMatch(
task -> nodeId.equals(task.getExecutorNode())) == false;
}
private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder,
MetaData.Builder metaData, PersistentTasks.Builder tasks,
MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks,
String node, boolean stopped) {
return addRandomTask(clusterStateBuilder, metaData, tasks, new Assignment(node, randomAsciiOfLength(10)),
randomAsciiOfLength(10), stopped);
}
private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder,
MetaData.Builder metaData, PersistentTasks.Builder tasks,
MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks,
Assignment assignment, String param, boolean stopped) {
return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasks.TYPE,
return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksCustomMetaData.TYPE,
tasks.addTask(randomAsciiOfLength(10), new TestRequest(param), stopped, randomBoolean(), assignment).build()));
}
private void addTask(PersistentTasks.Builder tasks, String action, String param, String node, boolean stopped) {
private void addTask(PersistentTasksCustomMetaData.Builder tasks, String action, String param, String node, boolean stopped) {
tasks.addTask(action, new TestRequest(param), stopped, randomBoolean(), new Assignment(node, "explanation: " + action));
}

54
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksTests.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksCustomMetaDataTests.java
View File

@ -21,12 +21,12 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.test.AbstractDiffableSerializationTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.Builder;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.Status;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestPersistentAction;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Builder;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.Status;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest;
import java.io.IOException;
import java.util.ArrayList;
@ -35,17 +35,17 @@ import java.util.Collections;
import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_GATEWAY;
import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_SNAPSHOT;
import static org.elasticsearch.xpack.persistent.TransportPersistentAction.NO_NODE_FOUND;
import static org.elasticsearch.xpack.persistent.PersistentTasksExecutor.NO_NODE_FOUND;
public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<Custom> {
public class PersistentTasksCustomMetaDataTests extends AbstractDiffableSerializationTestCase<Custom> {
@Override
protected PersistentTasks createTestInstance() {
protected PersistentTasksCustomMetaData createTestInstance() {
int numberOfTasks = randomInt(10);
PersistentTasks.Builder tasks = PersistentTasks.builder();
PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder();
for (int i = 0; i < numberOfTasks; i++) {
boolean stopped = randomBoolean();
tasks.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)),
tasks.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)),
stopped, randomBoolean(), stopped ? new Assignment(null, "stopped") : randomAssignment());
if (randomBoolean()) {
// From time to time update status
@ -57,22 +57,22 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
@Override
protected Writeable.Reader<Custom> instanceReader() {
return PersistentTasks::new;
return PersistentTasksCustomMetaData::new;
}
@Override
protected NamedWriteableRegistry getNamedWriteableRegistry() {
return new NamedWriteableRegistry(Arrays.asList(
new Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new),
new Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom),
new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new),
new Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::new),
new Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::readDiffFrom),
new Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new),
new Entry(Task.Status.class, Status.NAME, Status::new)
));
}
@Override
protected Custom makeTestChanges(Custom testInstance) {
PersistentTasks tasksInProgress = (PersistentTasks) testInstance;
PersistentTasksCustomMetaData tasksInProgress = (PersistentTasksCustomMetaData) testInstance;
Builder builder = new Builder();
switch (randomInt(3)) {
case 0:
@ -105,12 +105,12 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
@Override
protected Writeable.Reader<Diff<Custom>> diffReader() {
return PersistentTasks::readDiffFrom;
return PersistentTasksCustomMetaData::readDiffFrom;
}
@Override
protected PersistentTasks doParseInstance(XContentParser parser) throws IOException {
return PersistentTasks.fromXContent(parser);
protected PersistentTasksCustomMetaData doParseInstance(XContentParser parser) throws IOException {
return PersistentTasksCustomMetaData.fromXContent(parser);
}
@Override
@ -137,19 +137,19 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
private Builder addRandomTask(Builder builder) {
boolean stopped = randomBoolean();
builder.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(),
builder.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(),
stopped ? new Assignment(null, "stopped") : randomAssignment());
return builder;
}
private long pickRandomTask(PersistentTasks testInstance) {
private long pickRandomTask(PersistentTasksCustomMetaData testInstance) {
return randomFrom(new ArrayList<>(testInstance.tasks())).getId();
}
@Override
protected NamedXContentRegistry xContentRegistry() {
return new NamedXContentRegistry(Arrays.asList(
new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(TestPersistentAction.NAME),
new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(TestPersistentTasksExecutor.NAME),
TestRequest::fromXContent),
new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent)
));
@ -157,9 +157,9 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
@SuppressWarnings("unchecked")
public void testSerializationContext() throws Exception {
PersistentTasks testInstance = createTestInstance();
PersistentTasksCustomMetaData testInstance = createTestInstance();
for (int i = 0; i < randomInt(10); i++) {
testInstance = (PersistentTasks) makeTestChanges(testInstance);
testInstance = (PersistentTasksCustomMetaData) makeTestChanges(testInstance);
}
ToXContent.MapParams params = new ToXContent.MapParams(
@ -170,7 +170,7 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
XContentBuilder shuffled = shuffleXContent(builder);
XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled.bytes());
PersistentTasks newInstance = doParseInstance(parser);
PersistentTasksCustomMetaData newInstance = doParseInstance(parser);
assertNotSame(newInstance, testInstance);
assertEquals(testInstance.tasks().size(), newInstance.tasks().size());
@ -179,7 +179,7 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
assertNotNull(newTask);
// Things that should be serialized
assertEquals(testTask.getAction(), newTask.getAction());
assertEquals(testTask.getTaskName(), newTask.getTaskName());
assertEquals(testTask.getId(), newTask.getId());
assertEquals(testTask.getStatus(), newTask.getStatus());
assertEquals(testTask.getRequest(), newTask.getRequest());
@ -192,7 +192,7 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
}
public void testBuilder() {
PersistentTasks persistentTasks = null;
PersistentTasksCustomMetaData persistentTasks = null;
long lastKnownTask = -1;
for (int i = 0; i < randomIntBetween(10, 100); i++) {
final Builder builder;

68
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionFullRestartIT.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorFullRestartIT.java
View File

@ -8,14 +8,16 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.junit.annotations.TestLogging;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestPersistentAction;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest;
import org.elasticsearch.xpack.persistent.PersistentTasksExecutorIT.PersistentTaskOperationFuture;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
@ -23,10 +25,10 @@ import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, minNumDataNodes = 1)
public class PersistentActionFullRestartIT extends ESIntegTestCase {
public class PersistentTasksExecutorFullRestartIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Collections.singletonList(TestPersistentActionPlugin.class);
return Collections.singletonList(TestPersistentTasksPlugin.class);
}
@Override
@ -40,35 +42,38 @@ public class PersistentActionFullRestartIT extends ESIntegTestCase {
@TestLogging("org.elasticsearch.xpack.persistent:TRACE,org.elasticsearch.cluster.service:DEBUG")
public void testFullClusterRestart() throws Exception {
PersistentTasksService service = internalCluster().getInstance(PersistentTasksService.class);
int numberOfTasks = randomIntBetween(1, 10);
long[] taskIds = new long[numberOfTasks];
List<PersistentTaskOperationFuture> futures = new ArrayList<>(numberOfTasks);
boolean[] stopped = new boolean[numberOfTasks];
int runningTasks = 0;
for (int i = 0; i < numberOfTasks; i++) {
if (randomBoolean()) {
stopped[i] = randomBoolean();
if (stopped[i] == false) {
runningTasks++;
taskIds[i] = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId();
stopped[i] = false;
} else {
taskIds[i] = CreatePersistentTaskAction.INSTANCE.newRequestBuilder(client())
.setAction(TestPersistentAction.NAME)
.setRequest(new TestRequest("Blah"))
.setStopped(true)
.get().getTaskId();
stopped[i] = true;
}
PersistentTaskOperationFuture future = new PersistentTaskOperationFuture();
futures.add(future);
service.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), stopped[i], true, future);
}
for (int i = 0; i < numberOfTasks; i++) {
taskIds[i] = futures.get(i).get();
}
final int numberOfRunningTasks = runningTasks;
PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks));
if (numberOfRunningTasks > 0) {
// Make sure that at least one of the tasks is running
assertBusy(() -> {
// Wait for the task to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get()
.getTasks().size(), greaterThan(0));
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.getTasks().size(), greaterThan(0));
});
}
@ -76,7 +81,7 @@ public class PersistentActionFullRestartIT extends ESIntegTestCase {
internalCluster().fullRestart();
ensureYellow();
tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasks.TYPE);
tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks));
// Check that cluster state is correct
for (int i = 0; i < numberOfTasks; i++) {
@ -88,12 +93,13 @@ public class PersistentActionFullRestartIT extends ESIntegTestCase {
logger.info("Waiting for {} original tasks to start", numberOfRunningTasks);
assertBusy(() -> {
// Wait for the running task to start automatically
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(),
equalTo(numberOfRunningTasks));
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.getTasks().size(), equalTo(numberOfRunningTasks));
});
// Start all other tasks
tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasks.TYPE);
tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
service = internalCluster().getInstance(PersistentTasksService.class);
for (int i = 0; i < numberOfTasks; i++) {
PersistentTask<?> task = tasksInProgress.getTask(taskIds[i]);
assertNotNull(task);
@ -101,26 +107,28 @@ public class PersistentActionFullRestartIT extends ESIntegTestCase {
assertThat(task.isStopped(), equalTo(stopped[i]));
assertThat(task.getExecutorNode(), stopped[i] ? nullValue() : notNullValue());
if (stopped[i]) {
assertAcked(StartPersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(task.getId()).get());
PersistentTaskOperationFuture startFuture = new PersistentTaskOperationFuture();
service.startTask(task.getId(), startFuture);
assertEquals(startFuture.get(), (Long) task.getId());
}
}
logger.info("Waiting for {} tasks to start", numberOfTasks);
assertBusy(() -> {
// Wait for all tasks to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(),
equalTo(numberOfTasks));
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.getTasks().size(), equalTo(numberOfTasks));
});
logger.info("Complete all tasks");
// Complete the running task and make sure it finishes properly
assertThat(new TestPersistentActionPlugin.TestTasksRequestBuilder(client()).setOperation("finish").get().getTasks().size(),
assertThat(new TestPersistentTasksPlugin.TestTasksRequestBuilder(client()).setOperation("finish").get().getTasks().size(),
equalTo(numberOfTasks));
assertBusy(() -> {
// Make sure the task is removed from the cluster state
assertThat(((PersistentTasks) internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE)).tasks(), empty());
assertThat(((PersistentTasksCustomMetaData) internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasksCustomMetaData.TYPE)).tasks(), empty());
});
}

141
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionIT.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorIT.java
View File

@ -6,12 +6,15 @@
package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.BaseFuture;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.tasks.TaskId;
import org.elasticsearch.tasks.TaskInfo;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestPersistentAction;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestTasksRequestBuilder;
import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestTasksRequestBuilder;
import org.junit.After;
import java.util.Collection;
@ -26,11 +29,11 @@ import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, minNumDataNodes = 2)
public class PersistentActionIT extends ESIntegTestCase {
public class PersistentTasksExecutorIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Collections.singletonList(TestPersistentActionPlugin.class);
return Collections.singletonList(TestPersistentTasksPlugin.class);
}
@Override
@ -47,14 +50,30 @@ public class PersistentActionIT extends ESIntegTestCase {
assertNoRunningTasks();
}
public static class PersistentTaskOperationFuture extends BaseFuture<Long> implements PersistentTaskOperationListener {
@Override
public void onResponse(long taskId) {
set(taskId);
}
@Override
public void onFailure(Exception e) {
setException(e);
}
}
public void testPersistentActionRestart() throws Exception {
long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId();
PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class);
PersistentTaskOperationFuture future = new PersistentTaskOperationFuture();
persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future);
long taskId = future.get();
assertBusy(() -> {
// Wait for the task to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(),
equalTo(1));
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.getTasks().size(), equalTo(1));
});
TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]")
TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]")
.get().getTasks().get(0);
logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId());
// Verifying parent
@ -68,7 +87,7 @@ public class PersistentActionIT extends ESIntegTestCase {
assertBusy(() -> {
// Wait for the task to restart
List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get()
List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.getTasks();
logger.info("Found {} tasks", tasks.size());
assertThat(tasks.size(), equalTo(1));
@ -78,24 +97,29 @@ public class PersistentActionIT extends ESIntegTestCase {
logger.info("Removing persistent task with id {}", firstRunningTask.getId());
// Remove the persistent task
assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get());
PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture();
persistentTasksService.removeTask(taskId, removeFuture);
assertEquals(removeFuture.get(), (Long) taskId);
logger.info("Waiting for persistent task with id {} to disappear", firstRunningTask.getId());
assertBusy(() -> {
// Wait for the task to disappear completely
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(),
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(),
empty());
});
}
public void testPersistentActionCompletion() throws Exception {
long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId();
PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class);
PersistentTaskOperationFuture future = new PersistentTaskOperationFuture();
persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future);
long taskId = future.get();
assertBusy(() -> {
// Wait for the task to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(),
equalTo(1));
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.getTasks().size(), equalTo(1));
});
TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]")
TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]")
.get().getTasks().get(0);
logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId());
// Verifying parent
@ -106,14 +130,14 @@ public class PersistentActionIT extends ESIntegTestCase {
public void testPersistentActionCompletionWithoutRemoval() throws Exception {
boolean stopped = randomBoolean();
long taskId = CreatePersistentTaskAction.INSTANCE.newRequestBuilder(client())
.setAction(TestPersistentAction.NAME)
.setRequest(new TestPersistentActionPlugin.TestRequest("Blah"))
.setRemoveOnCompletion(false)
.setStopped(stopped).get().getTaskId();
PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class);
PersistentTaskOperationFuture future = new PersistentTaskOperationFuture();
persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), stopped, false,
future);
long taskId = future.get();
PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasksInProgress.tasks().size(), equalTo(1));
assertThat(tasksInProgress.getTask(taskId).isStopped(), equalTo(stopped));
assertThat(tasksInProgress.getTask(taskId).getExecutorNode(), stopped ? nullValue() : notNullValue());
@ -124,24 +148,26 @@ public class PersistentActionIT extends ESIntegTestCase {
for (int i = 0; i < numberOfIters; i++) {
logger.info("iteration {}", i);
if (stopped) {
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(),
empty());
assertAcked(StartPersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get());
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.getTasks(), empty());
PersistentTaskOperationFuture startFuture = new PersistentTaskOperationFuture();
persistentTasksService.startTask(taskId, startFuture);
assertEquals(startFuture.get(), (Long) taskId);
}
assertBusy(() -> {
// Wait for the task to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks()
.size(), equalTo(1));
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.getTasks().size(), equalTo(1));
});
TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]")
TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]")
.get().getTasks().get(0);
stopOrCancelTask(firstRunningTask.getTaskId());
assertBusy(() -> {
// Wait for the task to finish
List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get()
.getTasks();
List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]")
.get().getTasks();
logger.info("Found {} tasks", tasks.size());
assertThat(tasks.size(), equalTo(0));
});
@ -150,30 +176,36 @@ public class PersistentActionIT extends ESIntegTestCase {
assertBusy(() -> {
// Wait for the task to be marked as stopped
PersistentTasks tasks = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasks.tasks().size(), equalTo(1));
assertThat(tasks.getTask(taskId).isStopped(), equalTo(true));
assertThat(tasks.getTask(taskId).shouldRemoveOnCompletion(), equalTo(false));
});
logger.info("Removing action record from cluster state");
assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get());
PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture();
persistentTasksService.removeTask(taskId, removeFuture);
assertEquals(removeFuture.get(), (Long) taskId);
}
public void testPersistentActionWithNoAvailableNode() throws Exception {
long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah")
.executorNodeAttr("test").get().getTaskId();
PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class);
PersistentTaskOperationFuture future = new PersistentTaskOperationFuture();
TestRequest testRequest = new TestRequest("Blah");
testRequest.setExecutorNodeAttr("test");
persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, testRequest, future);
long taskId = future.get();
Settings nodeSettings = Settings.builder().put(nodeSettings(0)).put("node.attr.test_attr", "test").build();
String newNode = internalCluster().startNode(nodeSettings);
String newNodeId = internalCluster().clusterService(newNode).localNode().getId();
assertBusy(() -> {
// Wait for the task to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(),
equalTo(1));
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks()
.size(), equalTo(1));
});
TaskInfo taskInfo = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]")
TaskInfo taskInfo = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]")
.get().getTasks().get(0);
// Verifying the the task runs on the new node
@ -183,27 +215,32 @@ public class PersistentActionIT extends ESIntegTestCase {
assertBusy(() -> {
// Wait for the task to disappear completely
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(),
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(),
empty());
});
// Remove the persistent task
assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get());
PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture();
persistentTasksService.removeTask(taskId, removeFuture);
assertEquals(removeFuture.get(), (Long) taskId);
}
public void testPersistentActionStatusUpdate() throws Exception {
TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get();
PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class);
PersistentTaskOperationFuture future = new PersistentTaskOperationFuture();
persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future);
future.get();
assertBusy(() -> {
// Wait for the task to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(),
equalTo(1));
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks()
.size(), equalTo(1));
});
TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]")
TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]")
.get().getTasks().get(0);
PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasksInProgress.tasks().size(), equalTo(1));
assertThat(tasksInProgress.tasks().iterator().next().getStatus(), nullValue());
@ -216,8 +253,8 @@ public class PersistentActionIT extends ESIntegTestCase {
int finalI = i;
assertBusy(() -> {
PersistentTasks tasks = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE);
PersistentTasksCustomMetaData tasks = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasks.tasks().size(), equalTo(1));
assertThat(tasks.tasks().iterator().next().getStatus(), notNullValue());
assertThat(tasks.tasks().iterator().next().getStatus().toString(), equalTo("{\"phase\":\"phase " + (finalI + 1) + "\"}"));
@ -252,14 +289,14 @@ public class PersistentActionIT extends ESIntegTestCase {
private void assertNoRunningTasks() throws Exception {
assertBusy(() -> {
// Wait for the task to finish
List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get()
List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.getTasks();
logger.info("Found {} tasks", tasks.size());
assertThat(tasks.size(), equalTo(0));
// Make sure the task is removed from the cluster state
assertThat(((PersistentTasks) internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE)).tasks(), empty());
assertThat(((PersistentTasksCustomMetaData) internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasksCustomMetaData.TYPE)).tasks(), empty());
});
}

10
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionResponseTests.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorResponseTests.java
View File

@ -7,15 +7,15 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.test.AbstractStreamableTestCase;
public class PersistentActionResponseTests extends AbstractStreamableTestCase<PersistentActionResponse> {
public class PersistentTasksExecutorResponseTests extends AbstractStreamableTestCase<PersistentTaskResponse> {
@Override
protected PersistentActionResponse createTestInstance() {
return new PersistentActionResponse(randomLong());
protected PersistentTaskResponse createTestInstance() {
return new PersistentTaskResponse(randomLong());
}
@Override
protected PersistentActionResponse createBlankInstance() {
return new PersistentActionResponse();
protected PersistentTaskResponse createBlankInstance() {
return new PersistentTaskResponse();
}
}

6
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinatorStatusTests.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeServiceStatusTests.java
View File

@ -7,12 +7,12 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.test.AbstractWireSerializingTestCase;
import org.elasticsearch.xpack.persistent.PersistentActionCoordinator.State;
import org.elasticsearch.xpack.persistent.PersistentActionCoordinator.Status;
import org.elasticsearch.xpack.persistent.PersistentTasksNodeService.State;
import org.elasticsearch.xpack.persistent.PersistentTasksNodeService.Status;
import static org.hamcrest.Matchers.containsString;
public class PersistentActionCoordinatorStatusTests extends AbstractWireSerializingTestCase<Status> {
public class PersistentTasksNodeServiceStatusTests extends AbstractWireSerializingTestCase<Status> {
@Override
protected Status createTestInstance() {

204
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinatorTests.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeServiceTests.java
View File

@ -7,7 +7,6 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.MetaData;
@ -22,12 +21,13 @@ import org.elasticsearch.tasks.TaskManager;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.xpack.persistent.CompletionPersistentTaskAction.Response;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest;
import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
@ -37,7 +37,7 @@ import static org.hamcrest.core.IsEqual.equalTo;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class PersistentActionCoordinatorTests extends ESTestCase {
public class PersistentTasksNodeServiceTests extends ESTestCase {
private ClusterService createClusterService() {
ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
@ -58,21 +58,21 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
public void testStartTask() throws Exception {
ClusterService clusterService = createClusterService();
PersistentActionService persistentActionService = mock(PersistentActionService.class);
PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY);
@SuppressWarnings("unchecked") TransportPersistentAction<TestRequest> action = mock(TransportPersistentAction.class);
PersistentTasksService persistentTasksService = mock(PersistentTasksService.class);
@SuppressWarnings("unchecked") PersistentTasksExecutor<TestRequest> action = mock(PersistentTasksExecutor.class);
when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME);
registry.registerPersistentAction("test", action);
when(action.getTaskName()).thenReturn("test");
PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action));
int nonLocalNodesCount = randomInt(10);
MockExecutor executor = new MockExecutor();
PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService,
registry, new TaskManager(Settings.EMPTY), executor);
PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService,
registry, new TaskManager(Settings.EMPTY), mock(ThreadPool.class), executor);
ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY))
.build();
PersistentTasks.Builder tasks = PersistentTasks.builder();
PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder();
boolean added = false;
if (nonLocalNodesCount > 0) {
for (int i = 0; i < randomInt(5); i++) {
@ -91,7 +91,7 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
}
MetaData.Builder metaData = MetaData.builder(state.metaData());
metaData.putCustom(PersistentTasks.TYPE, tasks.build());
metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build());
ClusterState newClusterState = ClusterState.builder(state).metaData(metaData).build();
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
@ -149,24 +149,24 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
public void testTaskCancellation() {
ClusterService clusterService = createClusterService();
AtomicLong capturedTaskId = new AtomicLong();
AtomicReference<ActionListener<CancelTasksResponse>> capturedListener = new AtomicReference<>();
PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, null, null, null) {
AtomicReference<PersistentTaskOperationListener> capturedListener = new AtomicReference<>();
PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, null, null) {
@Override
public void sendCancellation(long taskId, ActionListener<CancelTasksResponse> listener) {
public void sendCancellation(long taskId, PersistentTaskOperationListener listener) {
capturedTaskId.set(taskId);
capturedListener.set(listener);
}
};
PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY);
@SuppressWarnings("unchecked") TransportPersistentAction<TestRequest> action = mock(TransportPersistentAction.class);
@SuppressWarnings("unchecked") PersistentTasksExecutor<TestRequest> action = mock(PersistentTasksExecutor.class);
when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME);
registry.registerPersistentAction("test", action);
when(action.getTaskName()).thenReturn("test");
PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action));
int nonLocalNodesCount = randomInt(10);
MockExecutor executor = new MockExecutor();
TaskManager taskManager = new TaskManager(Settings.EMPTY);
PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService,
registry, taskManager, executor);
PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService,
registry, taskManager, mock(ThreadPool.class), executor);
ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY))
.build();
@ -203,7 +203,7 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
// That should trigger cancellation request
assertThat(capturedTaskId.get(), equalTo(localId));
// Notify successful cancellation
capturedListener.get().onResponse(new CancelTasksResponse());
capturedListener.get().onResponse(localId);
// finish or fail task
if (randomBoolean()) {
@ -217,106 +217,118 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
}
public void testNotificationFailure() {
ClusterService clusterService = createClusterService();
AtomicLong capturedTaskId = new AtomicLong(-1L);
AtomicReference<Exception> capturedException = new AtomicReference<>();
AtomicReference<ActionListener<Response>> capturedListener = new AtomicReference<>();
PersistentActionService persistentActionService =
new PersistentActionService(Settings.EMPTY, mock(ThreadPool.class), clusterService, null) {
@Override
public void sendCompletionNotification(long taskId, Exception failure, ActionListener<Response> listener) {
capturedTaskId.set(taskId);
capturedException.set(failure);
capturedListener.set(listener);
}
};
PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY);
@SuppressWarnings("unchecked") TransportPersistentAction<TestRequest> action = mock(TransportPersistentAction.class);
when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME);
registry.registerPersistentAction("test", action);
int nonLocalNodesCount = randomInt(10);
MockExecutor executor = new MockExecutor();
TaskManager taskManager = new TaskManager(Settings.EMPTY);
PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService,
registry, taskManager, executor);
ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY))
public void testNotificationFailure() throws Exception {
Settings settings = Settings.builder()
.put("node.name", PersistentTasksNodeServiceTests.class.getSimpleName())
.build();
ThreadPool threadPool = new ThreadPool(settings);
try {
ClusterService clusterService = createClusterService();
AtomicLong capturedTaskId = new AtomicLong(-1L);
AtomicReference<Exception> capturedException = new AtomicReference<>();
AtomicReference<PersistentTaskOperationListener> capturedListener = new AtomicReference<>();
PersistentTasksService persistentTasksService =
new PersistentTasksService(Settings.EMPTY, clusterService, null) {
@Override
public void sendCompletionNotification(long taskId, Exception failure, PersistentTaskOperationListener listener) {
capturedTaskId.set(taskId);
capturedException.set(failure);
capturedListener.set(listener);
}
};
@SuppressWarnings("unchecked") PersistentTasksExecutor<TestRequest> action = mock(PersistentTasksExecutor.class);
when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME);
when(action.getTaskName()).thenReturn("test");
PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY,
Collections.singletonList(action));
ClusterState newClusterState = state;
// Allocate first task
state = newClusterState;
newClusterState = addTask(state, "test", new TestRequest(), "this_node");
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
int nonLocalNodesCount = randomInt(10);
MockExecutor executor = new MockExecutor();
TaskManager taskManager = new TaskManager(Settings.EMPTY);
PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService,
registry, taskManager, threadPool, executor);
// Fail the task
executor.get(0).listener.onFailure(new RuntimeException("test failure"));
ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY))
.build();
// Check that notification was sent
assertThat(capturedException.get().getMessage(), equalTo("test failure"));
capturedException.set(null);
ClusterState newClusterState = state;
// Allocate first task
state = newClusterState;
newClusterState = addTask(state, "test", new TestRequest(), "this_node");
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
// Simulate failure to notify
capturedListener.get().onFailure(new IOException("simulated notification failure"));
// Fail the task
executor.get(0).listener.onFailure(new RuntimeException("test failure"));
// Allocate another task
state = newClusterState;
newClusterState = addTask(state, "test", new TestRequest(), "other_node");
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
// Check that notification was sent
assertThat(capturedException.get().getMessage(), equalTo("test failure"));
capturedException.set(null);
// Check that notification was sent again
assertThat(capturedException.get().getMessage(), equalTo("test failure"));
// Simulate failure to notify
capturedListener.get().onFailure(new IOException("simulated notification failure"));
// Check the the task is still known by the task manager
assertThat(taskManager.getTasks().size(), equalTo(1));
long id = taskManager.getTasks().values().iterator().next().getParentTaskId().getId();
// Allocate another task
state = newClusterState;
newClusterState = addTask(state, "test", new TestRequest(), "other_node");
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
// This time acknowledge notification
capturedListener.get().onResponse(new Response());
// Check that notification was sent again
assertBusy(() -> {
assertNotNull(capturedException.get());
assertThat(capturedException.get().getMessage(), equalTo("test failure"));
});
// Reallocate failed task to another node
state = newClusterState;
newClusterState = reallocateTask(state, id, "other_node");
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
// Check the the task is still known by the task manager
assertThat(taskManager.getTasks().size(), equalTo(1));
long id = taskManager.getTasks().values().iterator().next().getParentTaskId().getId();
// Check the the task is now removed from task manager
assertThat(taskManager.getTasks().values(), empty());
// This time acknowledge notification
capturedListener.get().onResponse(id);
// Reallocate failed task to another node
state = newClusterState;
newClusterState = reallocateTask(state, id, "other_node");
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
// Check the the task is now removed from task manager
assertThat(taskManager.getTasks().values(), empty());
} finally {
assertTrue(ESTestCase.terminate(threadPool));
}
}
private <Request extends PersistentActionRequest> ClusterState addTask(ClusterState state, String action, Request request,
String node) {
PersistentTasks.Builder builder =
PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE));
return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE,
private <Request extends PersistentTaskRequest> ClusterState addTask(ClusterState state, String action, Request request,
String node) {
PersistentTasksCustomMetaData.Builder builder =
PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE));
return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE,
builder.addTask(action, request, false, true, new Assignment(node, "test assignment")).build())).build();
}
private ClusterState reallocateTask(ClusterState state, long taskId, String node) {
PersistentTasks.Builder builder =
PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE));
PersistentTasksCustomMetaData.Builder builder =
PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE));
assertTrue(builder.hasTask(taskId));
return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE,
return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE,
builder.reassignTask(taskId, new Assignment(node, "test assignment")).build())).build();
}
private ClusterState removeTask(ClusterState state, long taskId) {
PersistentTasks.Builder builder =
PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE));
PersistentTasksCustomMetaData.Builder builder =
PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE));
assertTrue(builder.hasTask(taskId));
return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE,
return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE,
builder.removeTask(taskId).build())).build();
}
private class Execution {
private final PersistentActionRequest request;
private final PersistentTaskRequest request;
private final NodePersistentTask task;
private final PersistentActionRegistry.PersistentActionHolder<?> holder;
private final PersistentTasksExecutor<?> holder;
private final ActionListener<Empty> listener;
Execution(PersistentActionRequest request, NodePersistentTask task, PersistentActionRegistry.PersistentActionHolder<?> holder,
Execution(PersistentTaskRequest request, NodePersistentTask task, PersistentTasksExecutor<?> holder,
ActionListener<Empty> listener) {
this.request = request;
this.task = task;
@ -325,7 +337,7 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
}
}
private class MockExecutor extends PersistentActionExecutor {
private class MockExecutor extends NodePersistentTasksExecutor {
private List<Execution> executions = new ArrayList<>();
MockExecutor() {
@ -333,10 +345,10 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
}
@Override
public <Request extends PersistentActionRequest> void executeAction(Request request, NodePersistentTask task,
PersistentActionRegistry.PersistentActionHolder<Request> holder,
ActionListener<Empty> listener) {
executions.add(new Execution(request, task, holder, listener));
public <Request extends PersistentTaskRequest> void executeTask(Request request, NodePersistentTask task,
PersistentTasksExecutor<Request> action,
ActionListener<Empty> listener) {
executions.add(new Execution(request, task, action, listener));
}
public Execution get(int i) {

6
plugin/src/test/java/org/elasticsearch/xpack/persistent/StartPersistentActionRequestTests.java
View File

@ -8,8 +8,8 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
import org.elasticsearch.xpack.persistent.CreatePersistentTaskAction.Request;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestPersistentAction;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest;
import org.elasticsearch.test.AbstractStreamableTestCase;
import java.util.Collections;
@ -39,7 +39,7 @@ public class StartPersistentActionRequestTests extends AbstractStreamableTestCas
@Override
protected NamedWriteableRegistry getNamedWriteableRegistry() {
return new NamedWriteableRegistry(Collections.singletonList(
new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new)
new Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new)
));
}
}

106
plugin/src/test/java/org/elasticsearch/xpack/persistent/TestPersistentActionPlugin.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/TestPersistentTasksPlugin.java
View File

@ -8,7 +8,6 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.action.Action;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.FailedNodeException;
@ -49,7 +48,7 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.watcher.ResourceWatcherService;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import java.io.IOException;
import java.util.ArrayList;
@ -72,12 +71,11 @@ import static org.junit.Assert.fail;
/**
* A plugin that adds a test persistent task.
*/
public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin {
@Override
public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
return Arrays.asList(
new ActionHandler<>(TestPersistentAction.INSTANCE, TransportTestPersistentAction.class),
new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class),
new ActionHandler<>(CreatePersistentTaskAction.INSTANCE, CreatePersistentTaskAction.TransportAction.class),
new ActionHandler<>(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class),
@ -91,24 +89,28 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
ResourceWatcherService resourceWatcherService, ScriptService scriptService,
NamedXContentRegistry xContentRegistry) {
PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, threadPool, clusterService, client);
PersistentActionRegistry persistentActionRegistry = new PersistentActionRegistry(Settings.EMPTY);
PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, client);
TestPersistentTasksExecutor testPersistentAction = new TestPersistentTasksExecutor(Settings.EMPTY, persistentTasksService,
clusterService);
PersistentTasksExecutorRegistry persistentTasksExecutorRegistry = new PersistentTasksExecutorRegistry(Settings.EMPTY,
Collections.singletonList(testPersistentAction));
return Arrays.asList(
persistentActionService,
persistentActionRegistry,
new PersistentTaskClusterService(Settings.EMPTY, persistentActionRegistry, clusterService)
persistentTasksService,
persistentTasksExecutorRegistry,
new PersistentTasksClusterService(Settings.EMPTY, persistentTasksExecutorRegistry, clusterService)
);
}
@Override
public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
return Arrays.asList(
new NamedWriteableRegistry.Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new),
new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new),
new NamedWriteableRegistry.Entry(Task.Status.class,
PersistentActionCoordinator.Status.NAME, PersistentActionCoordinator.Status::new),
new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new),
new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom),
PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new),
new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE,
PersistentTasksCustomMetaData::new),
new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE,
PersistentTasksCustomMetaData::readDiffFrom),
new NamedWriteableRegistry.Entry(Task.Status.class, Status.NAME, Status::new)
);
}
@ -116,18 +118,18 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
@Override
public List<NamedXContentRegistry.Entry> getNamedXContent() {
return Arrays.asList(
new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasks.TYPE),
PersistentTasks::fromXContent),
new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(TestPersistentAction.NAME),
new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksCustomMetaData.TYPE),
PersistentTasksCustomMetaData::fromXContent),
new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(TestPersistentTasksExecutor.NAME),
TestRequest::fromXContent),
new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent)
);
}
public static class TestRequest extends PersistentActionRequest {
public static class TestRequest extends PersistentTaskRequest {
public static final ConstructingObjectParser<TestRequest, Void> REQUEST_PARSER =
new ConstructingObjectParser<>(TestPersistentAction.NAME, args -> new TestRequest((String) args[0]));
new ConstructingObjectParser<>(TestPersistentTasksExecutor.NAME, args -> new TestRequest((String) args[0]));
static {
REQUEST_PARSER.declareString(constructorArg(), new ParseField("param"));
@ -158,7 +160,7 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
@Override
public String getWriteableName() {
return TestPersistentAction.NAME;
return TestPersistentTasksExecutor.NAME;
}
public void setExecutorNodeAttr(String executorNodeAttr) {
@ -226,53 +228,13 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
}
}
public static class TestPersistentTaskRequestBuilder extends
ActionRequestBuilder<TestRequest, PersistentActionResponse, TestPersistentTaskRequestBuilder> {
protected TestPersistentTaskRequestBuilder(ElasticsearchClient client, Action<TestRequest, PersistentActionResponse,
TestPersistentTaskRequestBuilder> action, TestRequest request) {
super(client, action, request);
}
public TestPersistentTaskRequestBuilder testParam(String testParam) {
request.setTestParam(testParam);
return this;
}
public TestPersistentTaskRequestBuilder executorNodeAttr(String targetNode) {
request.setExecutorNodeAttr(targetNode);
return this;
}
}
public static class TestPersistentAction extends Action<TestRequest, PersistentActionResponse, TestPersistentTaskRequestBuilder> {
public static final TestPersistentAction INSTANCE = new TestPersistentAction();
public static final String NAME = "cluster:admin/persistent/test";
private TestPersistentAction() {
super(NAME);
}
@Override
public PersistentActionResponse newResponse() {
return new PersistentActionResponse();
}
@Override
public TestPersistentTaskRequestBuilder newRequestBuilder(ElasticsearchClient client) {
return new TestPersistentTaskRequestBuilder(client, this, new TestRequest());
}
}
public static class Status implements Task.Status {
public static final String NAME = "test";
private final String phase;
public static final ConstructingObjectParser<Status, Void> STATUS_PARSER =
new ConstructingObjectParser<>(TestPersistentAction.NAME, args -> new Status((String) args[0]));
new ConstructingObjectParser<>(TestPersistentTasksExecutor.NAME, args -> new Status((String) args[0]));
static {
STATUS_PARSER.declareString(constructorArg(), new ParseField("phase"));
@ -336,19 +298,15 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
}
public static class TransportTestPersistentAction extends TransportPersistentAction<TestRequest> {
public static class TestPersistentTasksExecutor extends PersistentTasksExecutor<TestRequest> {
private final TransportService transportService;
public static final String NAME = "cluster:admin/persistent/test";
private final ClusterService clusterService;
@Inject
public TransportTestPersistentAction(Settings settings, ThreadPool threadPool, TransportService transportService,
PersistentActionService persistentActionService,
PersistentActionRegistry persistentActionRegistry, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, TestPersistentAction.NAME, false, threadPool, transportService, persistentActionService,
persistentActionRegistry, actionFilters, indexNameExpressionResolver, TestRequest::new,
ThreadPool.Names.GENERIC);
this.transportService = transportService;
public TestPersistentTasksExecutor(Settings settings, PersistentTasksService persistentTasksService,
ClusterService clusterService) {
super(settings, NAME, persistentTasksService, ThreadPool.Names.GENERIC);
this.clusterService = clusterService;
}
@Override
@ -377,9 +335,9 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
// wait for something to happen
assertTrue(awaitBusy(() -> testTask.isCancelled() ||
testTask.getOperation() != null ||
transportService.lifecycleState() != Lifecycle.State.STARTED, // speedup finishing on closed nodes
clusterService.lifecycleState() != Lifecycle.State.STARTED, // speedup finishing on closed nodes
30, TimeUnit.SECONDS)); // This can take a while during large cluster restart
if (transportService.lifecycleState() != Lifecycle.State.STARTED) {
if (clusterService.lifecycleState() != Lifecycle.State.STARTED) {
return;
}
if ("finish".equals(testTask.getOperation())) {

2
plugin/src/test/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskRequestTests.java
View File

@ -8,7 +8,7 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.test.AbstractStreamableTestCase;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.Status;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.Status;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction.Request;
import java.util.Collections;

2
plugin/src/test/resources/org/elasticsearch/transport/actions
View File

@ -143,4 +143,4 @@ cluster:admin/persistent/create
cluster:admin/persistent/start
cluster:admin/persistent/completion
cluster:admin/persistent/update_status
cluster:admin/persistent/remove
cluster:admin/persistent/remove