From 5c4933f5ea3ef626c297ab5897d413fe1be4717f Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 22 Mar 2017 14:02:30 -0400 Subject: [PATCH] Make PersistentAction independent from TransportActions (elastic/x-pack-elasticsearch#742) Removes the transport layer dependency from PersistentActions, makes PersistentActionRegistry immutable and rename actions into tasks in class and variable names. Original commit: elastic/x-pack-elasticsearch@e3e5b79c2845a73f4a9e384713b8c54e83a71782 --- dev-tools/checkstyle_suppressions.xml | 32 +-- .../xpack/ml/MachineLearning.java | 55 ++--- .../elasticsearch/xpack/ml/MlMetadata.java | 22 +- .../xpack/ml/action/CloseJobAction.java | 12 +- .../xpack/ml/action/DeleteDatafeedAction.java | 6 +- .../ml/action/GetDatafeedsStatsAction.java | 8 +- .../xpack/ml/action/GetJobsStatsAction.java | 10 +- .../xpack/ml/action/OpenJobAction.java | 164 +++++++++----- .../xpack/ml/action/StartDatafeedAction.java | 133 ++++++++---- .../xpack/ml/action/StopDatafeedAction.java | 8 +- .../ml/action/TransportJobTaskAction.java | 8 +- .../xpack/ml/action/UpdateDatafeedAction.java | 6 +- .../xpack/ml/action/UpdateJobAction.java | 5 +- .../ml/client/MachineLearningClient.java | 5 +- .../xpack/ml/datafeed/DatafeedJobRunner.java | 24 ++- .../xpack/ml/job/JobManager.java | 9 +- .../autodetect/AutodetectProcessManager.java | 75 ++++--- .../datafeeds/RestStartDatafeedAction.java | 7 +- .../xpack/ml/rest/job/RestOpenJobAction.java | 8 +- .../xpack/ml/utils/DatafeedStateObserver.java | 4 +- .../xpack/ml/utils/JobStateObserver.java | 6 +- .../CompletionPersistentTaskAction.java | 38 +++- .../CreatePersistentTaskAction.java | 50 ++--- ....java => NodePersistentTasksExecutor.java} | 18 +- .../persistent/PersistentActionRegistry.java | 91 -------- .../persistent/PersistentActionService.java | 90 -------- ...equest.java => PersistentTaskRequest.java} | 4 +- ...ponse.java => PersistentTaskResponse.java} | 10 +- ...ava => PersistentTasksClusterService.java} | 64 +++--- ...ava => PersistentTasksCustomMetaData.java} | 107 ++++----- ...tion.java => PersistentTasksExecutor.java} | 54 ++--- .../PersistentTasksExecutorRegistry.java | 41 ++++ ...r.java => PersistentTasksNodeService.java} | 78 ++++--- .../persistent/PersistentTasksService.java | 135 ++++++++++++ .../RemovePersistentTaskAction.java | 8 +- .../persistent/StartPersistentTaskAction.java | 10 +- .../UpdatePersistentTaskStatusAction.java | 8 +- .../xpack/persistent/package-info.java | 31 ++- .../MachineLearningLicensingTests.java | 55 +++-- .../xpack/ml/MlMetadataTests.java | 24 +-- .../xpack/ml/action/CloseJobActionTests.java | 22 +- .../xpack/ml/action/DatafeedJobsIT.java | 21 +- .../xpack/ml/action/OpenJobActionTests.java | 45 ++-- .../ml/action/StartDatafeedActionTests.java | 30 +-- .../StopDatafeedActionRequestTests.java | 22 +- .../ml/datafeed/DatafeedJobRunnerTests.java | 25 +-- .../integration/BasicDistributedJobsIT.java | 18 +- .../integration/MlDistributedFailureIT.java | 6 +- .../xpack/ml/integration/TooManyJobsIT.java | 6 +- .../AutodetectProcessManagerTests.java | 24 ++- .../PersistentActionRegistryTests.java | 50 ----- ...> PersistentTasksClusterServiceTests.java} | 89 ++++---- ...> PersistentTasksCustomMetaDataTests.java} | 54 ++--- ...PersistentTasksExecutorFullRestartIT.java} | 68 +++--- ...IT.java => PersistentTasksExecutorIT.java} | 141 +++++++----- ...PersistentTasksExecutorResponseTests.java} | 10 +- ...ersistentTasksNodeServiceStatusTests.java} | 6 +- ...a => PersistentTasksNodeServiceTests.java} | 204 +++++++++--------- .../StartPersistentActionRequestTests.java | 6 +- ...in.java => TestPersistentTasksPlugin.java} | 106 +++------ .../UpdatePersistentTaskRequestTests.java | 2 +- .../org/elasticsearch/transport/actions | 2 +- 62 files changed, 1315 insertions(+), 1165 deletions(-) rename plugin/src/main/java/org/elasticsearch/xpack/persistent/{PersistentActionExecutor.java => NodePersistentTasksExecutor.java} (68%) delete mode 100644 plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRegistry.java delete mode 100644 plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionService.java rename plugin/src/main/java/org/elasticsearch/xpack/persistent/{PersistentActionRequest.java => PersistentTaskRequest.java} (81%) rename plugin/src/main/java/org/elasticsearch/xpack/persistent/{PersistentActionResponse.java => PersistentTaskResponse.java} (81%) rename plugin/src/main/java/org/elasticsearch/xpack/persistent/{PersistentTaskClusterService.java => PersistentTasksClusterService.java} (80%) rename plugin/src/main/java/org/elasticsearch/xpack/persistent/{PersistentTasks.java => PersistentTasksCustomMetaData.java} (84%) rename plugin/src/main/java/org/elasticsearch/xpack/persistent/{TransportPersistentAction.java => PersistentTasksExecutor.java} (62%) create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorRegistry.java rename plugin/src/main/java/org/elasticsearch/xpack/persistent/{PersistentActionCoordinator.java => PersistentTasksNodeService.java} (80%) create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksService.java delete mode 100644 plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionRegistryTests.java rename plugin/src/test/java/org/elasticsearch/xpack/persistent/{PersistentTaskClusterServiceTests.java => PersistentTasksClusterServiceTests.java} (84%) rename plugin/src/test/java/org/elasticsearch/xpack/persistent/{PersistentTasksTests.java => PersistentTasksCustomMetaDataTests.java} (81%) rename plugin/src/test/java/org/elasticsearch/xpack/persistent/{PersistentActionFullRestartIT.java => PersistentTasksExecutorFullRestartIT.java} (59%) rename plugin/src/test/java/org/elasticsearch/xpack/persistent/{PersistentActionIT.java => PersistentTasksExecutorIT.java} (61%) rename plugin/src/test/java/org/elasticsearch/xpack/persistent/{PersistentActionResponseTests.java => PersistentTasksExecutorResponseTests.java} (53%) rename plugin/src/test/java/org/elasticsearch/xpack/persistent/{PersistentActionCoordinatorStatusTests.java => PersistentTasksNodeServiceStatusTests.java} (75%) rename plugin/src/test/java/org/elasticsearch/xpack/persistent/{PersistentActionCoordinatorTests.java => PersistentTasksNodeServiceTests.java} (58%) rename plugin/src/test/java/org/elasticsearch/xpack/persistent/{TestPersistentActionPlugin.java => TestPersistentTasksPlugin.java} (82%) diff --git a/dev-tools/checkstyle_suppressions.xml b/dev-tools/checkstyle_suppressions.xml index da1a06944e2..ae10c3804da 100644 --- a/dev-tools/checkstyle_suppressions.xml +++ b/dev-tools/checkstyle_suppressions.xml @@ -376,16 +376,17 @@ - - - - - - - + + + + + + + + + - @@ -1089,16 +1090,15 @@ - - - - - - - + + + + + + - + diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 69ce26b53d0..f4ae6fa068f 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -123,12 +123,12 @@ import org.elasticsearch.xpack.ml.rest.validate.RestValidateDetectorAction; import org.elasticsearch.xpack.ml.rest.validate.RestValidateJobConfigAction; import org.elasticsearch.xpack.persistent.CompletionPersistentTaskAction; import org.elasticsearch.xpack.persistent.CreatePersistentTaskAction; -import org.elasticsearch.xpack.persistent.PersistentActionCoordinator; -import org.elasticsearch.xpack.persistent.PersistentActionRegistry; -import org.elasticsearch.xpack.persistent.PersistentActionRequest; -import org.elasticsearch.xpack.persistent.PersistentActionService; -import org.elasticsearch.xpack.persistent.PersistentTaskClusterService; -import org.elasticsearch.xpack.persistent.PersistentTasks; +import org.elasticsearch.xpack.persistent.PersistentTasksClusterService; +import org.elasticsearch.xpack.persistent.PersistentTasksNodeService; +import org.elasticsearch.xpack.persistent.PersistentTasksExecutorRegistry; +import org.elasticsearch.xpack.persistent.PersistentTaskRequest; +import org.elasticsearch.xpack.persistent.PersistentTasksService; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.xpack.persistent.RemovePersistentTaskAction; import org.elasticsearch.xpack.persistent.StartPersistentTaskAction; import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction; @@ -208,16 +208,18 @@ public class MachineLearning implements ActionPlugin { // Custom metadata new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new), new NamedWriteableRegistry.Entry(NamedDiff.class, "ml", MlMetadata.MlMetadataDiff::new), - new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new), - new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom), + new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, + PersistentTasksCustomMetaData::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE, + PersistentTasksCustomMetaData::readDiffFrom), // Persistent action requests - new NamedWriteableRegistry.Entry(PersistentActionRequest.class, StartDatafeedAction.NAME, StartDatafeedAction.Request::new), - new NamedWriteableRegistry.Entry(PersistentActionRequest.class, OpenJobAction.NAME, OpenJobAction.Request::new), + new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, StartDatafeedAction.NAME, StartDatafeedAction.Request::new), + new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, OpenJobAction.NAME, OpenJobAction.Request::new), // Task statuses - new NamedWriteableRegistry.Entry(Task.Status.class, PersistentActionCoordinator.Status.NAME, - PersistentActionCoordinator.Status::new), + new NamedWriteableRegistry.Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, + PersistentTasksNodeService.Status::new), new NamedWriteableRegistry.Entry(Task.Status.class, JobState.NAME, JobState::fromStream), new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream) ); @@ -228,13 +230,13 @@ public class MachineLearning implements ActionPlugin { // Custom metadata new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField("ml"), parser -> MlMetadata.ML_METADATA_PARSER.parse(parser, null).build()), - new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasks.TYPE), - PersistentTasks::fromXContent), + new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksCustomMetaData.TYPE), + PersistentTasksCustomMetaData::fromXContent), // Persistent action requests - new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(StartDatafeedAction.NAME), + new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(StartDatafeedAction.NAME), StartDatafeedAction.Request::fromXContent), - new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(OpenJobAction.NAME), + new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(OpenJobAction.NAME), OpenJobAction.Request::fromXContent), // Task statuses @@ -290,17 +292,22 @@ public class MachineLearning implements ActionPlugin { } NormalizerFactory normalizerFactory = new NormalizerFactory(normalizerProcessFactory, threadPool.executor(MachineLearning.THREAD_POOL_NAME)); + PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, internalClient); AutodetectProcessManager autodetectProcessManager = new AutodetectProcessManager(settings, internalClient, threadPool, jobManager, jobProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, - normalizerFactory); + normalizerFactory, persistentTasksService); DatafeedJobRunner datafeedJobRunner = new DatafeedJobRunner(threadPool, internalClient, clusterService, jobProvider, - System::currentTimeMillis, auditor); - PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, threadPool, clusterService, - internalClient); - PersistentActionRegistry persistentActionRegistry = new PersistentActionRegistry(Settings.EMPTY); + System::currentTimeMillis, persistentTasksService, auditor); InvalidLicenseEnforcer invalidLicenseEnforcer = new InvalidLicenseEnforcer(settings, licenseState, threadPool, datafeedJobRunner, autodetectProcessManager); + PersistentTasksExecutorRegistry persistentTasksExecutorRegistry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Arrays.asList( + new OpenJobAction.OpenJobPersistentTasksExecutor(settings, threadPool, licenseState, persistentTasksService, clusterService, + autodetectProcessManager, auditor), + new StartDatafeedAction.StartDatafeedPersistentTasksExecutor(settings, threadPool, licenseState, persistentTasksService, + datafeedJobRunner, auditor) + )); + return Arrays.asList( mlLifeCycleService, jobProvider, @@ -310,9 +317,9 @@ public class MachineLearning implements ActionPlugin { new MlInitializationService(settings, threadPool, clusterService, internalClient), jobDataCountsPersister, datafeedJobRunner, - persistentActionService, - persistentActionRegistry, - new PersistentTaskClusterService(Settings.EMPTY, persistentActionRegistry, clusterService), + persistentTasksService, + persistentTasksExecutorRegistry, + new PersistentTasksClusterService(Settings.EMPTY, persistentTasksExecutorRegistry, clusterService), auditor, new CloseJobService(internalClient, threadPool, clusterService), invalidLicenseEnforcer diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java index da9586c52e8..9edbbe03263 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java @@ -32,8 +32,8 @@ import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.messages.Messages; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.persistent.PersistentTasks; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.Collection; @@ -240,7 +240,7 @@ public class MlMetadata implements MetaData.Custom { return this; } - public Builder deleteJob(String jobId, PersistentTasks tasks) { + public Builder deleteJob(String jobId, PersistentTasksCustomMetaData tasks) { Optional datafeed = getDatafeedByJobId(jobId); if (datafeed.isPresent()) { throw ExceptionsHelper.conflictStatusException("Cannot delete job [" + jobId + "] while datafeed [" @@ -285,7 +285,7 @@ public class MlMetadata implements MetaData.Custom { } } - public Builder updateDatafeed(DatafeedUpdate update, PersistentTasks persistentTasks) { + public Builder updateDatafeed(DatafeedUpdate update, PersistentTasksCustomMetaData persistentTasks) { String datafeedId = update.getId(); DatafeedConfig oldDatafeedConfig = datafeeds.get(datafeedId); if (oldDatafeedConfig == null) { @@ -303,7 +303,7 @@ public class MlMetadata implements MetaData.Custom { return this; } - public Builder removeDatafeed(String datafeedId, PersistentTasks persistentTasks) { + public Builder removeDatafeed(String datafeedId, PersistentTasksCustomMetaData persistentTasks) { DatafeedConfig datafeed = datafeeds.get(datafeedId); if (datafeed == null) { throw ExceptionsHelper.missingDatafeedException(datafeedId); @@ -318,7 +318,7 @@ public class MlMetadata implements MetaData.Custom { return datafeeds.values().stream().filter(s -> s.getJobId().equals(jobId)).findFirst(); } - private void checkDatafeedIsStopped(Supplier msg, String datafeedId, PersistentTasks persistentTasks) { + private void checkDatafeedIsStopped(Supplier msg, String datafeedId, PersistentTasksCustomMetaData persistentTasks) { if (persistentTasks != null) { Predicate> predicate = t -> { StartDatafeedAction.Request storedRequest = (StartDatafeedAction.Request) t.getRequest(); @@ -348,7 +348,7 @@ public class MlMetadata implements MetaData.Custom { return new MlMetadata(jobs, datafeeds); } - public void markJobAsDeleted(String jobId, PersistentTasks tasks) { + public void markJobAsDeleted(String jobId, PersistentTasksCustomMetaData tasks) { Job job = jobs.get(jobId); if (job == null) { throw ExceptionsHelper.missingJobException(jobId); @@ -375,7 +375,7 @@ public class MlMetadata implements MetaData.Custom { } @Nullable - public static PersistentTask getJobTask(String jobId, @Nullable PersistentTasks tasks) { + public static PersistentTask getJobTask(String jobId, @Nullable PersistentTasksCustomMetaData tasks) { if (tasks != null) { Predicate> p = t -> { OpenJobAction.Request storedRequest = (OpenJobAction.Request) t.getRequest(); @@ -389,7 +389,7 @@ public class MlMetadata implements MetaData.Custom { } @Nullable - public static PersistentTask getDatafeedTask(String datafeedId, @Nullable PersistentTasks tasks) { + public static PersistentTask getDatafeedTask(String datafeedId, @Nullable PersistentTasksCustomMetaData tasks) { if (tasks != null) { Predicate> p = t -> { StartDatafeedAction.Request storedRequest = (StartDatafeedAction.Request) t.getRequest(); @@ -402,7 +402,7 @@ public class MlMetadata implements MetaData.Custom { return null; } - public static JobState getJobState(String jobId, @Nullable PersistentTasks tasks) { + public static JobState getJobState(String jobId, @Nullable PersistentTasksCustomMetaData tasks) { PersistentTask task = getJobTask(jobId, tasks); if (task != null && task.getStatus() != null) { JobState jobTaskState = (JobState) task.getStatus(); @@ -414,7 +414,7 @@ public class MlMetadata implements MetaData.Custom { return JobState.CLOSED; } - public static DatafeedState getDatafeedState(String datafeedId, @Nullable PersistentTasks tasks) { + public static DatafeedState getDatafeedState(String datafeedId, @Nullable PersistentTasksCustomMetaData tasks) { PersistentTask task = getDatafeedTask(datafeedId, tasks); if (task != null && task.getStatus() != null) { return (DatafeedState) task.getStatus(); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/CloseJobAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/CloseJobAction.java index e238931a353..600121cd70f 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/CloseJobAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/CloseJobAction.java @@ -43,8 +43,8 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.persistent.PersistentTasks; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.Date; @@ -286,7 +286,7 @@ public class CloseJobAction extends Action datafeed = mlMetadata.getDatafeedByJobId(jobId); if (datafeed.isPresent()) { DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeed.get().getId(), tasks); @@ -311,12 +311,12 @@ public class CloseJobAction extends Action task = validateAndFindTask(jobId, currentState); - PersistentTasks currentTasks = currentState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData currentTasks = currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); Map> updatedTasks = new HashMap<>(currentTasks.taskMap()); PersistentTask taskToUpdate = currentTasks.getTask(task.getId()); taskToUpdate = new PersistentTask<>(taskToUpdate, JobState.CLOSING); updatedTasks.put(taskToUpdate.getId(), taskToUpdate); - PersistentTasks newTasks = new PersistentTasks(currentTasks.getCurrentId(), updatedTasks); + PersistentTasksCustomMetaData newTasks = new PersistentTasksCustomMetaData(currentTasks.getCurrentId(), updatedTasks); MlMetadata mlMetadata = currentState.metaData().custom(MlMetadata.TYPE); Job.Builder jobBuilder = new Job.Builder(mlMetadata.getJobs().get(jobId)); @@ -328,7 +328,7 @@ public class CloseJobAction extends Action d.getId()).collect(Collectors.toList()) : Collections.singletonList(request.getDatafeedId()); - PersistentTasks tasksInProgress = state.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasksInProgress = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); List results = expandedDatafeedsIds.stream() .map(datafeedId -> getDatafeedStats(datafeedId, state, tasksInProgress)) .collect(Collectors.toList()); @@ -328,7 +328,7 @@ public class GetDatafeedsStatsAction extends Action task = MlMetadata.getDatafeedTask(datafeedId, tasks); DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeedId, tasks); DiscoveryNode node = null; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsStatsAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsStatsAction.java index 423c2fa0433..5145a2f9897 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsStatsAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsStatsAction.java @@ -45,7 +45,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManage import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.persistent.PersistentTasks; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; import java.io.IOException; import java.time.Duration; @@ -410,10 +410,10 @@ public class GetJobsStatsAction extends Action> stats = processManager.getStatistics(jobId); if (stats.isPresent()) { - PersistentTasks.PersistentTask pTask = MlMetadata.getJobTask(jobId, tasks); + PersistentTasksCustomMetaData.PersistentTask pTask = MlMetadata.getJobTask(jobId, tasks); DiscoveryNode node = state.nodes().get(pTask.getExecutorNode()); JobState jobState = MlMetadata.getJobState(jobId, tasks); String assignmentExplanation = pTask.getAssignment().getExplanation(); @@ -437,13 +437,13 @@ public class GetJobsStatsAction extends Action jobStats = new AtomicArray<>(jobIds.size()); - PersistentTasks tasks = clusterService.state().getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); for (int i = 0; i < jobIds.size(); i++) { int slot = i; String jobId = jobIds.get(i); gatherDataCountsAndModelSizeStats(jobId, (dataCounts, modelSizeStats) -> { JobState jobState = MlMetadata.getJobState(request.jobId, tasks); - PersistentTasks.PersistentTask pTask = MlMetadata.getJobTask(jobId, tasks); + PersistentTasksCustomMetaData.PersistentTask pTask = MlMetadata.getJobTask(jobId, tasks); String assignmentExplanation = null; if (pTask != null) { assignmentExplanation = pTask.getAssignment().getExplanation(); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/OpenJobAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/OpenJobAction.java index 522a9cb1aad..3c1b954c51a 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/OpenJobAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/OpenJobAction.java @@ -12,6 +12,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -50,14 +52,13 @@ import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.JobStateObserver; import org.elasticsearch.xpack.persistent.NodePersistentTask; -import org.elasticsearch.xpack.persistent.PersistentActionRegistry; -import org.elasticsearch.xpack.persistent.PersistentActionRequest; -import org.elasticsearch.xpack.persistent.PersistentActionResponse; -import org.elasticsearch.xpack.persistent.PersistentActionService; -import org.elasticsearch.xpack.persistent.PersistentTasks; -import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; -import org.elasticsearch.xpack.persistent.TransportPersistentAction; +import org.elasticsearch.xpack.persistent.PersistentTasksExecutor; +import org.elasticsearch.xpack.persistent.PersistentTaskRequest; +import org.elasticsearch.xpack.persistent.PersistentTasksService; +import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.ArrayList; @@ -68,7 +69,7 @@ import java.util.Objects; import static org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE; -public class OpenJobAction extends Action { +public class OpenJobAction extends Action { public static final OpenJobAction INSTANCE = new OpenJobAction(); public static final String NAME = "cluster:admin/ml/job/open"; @@ -83,11 +84,11 @@ public class OpenJobAction extends Action { + static class RequestBuilder extends ActionRequestBuilder { RequestBuilder(ElasticsearchClient client, OpenJobAction action) { super(client, action, new Request()); } } - public static class TransportAction extends TransportPersistentAction { + public static class TransportAction extends HandledTransportAction { private final JobStateObserver observer; - private final ClusterService clusterService; - private final AutodetectProcessManager autodetectProcessManager; private final XPackLicenseState licenseState; - private final Auditor auditor; - - private volatile int maxConcurrentJobAllocations; + private final PersistentTasksService persistentTasksService; @Inject public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, XPackLicenseState licenseState, - PersistentActionService persistentActionService, PersistentActionRegistry persistentActionRegistry, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - ClusterService clusterService, AutodetectProcessManager autodetectProcessManager, Auditor auditor) { - super(settings, OpenJobAction.NAME, false, threadPool, transportService, persistentActionService, - persistentActionRegistry, actionFilters, indexNameExpressionResolver, Request::new, ThreadPool.Names.MANAGEMENT); + PersistentTasksService persistentTasksService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); this.licenseState = licenseState; - this.clusterService = clusterService; - this.autodetectProcessManager = autodetectProcessManager; - this.auditor = auditor; + this.persistentTasksService = persistentTasksService; this.observer = new JobStateObserver(threadPool, clusterService); - this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings); - clusterService.getClusterSettings() - .addSettingsUpdateConsumer(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, this::setMaxConcurrentJobAllocations); } @Override - protected void doExecute(Request request, ActionListener listener) { + protected void doExecute(Request request, ActionListener listener) { if (licenseState.isMachineLearningAllowed()) { - // If we already know that we can't find an ml node because all ml nodes are running at capacity or - // simply because there are no ml nodes in the cluster then we fail quickly here: - ClusterState clusterState = clusterService.state(); - validate(request, clusterState); - Assignment assignment = selectLeastLoadedMlNode(request.getJobId(), clusterState, maxConcurrentJobAllocations, logger); - if (assignment.getExecutorNode() == null) { - throw new ElasticsearchStatusException("cannot open job [" + request.getJobId() + "], no suitable nodes found, " + - "allocation explanation [{}]", RestStatus.TOO_MANY_REQUESTS, assignment.getExplanation()); - } + PersistentTaskOperationListener finalListener = new PersistentTaskOperationListener() { + @Override + public void onResponse(long taskId) { + waitForJobStarted(request, listener); + } - ActionListener finalListener = - ActionListener.wrap(response -> waitForJobStarted(request, response, listener), listener::onFailure); - super.doExecute(request, finalListener); + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }; + persistentTasksService.createPersistentActionTask(NAME, request, finalListener); } else { listener.onFailure(LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING)); } } - void waitForJobStarted(Request request, PersistentActionResponse response, ActionListener listener) { + void waitForJobStarted(Request request, ActionListener listener) { observer.waitForState(request.getJobId(), request.timeout, JobState.OPENED, e -> { if (e != null) { listener.onFailure(e); } else { - listener.onResponse(response); + listener.onResponse(new Response(true)); } }); } + } + + public static class OpenJobPersistentTasksExecutor extends PersistentTasksExecutor { + + private final AutodetectProcessManager autodetectProcessManager; + private final XPackLicenseState licenseState; + private final Auditor auditor; + private final ThreadPool threadPool; + + private volatile int maxConcurrentJobAllocations; + + public OpenJobPersistentTasksExecutor(Settings settings, ThreadPool threadPool, XPackLicenseState licenseState, + PersistentTasksService persistentTasksService, ClusterService clusterService, + AutodetectProcessManager autodetectProcessManager, Auditor auditor) { + super(settings, NAME, persistentTasksService, ThreadPool.Names.MANAGEMENT); + this.licenseState = licenseState; + this.autodetectProcessManager = autodetectProcessManager; + this.auditor = auditor; + this.threadPool = threadPool; + this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, this::setMaxConcurrentJobAllocations); + } @Override public Assignment getAssignment(Request request, ClusterState clusterState) { @@ -317,9 +364,20 @@ public class OpenJobAction extends Action reasons = new LinkedList<>(); DiscoveryNode minLoadedNode = null; - PersistentTasks persistentTasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData persistentTasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); for (DiscoveryNode node : clusterState.getNodes()) { Map nodeAttributes = node.getAttributes(); String maxNumberOfOpenJobsStr = nodeAttributes.get(AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE.getKey()); @@ -450,7 +508,7 @@ public class OpenJobAction extends Action { + extends Action { public static final ParseField START_TIME = new ParseField("start"); public static final ParseField END_TIME = new ParseField("end"); @@ -88,11 +89,11 @@ public class StartDatafeedAction } @Override - public PersistentActionResponse newResponse() { - return new PersistentActionResponse(); + public Response newResponse() { + return new Response(); } - public static class Request extends PersistentActionRequest implements ToXContent { + public static class Request extends PersistentTaskRequest implements ToXContent { public static ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); @@ -250,7 +251,41 @@ public class StartDatafeedAction } } - static class RequestBuilder extends ActionRequestBuilder { + public static class Response extends AcknowledgedResponse { + public Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeAcknowledged(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AcknowledgedResponse that = (AcknowledgedResponse) o; + return isAcknowledged() == that.isAcknowledged(); + } + + @Override + public int hashCode() { + return Objects.hash(isAcknowledged()); + } + + } + + static class RequestBuilder extends ActionRequestBuilder { RequestBuilder(ElasticsearchClient client, StartDatafeedAction action) { super(client, action, new Request()); @@ -303,48 +338,68 @@ public class StartDatafeedAction } } - public static class TransportAction extends TransportPersistentAction { + public static class TransportAction extends HandledTransportAction { private final DatafeedStateObserver observer; - private final DatafeedJobRunner datafeedJobRunner; private final XPackLicenseState licenseState; - private final Auditor auditor; + private final PersistentTasksService persistentTasksService; @Inject public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, XPackLicenseState licenseState, - PersistentActionService persistentActionService, PersistentActionRegistry persistentActionRegistry, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - ClusterService clusterService, DatafeedJobRunner datafeedJobRunner, Auditor auditor) { - super(settings, NAME, false, threadPool, transportService, persistentActionService, persistentActionRegistry, - actionFilters, indexNameExpressionResolver, Request::new, ThreadPool.Names.MANAGEMENT); + PersistentTasksService persistentTasksService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); this.licenseState = licenseState; - this.datafeedJobRunner = datafeedJobRunner; - this.auditor = auditor; + this.persistentTasksService = persistentTasksService; this.observer = new DatafeedStateObserver(threadPool, clusterService); } @Override - protected void doExecute(Request request, ActionListener listener) { + protected void doExecute(Request request, ActionListener listener) { if (licenseState.isMachineLearningAllowed()) { - ActionListener finalListener = ActionListener - .wrap(response -> waitForDatafeedStarted(request, response, listener), listener::onFailure); - super.doExecute(request, finalListener); + PersistentTaskOperationListener finalListener = new PersistentTaskOperationListener() { + @Override + public void onResponse(long taskId) { + waitForDatafeedStarted(request, listener); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }; + persistentTasksService.createPersistentActionTask(NAME, request, finalListener); } else { listener.onFailure(LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING)); } } - void waitForDatafeedStarted(Request request, - PersistentActionResponse response, - ActionListener listener) { + void waitForDatafeedStarted(Request request, ActionListener listener) { observer.waitForState(request.getDatafeedId(), request.timeout, DatafeedState.STARTED, e -> { if (e != null) { listener.onFailure(e); } else { - listener.onResponse(response); + listener.onResponse(new Response(true)); } }); } + } + + public static class StartDatafeedPersistentTasksExecutor extends PersistentTasksExecutor { + private final DatafeedJobRunner datafeedJobRunner; + private final XPackLicenseState licenseState; + private final Auditor auditor; + private final ThreadPool threadPool; + + public StartDatafeedPersistentTasksExecutor(Settings settings, ThreadPool threadPool, XPackLicenseState licenseState, + PersistentTasksService persistentTasksService, DatafeedJobRunner datafeedJobRunner, + Auditor auditor) { + super(settings, NAME, persistentTasksService, ThreadPool.Names.MANAGEMENT); + this.licenseState = licenseState; + this.datafeedJobRunner = datafeedJobRunner; + this.auditor = auditor; + this.threadPool = threadPool; + } @Override public Assignment getAssignment(Request request, ClusterState clusterState) { @@ -355,10 +410,14 @@ public class StartDatafeedAction @Override public void validate(Request request, ClusterState clusterState) { - MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE); - PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); - DiscoveryNodes nodes = clusterState.getNodes(); - StartDatafeedAction.validate(request.getDatafeedId(), mlMetadata, tasks, nodes); + if (licenseState.isMachineLearningAllowed()) { + MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE); + PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + DiscoveryNodes nodes = clusterState.getNodes(); + StartDatafeedAction.validate(request.getDatafeedId(), mlMetadata, tasks, nodes); + } else { + throw LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING); + } } @Override @@ -403,7 +462,7 @@ public class StartDatafeedAction } - static void validate(String datafeedId, MlMetadata mlMetadata, PersistentTasks tasks, DiscoveryNodes nodes) { + static void validate(String datafeedId, MlMetadata mlMetadata, PersistentTasksCustomMetaData tasks, DiscoveryNodes nodes) { DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId); if (datafeed == null) { throw ExceptionsHelper.missingDatafeedException(datafeedId); @@ -441,7 +500,7 @@ public class StartDatafeedAction public static Assignment selectNode(Logger logger, String datafeedId, ClusterState clusterState) { MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE); - PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId); DiscoveryNodes nodes = clusterState.getNodes(); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/StopDatafeedAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/StopDatafeedAction.java index 7206a9bfda6..b4d77a73d33 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/StopDatafeedAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/StopDatafeedAction.java @@ -44,8 +44,8 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.job.messages.Messages; import org.elasticsearch.xpack.ml.utils.DatafeedStateObserver; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.persistent.PersistentTasks; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.List; @@ -217,7 +217,7 @@ public class StopDatafeedAction ClusterState state = clusterService.state(); MetaData metaData = state.metaData(); MlMetadata mlMetadata = metaData.custom(MlMetadata.TYPE); - PersistentTasks tasks = metaData.custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = metaData.custom(PersistentTasksCustomMetaData.TYPE); String nodeId = validateAndReturnNodeId(request.getDatafeedId(), mlMetadata, tasks); request.setNodes(nodeId); ActionListener finalListener = @@ -262,7 +262,7 @@ public class StopDatafeedAction } } - static String validateAndReturnNodeId(String datafeedId, MlMetadata mlMetadata, PersistentTasks tasks) { + static String validateAndReturnNodeId(String datafeedId, MlMetadata mlMetadata, PersistentTasksCustomMetaData tasks) { DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId); if (datafeed == null) { throw new ResourceNotFoundException(Messages.getMessage(Messages.DATAFEED_NOT_FOUND, datafeedId)); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java index 9812f1ee56d..e67bcaf37d4 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java @@ -30,7 +30,7 @@ import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.persistent.PersistentTasks; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; import java.io.IOException; import java.util.List; @@ -62,8 +62,8 @@ public abstract class TransportJobTaskAction jobTask = MlMetadata.getJobTask(jobId, tasks); + PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + PersistentTasksCustomMetaData.PersistentTask jobTask = MlMetadata.getJobTask(jobId, tasks); if (jobTask == null || jobTask.isAssigned() == false) { listener.onFailure( new ElasticsearchStatusException("job [" + jobId + "] state is [" + JobState.CLOSED + "], but must be [" + JobState.OPENED + "] to perform requested action", RestStatus.CONFLICT)); @@ -76,7 +76,7 @@ public abstract class TransportJobTaskAction listener) { ClusterState state = clusterService.state(); - PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE); JobState jobState = MlMetadata.getJobState(request.getJobId(), tasks); if (jobState == JobState.OPENED) { innerTaskOperation(request, task, listener); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateDatafeedAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateDatafeedAction.java index bec24c383b7..92e1044912e 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateDatafeedAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateDatafeedAction.java @@ -32,7 +32,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.ml.datafeed.DatafeedUpdate; import org.elasticsearch.xpack.ml.MlMetadata; -import org.elasticsearch.xpack.persistent.PersistentTasks; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; import java.io.IOException; import java.util.Objects; @@ -161,8 +161,8 @@ public class UpdateDatafeedAction extends Action new Semaphore(1)).acquire(); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/client/MachineLearningClient.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/client/MachineLearningClient.java index a462860d2bb..33745389e43 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/client/MachineLearningClient.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/client/MachineLearningClient.java @@ -34,7 +34,6 @@ import org.elasticsearch.xpack.ml.action.StopDatafeedAction; import org.elasticsearch.xpack.ml.action.UpdateDatafeedAction; import org.elasticsearch.xpack.ml.action.UpdateJobAction; import org.elasticsearch.xpack.ml.action.UpdateModelSnapshotAction; -import org.elasticsearch.xpack.persistent.PersistentActionResponse; public class MachineLearningClient { @@ -109,7 +108,7 @@ public class MachineLearningClient { client.execute(GetRecordsAction.INSTANCE, request, listener); } - public void openJob(OpenJobAction.Request request, ActionListener listener) { + public void openJob(OpenJobAction.Request request, ActionListener listener) { client.execute(OpenJobAction.INSTANCE, request, listener); } @@ -134,7 +133,7 @@ public class MachineLearningClient { client.execute(RevertModelSnapshotAction.INSTANCE, request, listener); } - public void startDatafeed(StartDatafeedAction.Request request, ActionListener listener) { + public void startDatafeed(StartDatafeedAction.Request request, ActionListener listener) { client.execute(StartDatafeedAction.INSTANCE, request, listener); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunner.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunner.java index de346bb515d..57d2263d52c 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunner.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunner.java @@ -34,9 +34,10 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.job.results.Bucket; import org.elasticsearch.xpack.ml.job.results.Result; import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.elasticsearch.xpack.persistent.PersistentTasksService; +import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.xpack.ml.utils.DatafeedStateObserver; -import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; -import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction; import java.time.Duration; import java.util.Collections; @@ -60,17 +61,19 @@ public class DatafeedJobRunner extends AbstractComponent { private final JobProvider jobProvider; private final ThreadPool threadPool; private final Supplier currentTimeSupplier; + private final PersistentTasksService persistentTasksService; private final Auditor auditor; private final ConcurrentMap runningDatafeeds = new ConcurrentHashMap<>(); public DatafeedJobRunner(ThreadPool threadPool, Client client, ClusterService clusterService, JobProvider jobProvider, - Supplier currentTimeSupplier, Auditor auditor) { + Supplier currentTimeSupplier, PersistentTasksService persistentTasksService, Auditor auditor) { super(Settings.EMPTY); this.client = Objects.requireNonNull(client); this.clusterService = Objects.requireNonNull(clusterService); this.jobProvider = Objects.requireNonNull(jobProvider); this.threadPool = threadPool; this.currentTimeSupplier = Objects.requireNonNull(currentTimeSupplier); + this.persistentTasksService = persistentTasksService; this.auditor = auditor; } @@ -268,10 +271,17 @@ public class DatafeedJobRunner extends AbstractComponent { } private void updateDatafeedState(long persistentTaskId, DatafeedState datafeedState, Consumer handler) { - UpdatePersistentTaskStatusAction.Request request = new UpdatePersistentTaskStatusAction.Request(persistentTaskId, datafeedState); - client.execute(UpdatePersistentTaskStatusAction.INSTANCE, request, ActionListener.wrap(r -> { - handler.accept(null); - }, handler)); + persistentTasksService.updateStatus(persistentTaskId, datafeedState, new PersistentTaskOperationListener() { + @Override + public void onResponse(long taskId) { + handler.accept(null); + } + + @Override + public void onFailure(Exception e) { + handler.accept(e); + } + }); } private static Duration getFrequencyOrDefault(DatafeedConfig datafeed, Job job) { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 8418b64aa99..23e414ee876 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.ml.MlMetadata; -import org.elasticsearch.xpack.ml.MachineLearningTemplateRegistry; import org.elasticsearch.xpack.ml.action.DeleteJobAction; import org.elasticsearch.xpack.ml.action.PutJobAction; import org.elasticsearch.xpack.ml.action.RevertModelSnapshotAction; @@ -34,7 +33,7 @@ import org.elasticsearch.xpack.ml.job.persistence.JobStorageDeletionTask; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.persistent.PersistentTasks; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; import java.util.Collections; import java.util.List; @@ -129,7 +128,7 @@ public class JobManager extends AbstractComponent { } public JobState getJobState(String jobId) { - PersistentTasks tasks = clusterService.state().getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); return MlMetadata.getJobState(jobId, tasks); } @@ -281,7 +280,7 @@ public class JobManager extends AbstractComponent { @Override public ClusterState execute(ClusterState currentState) throws Exception { MlMetadata.Builder builder = createMlMetadataBuilder(currentState); - builder.deleteJob(jobId, currentState.getMetaData().custom(PersistentTasks.TYPE)); + builder.deleteJob(jobId, currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); return buildNewClusterState(currentState, builder); } }); @@ -307,7 +306,7 @@ public class JobManager extends AbstractComponent { @Override public ClusterState execute(ClusterState currentState) throws Exception { MlMetadata currentMlMetadata = currentState.metaData().custom(MlMetadata.TYPE); - PersistentTasks tasks = currentState.metaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = currentState.metaData().custom(PersistentTasksCustomMetaData.TYPE); MlMetadata.Builder builder = new MlMetadata.Builder(currentMlMetadata); builder.markJobAsDeleted(jobId, tasks); return buildNewClusterState(currentState, builder); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index f5cb3644168..6d8a5da4d08 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.ml.job.process.autodetect; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.collect.Tuple; @@ -41,7 +40,8 @@ import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer; import org.elasticsearch.xpack.ml.job.process.normalizer.ScoresUpdater; import org.elasticsearch.xpack.ml.job.process.normalizer.ShortCircuitingRenormalizer; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction; +import org.elasticsearch.xpack.persistent.PersistentTasksService; +import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener; import java.io.IOException; import java.io.InputStream; @@ -74,6 +74,7 @@ public class AutodetectProcessManager extends AbstractComponent { private final JobResultsPersister jobResultsPersister; private final JobDataCountsPersister jobDataCountsPersister; + private final PersistentTasksService persistentTasksService; private final ConcurrentMap autoDetectCommunicatorByJob; @@ -82,7 +83,8 @@ public class AutodetectProcessManager extends AbstractComponent { public AutodetectProcessManager(Settings settings, Client client, ThreadPool threadPool, JobManager jobManager, JobProvider jobProvider, JobResultsPersister jobResultsPersister, JobDataCountsPersister jobDataCountsPersister, - AutodetectProcessFactory autodetectProcessFactory, NormalizerFactory normalizerFactory) { + AutodetectProcessFactory autodetectProcessFactory, NormalizerFactory normalizerFactory, + PersistentTasksService persistentTasksService) { super(settings); this.client = client; this.threadPool = threadPool; @@ -94,6 +96,7 @@ public class AutodetectProcessManager extends AbstractComponent { this.jobResultsPersister = jobResultsPersister; this.jobDataCountsPersister = jobDataCountsPersister; + this.persistentTasksService = persistentTasksService; this.autoDetectCommunicatorByJob = new ConcurrentHashMap<>(); } @@ -113,18 +116,18 @@ public class AutodetectProcessManager extends AbstractComponent { * Passes data to the native process. * This is a blocking call that won't return until all the data has been * written to the process. - * + *

* An ElasticsearchStatusException will be thrown is any of these error conditions occur: *

    - *
  1. If a configured field is missing from the CSV header
    2. - *
  2. If JSON data is malformed and we cannot recover parsing
    3. - *
  3. If a high proportion of the records the timestamp field that cannot be parsed
    4. - *
  4. If a high proportion of the records chronologically out of order
    5. + *
  5. If a configured field is missing from the CSV header
    6. + *
  6. If JSON data is malformed and we cannot recover parsing
    7. + *
  7. If a high proportion of the records the timestamp field that cannot be parsed
    8. + *
  8. If a high proportion of the records chronologically out of order
    9. *
* - * @param jobId the jobId - * @param input Data input stream - * @param params Data processing parameters + * @param jobId the jobId + * @param input Data input stream + * @param params Data processing parameters * @return Count of records, fields, bytes, etc written */ public DataCounts processData(String jobId, InputStream input, DataLoadParams params) { @@ -152,9 +155,9 @@ public class AutodetectProcessManager extends AbstractComponent { * opportunity to process all data previously sent to it with none left * sitting in buffers. * - * @param jobId The job to flush + * @param jobId The job to flush * @param params Parameters about whether interim results calculation - * should occur and for which period of time + * should occur and for which period of time */ public void flushJob(String jobId, InterimResultsParams params) { logger.debug("Flushing job {}", jobId); @@ -278,9 +281,10 @@ public class AutodetectProcessManager extends AbstractComponent { /** * Stop the running job and mark it as finished.
- * @param jobId The job to stop - * @param restart Whether the job should be restarted by persistent tasks - * @param reason The reason for closing the job + * + * @param jobId The job to stop + * @param restart Whether the job should be restarted by persistent tasks + * @param reason The reason for closing the job */ public void closeJob(String jobId, boolean restart, String reason) { logger.debug("Attempting to close job [{}], because [{}]", jobId, reason); @@ -321,34 +325,39 @@ public class AutodetectProcessManager extends AbstractComponent { } private void setJobState(long taskId, String jobId, JobState state) { - UpdatePersistentTaskStatusAction.Request request = new UpdatePersistentTaskStatusAction.Request(taskId, state); - client.execute(UpdatePersistentTaskStatusAction.INSTANCE, request, new ActionListener() { + persistentTasksService.updateStatus(taskId, state, new PersistentTaskOperationListener() { @Override - public void onResponse(UpdatePersistentTaskStatusAction.Response response) { - if (response.isAcknowledged()) { - logger.info("Successfully set job state to [{}] for job [{}]", state, jobId); - } else { - logger.info("Changing job state to [{}] for job [{}] wasn't acked", state, jobId); - } + public void onResponse(long taskId) { + logger.info("Successfully set job state to [{}] for job [{}]", state, jobId); } @Override public void onFailure(Exception e) { - logger.error("Could not set job state to [" + state + "] for job [" + jobId +"]", e); + logger.error("Could not set job state to [" + state + "] for job [" + jobId + "]", e); } }); } public void setJobState(long taskId, JobState state, CheckedConsumer handler) { - UpdatePersistentTaskStatusAction.Request request = new UpdatePersistentTaskStatusAction.Request(taskId, state); - client.execute(UpdatePersistentTaskStatusAction.INSTANCE, request, - ActionListener.wrap(r -> handler.accept(null), e -> { - try { - handler.accept(e); - } catch (IOException e1) { - logger.warn("Error while delagating exception [" + e.getMessage() + "]", e1); + persistentTasksService.updateStatus(taskId, state, new PersistentTaskOperationListener() { + @Override + public void onResponse(long taskId) { + try { + handler.accept(null); + } catch (IOException e1) { + logger.warn("Error while delegating response", e1); + } } - })); + + @Override + public void onFailure(Exception e) { + try { + handler.accept(e); + } catch (IOException e1) { + logger.warn("Error while delegating exception [" + e.getMessage() + "]", e1); + } + } + }); } public Optional> getStatistics(String jobId) { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java index bf349efa5bd..bb77ad1aa88 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java @@ -20,7 +20,6 @@ import org.elasticsearch.rest.action.RestBuilderListener; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; -import org.elasticsearch.xpack.persistent.PersistentActionResponse; import java.io.IOException; @@ -55,12 +54,12 @@ public class RestStartDatafeedAction extends BaseRestHandler { } return channel -> { client.execute(StartDatafeedAction.INSTANCE, jobDatafeedRequest, - new RestBuilderListener(channel) { + new RestBuilderListener(channel) { @Override - public RestResponse buildResponse(PersistentActionResponse r, XContentBuilder builder) throws Exception { + public RestResponse buildResponse(StartDatafeedAction.Response r, XContentBuilder builder) throws Exception { builder.startObject(); - builder.field("started", true); + builder.field("started", r.isAcknowledged()); builder.endObject(); return new BytesRestResponse(RestStatus.OK, builder); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java index 9807e618382..1b30097e805 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java @@ -19,7 +19,6 @@ import org.elasticsearch.rest.action.RestBuilderListener; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.action.OpenJobAction; import org.elasticsearch.xpack.ml.job.config.Job; -import org.elasticsearch.xpack.persistent.PersistentActionResponse; import java.io.IOException; @@ -45,12 +44,11 @@ public class RestOpenJobAction extends BaseRestHandler { } } return channel -> { - client.execute(OpenJobAction.INSTANCE, request, new RestBuilderListener(channel) { - + client.execute(OpenJobAction.INSTANCE, request, new RestBuilderListener(channel) { @Override - public RestResponse buildResponse(PersistentActionResponse r, XContentBuilder builder) throws Exception { + public RestResponse buildResponse(OpenJobAction.Response r, XContentBuilder builder) throws Exception { builder.startObject(); - builder.field("opened", true); + builder.field("opened", r.isAcknowledged()); builder.endObject(); return new BytesRestResponse(RestStatus.OK, builder); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/utils/DatafeedStateObserver.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/utils/DatafeedStateObserver.java index ce8251d2883..b75ffe3bced 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/utils/DatafeedStateObserver.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/utils/DatafeedStateObserver.java @@ -14,7 +14,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.MlMetadata; -import org.elasticsearch.xpack.persistent.PersistentTasks; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; import java.util.function.Consumer; import java.util.function.Predicate; @@ -35,7 +35,7 @@ public class DatafeedStateObserver { ClusterStateObserver observer = new ClusterStateObserver(clusterService, LOGGER, threadPool.getThreadContext()); Predicate predicate = (newState) -> { - PersistentTasks tasks = newState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeedId, tasks); return datafeedState == expectedState; }; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/utils/JobStateObserver.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/utils/JobStateObserver.java index b030b7d5e29..e7ef98437ab 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/utils/JobStateObserver.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/utils/JobStateObserver.java @@ -16,7 +16,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.job.config.JobState; -import org.elasticsearch.xpack.persistent.PersistentTasks; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; import java.util.function.Consumer; import java.util.function.Predicate; @@ -66,7 +66,7 @@ public class JobStateObserver { handler.accept(null); } } else { - PersistentTasks tasks = state.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); JobState actual = MlMetadata.getJobState(jobId, tasks); Exception e = new IllegalArgumentException("Timeout expired while waiting for job state [" + actual + "] to change to [" + expectedState + "]"); @@ -90,7 +90,7 @@ public class JobStateObserver { @Override public boolean test(ClusterState newState) { - PersistentTasks tasks = newState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); JobState jobState = MlMetadata.getJobState(jobId, tasks); if (jobState == JobState.FAILED) { failed = true; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/CompletionPersistentTaskAction.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/CompletionPersistentTaskAction.java index db20020951e..7f0d39da328 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/CompletionPersistentTaskAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/CompletionPersistentTaskAction.java @@ -105,6 +105,36 @@ public class CompletionPersistentTaskAction extends Action { - private final PersistentTaskClusterService persistentTaskClusterService; + private final PersistentTasksClusterService persistentTasksClusterService; @Inject public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - PersistentTaskClusterService persistentTaskClusterService, + PersistentTasksClusterService persistentTasksClusterService, IndexNameExpressionResolver indexNameExpressionResolver) { super(settings, CompletionPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); - this.persistentTaskClusterService = persistentTaskClusterService; + this.persistentTasksClusterService = persistentTasksClusterService; } @Override @@ -148,7 +178,7 @@ public class CompletionPersistentTaskAction extends Action listener) { - persistentTaskClusterService.completeOrRestartPersistentTask(request.taskId, request.exception, new ActionListener() { + persistentTasksClusterService.completeOrRestartPersistentTask(request.taskId, request.exception, new ActionListener() { @Override public void onResponse(Empty empty) { listener.onResponse(newResponse()); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/CreatePersistentTaskAction.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/CreatePersistentTaskAction.java index 1970c9c059e..3f1a059cba0 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/CreatePersistentTaskAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/CreatePersistentTaskAction.java @@ -34,7 +34,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; * This action can be used to add the record for the persistent action to the cluster state. */ public class CreatePersistentTaskAction extends Action { public static final CreatePersistentTaskAction INSTANCE = new CreatePersistentTaskAction(); @@ -50,15 +50,15 @@ public class CreatePersistentTaskAction extends Action { private String action; - private PersistentActionRequest request; + private PersistentTaskRequest request; private boolean stopped; @@ -68,7 +68,7 @@ public class CreatePersistentTaskAction extends Action { + PersistentTaskResponse, CreatePersistentTaskAction.RequestBuilder> { protected RequestBuilder(ElasticsearchClient client, CreatePersistentTaskAction action) { super(client, action, new Request()); @@ -166,8 +166,8 @@ public class CreatePersistentTaskAction extends Action { + public static class TransportAction extends TransportMasterNodeAction { - private final PersistentTaskClusterService persistentTaskClusterService; + private final PersistentTasksClusterService persistentTasksClusterService; @Inject public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - PersistentTaskClusterService persistentTaskClusterService, - PersistentActionRegistry persistentActionRegistry, - PersistentActionService persistentActionService, + PersistentTasksClusterService persistentTasksClusterService, + PersistentTasksExecutorRegistry persistentTasksExecutorRegistry, + PersistentTasksService persistentTasksService, IndexNameExpressionResolver indexNameExpressionResolver) { super(settings, CreatePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); - this.persistentTaskClusterService = persistentTaskClusterService; - PersistentActionExecutor executor = new PersistentActionExecutor(threadPool); - clusterService.addListener(new PersistentActionCoordinator(settings, persistentActionService, persistentActionRegistry, - transportService.getTaskManager(), executor)); + this.persistentTasksClusterService = persistentTasksClusterService; + NodePersistentTasksExecutor executor = new NodePersistentTasksExecutor(threadPool); + clusterService.addListener(new PersistentTasksNodeService(settings, persistentTasksService, persistentTasksExecutorRegistry, + transportService.getTaskManager(), threadPool, executor)); } @Override @@ -213,8 +213,8 @@ public class CreatePersistentTaskAction extends Action listener) { - persistentTaskClusterService.createPersistentTask(request.action, request.request, request.stopped, request.removeOnCompletion, + final ActionListener listener) { + persistentTasksClusterService.createPersistentTask(request.action, request.request, request.stopped, request.removeOnCompletion, new ActionListener() { @Override public void onResponse(Long newTaskId) { - listener.onResponse(new PersistentActionResponse(newTaskId)); + listener.onResponse(new PersistentTaskResponse(newTaskId)); } @Override diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionExecutor.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/NodePersistentTasksExecutor.java similarity index 68% rename from plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionExecutor.java rename to plugin/src/main/java/org/elasticsearch/xpack/persistent/NodePersistentTasksExecutor.java index 540caff25e8..de4802c24ca 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionExecutor.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/NodePersistentTasksExecutor.java @@ -11,20 +11,20 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; /** - * This component is responsible for execution of persistent actions. + * This component is responsible for execution of persistent tasks. */ -public class PersistentActionExecutor { +public class NodePersistentTasksExecutor { private final ThreadPool threadPool; - public PersistentActionExecutor(ThreadPool threadPool) { + public NodePersistentTasksExecutor(ThreadPool threadPool) { this.threadPool = threadPool; } - public void executeAction(Request request, - NodePersistentTask task, - PersistentActionRegistry.PersistentActionHolder holder, - ActionListener listener) { - threadPool.executor(holder.getExecutor()).execute(new AbstractRunnable() { + public void executeTask(Request request, + NodePersistentTask task, + PersistentTasksExecutor action, + ActionListener listener) { + threadPool.executor(action.getExecutor()).execute(new AbstractRunnable() { @Override public void onFailure(Exception e) { listener.onFailure(e); @@ -34,7 +34,7 @@ public class PersistentActionExecutor { @Override protected void doRun() throws Exception { try { - holder.getPersistentAction().nodeOperation(task, request, listener); + action.nodeOperation(task, request, listener); } catch (Exception ex) { listener.onFailure(ex); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRegistry.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRegistry.java deleted file mode 100644 index 155f87345cc..00000000000 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRegistry.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.persistent; - -import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.settings.Settings; - -import java.util.Collections; -import java.util.Map; - -/** - * Components that registers all persistent actions - */ -public class PersistentActionRegistry extends AbstractComponent { - - private volatile Map> actions = Collections.emptyMap(); - - private final Object actionHandlerMutex = new Object(); - - public PersistentActionRegistry(Settings settings) { - super(settings); - } - - public void registerPersistentAction(String action, - TransportPersistentAction persistentAction) { - registerPersistentAction(new PersistentActionHolder<>(action, persistentAction, persistentAction.getExecutor())); - } - - private void registerPersistentAction( - PersistentActionHolder reg) { - - synchronized (actionHandlerMutex) { - PersistentActionHolder replaced = actions.get(reg.getAction()); - actions = MapBuilder.newMapBuilder(actions).put(reg.getAction(), reg).immutableMap(); - if (replaced != null) { - logger.warn("registered two handlers for persistent action {}, handlers: {}, {}", reg.getAction(), reg, replaced); - } - } - } - - public void removeHandler(String action) { - synchronized (actionHandlerMutex) { - actions = MapBuilder.newMapBuilder(actions).remove(action).immutableMap(); - } - } - - @SuppressWarnings("unchecked") - public PersistentActionHolder getPersistentActionHolderSafe(String action) { - PersistentActionHolder holder = (PersistentActionHolder) actions.get(action); - if (holder == null) { - throw new IllegalStateException("Unknown persistent action [" + action + "]"); - } - return holder; - } - - public - TransportPersistentAction getPersistentActionSafe(String action) { - PersistentActionHolder holder = getPersistentActionHolderSafe(action); - return holder.getPersistentAction(); - } - - public static final class PersistentActionHolder { - - private final String action; - private final TransportPersistentAction persistentAction; - private final String executor; - - - public PersistentActionHolder(String action, TransportPersistentAction persistentAction, String executor) { - this.action = action; - this.persistentAction = persistentAction; - this.executor = executor; - } - - public String getAction() { - return action; - } - - public TransportPersistentAction getPersistentAction() { - return persistentAction; - } - - public String getExecutor() { - return executor; - } - } -} \ No newline at end of file diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionService.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionService.java deleted file mode 100644 index 9686ed9fd37..00000000000 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionService.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.persistent; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.threadpool.ThreadPool; - -/** - * Service responsible for executing restartable actions that can survive disappearance of a coordinating and executor nodes. - */ -public class PersistentActionService extends AbstractComponent { - - private final Client client; - private final ThreadPool threadPool; - private final ClusterService clusterService; - - public PersistentActionService(Settings settings, ThreadPool threadPool, ClusterService clusterService, Client client) { - super(settings); - this.client = client; - this.threadPool = threadPool; - this.clusterService = clusterService; - } - - public void sendRequest(String action, Request request, - ActionListener listener) { - CreatePersistentTaskAction.Request startRequest = new CreatePersistentTaskAction.Request(action, request); - try { - client.execute(CreatePersistentTaskAction.INSTANCE, startRequest, listener); - } catch (Exception e) { - listener.onFailure(e); - } - } - - public void sendCompletionNotification(long taskId, Exception failure, - ActionListener listener) { - CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure); - // Need to fork otherwise: java.lang.AssertionError: should not be called by a cluster state applier. - // reason [the applied cluster state is not yet available]) - try { - threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - - @Override - protected void doRun() throws Exception { - client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, listener); - } - }); - } catch (Exception e) { - listener.onFailure(e); - } - } - - public void sendCancellation(long taskId, ActionListener listener) { - DiscoveryNode localNode = clusterService.localNode(); - CancelTasksRequest cancelTasksRequest = new CancelTasksRequest(); - cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId)); - cancelTasksRequest.setReason("persistent action was removed"); - try { - client.admin().cluster().cancelTasks(cancelTasksRequest, listener); - } catch (Exception e) { - listener.onFailure(e); - } - } - - public void updateStatus(long taskId, Task.Status status, ActionListener listener) { - UpdatePersistentTaskStatusAction.Request updateStatusRequest = new UpdatePersistentTaskStatusAction.Request(taskId, status); - try { - client.execute(UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, listener); - } catch (Exception e) { - listener.onFailure(e); - } - } - -} \ No newline at end of file diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRequest.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskRequest.java similarity index 81% rename from plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRequest.java rename to plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskRequest.java index 3bec793a402..2f753fd8220 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRequest.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskRequest.java @@ -12,9 +12,9 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; /** - * Base class for a request for a persistent action + * Base class for a request for a persistent task */ -public abstract class PersistentActionRequest extends ActionRequest implements NamedWriteable, ToXContent { +public abstract class PersistentTaskRequest extends ActionRequest implements NamedWriteable, ToXContent { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId) { return new NodePersistentTask(id, type, action, getDescription(), parentTaskId); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionResponse.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskResponse.java similarity index 81% rename from plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionResponse.java rename to plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskResponse.java index 1c99ae085d0..af55acf8f04 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionResponse.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskResponse.java @@ -13,16 +13,16 @@ import java.io.IOException; import java.util.Objects; /** - * Response upon a successful start or an persistent action + * Response upon a successful start or an persistent task */ -public class PersistentActionResponse extends ActionResponse { +public class PersistentTaskResponse extends ActionResponse { private long taskId; - public PersistentActionResponse() { + public PersistentTaskResponse() { super(); } - public PersistentActionResponse(long taskId) { + public PersistentTaskResponse(long taskId) { this.taskId = taskId; } @@ -46,7 +46,7 @@ public class PersistentActionResponse extends ActionResponse { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - PersistentActionResponse that = (PersistentActionResponse) o; + PersistentTaskResponse that = (PersistentTaskResponse) o; return taskId == that.taskId; } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterService.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksClusterService.java similarity index 80% rename from plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterService.java rename to plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksClusterService.java index 0b74fafd3db..7a63bae2b1b 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterService.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksClusterService.java @@ -19,20 +19,20 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportResponse.Empty; -import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.util.Objects; /** * Component that runs only on the master node and is responsible for assigning running tasks to nodes */ -public class PersistentTaskClusterService extends AbstractComponent implements ClusterStateListener { +public class PersistentTasksClusterService extends AbstractComponent implements ClusterStateListener { private final ClusterService clusterService; - private final PersistentActionRegistry registry; + private final PersistentTasksExecutorRegistry registry; - public PersistentTaskClusterService(Settings settings, PersistentActionRegistry registry, ClusterService clusterService) { + public PersistentTasksClusterService(Settings settings, PersistentTasksExecutorRegistry registry, ClusterService clusterService) { super(settings); this.clusterService = clusterService; clusterService.addListener(this); @@ -47,15 +47,17 @@ public class PersistentTaskClusterService extends AbstractComponent implements C * @param request request * @param listener the listener that will be called when task is started */ - public void createPersistentTask(String action, Request request, boolean stopped, - boolean removeOnCompletion, - ActionListener listener) { + public void createPersistentTask(String action, Request request, boolean stopped, + boolean removeOnCompletion, + ActionListener listener) { clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { + validate(action, clusterService.state(), request); final Assignment assignment; if (stopped) { - assignment = PersistentTasks.FINISHED_TASK_ASSIGNMENT; // the task is stopped no need to assign it anywhere + // the task is stopped no need to assign it anywhere + assignment = PersistentTasksCustomMetaData.FINISHED_TASK_ASSIGNMENT; } else { assignment = getAssignement(action, currentState, request); } @@ -70,7 +72,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { listener.onResponse( - ((PersistentTasks) newState.getMetaData().custom(PersistentTasks.TYPE)).getCurrentId()); + ((PersistentTasksCustomMetaData) newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)).getCurrentId()); } }); } @@ -94,7 +96,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasks.Builder tasksInProgress = builder(currentState); + PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { if (failure != null) { // If the task failed - we need to restart it on another node, otherwise we just remove it @@ -132,7 +134,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C clusterService.submitStateUpdateTask("start persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasks.Builder tasksInProgress = builder(currentState); + PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { return update(currentState, tasksInProgress .assignTask(id, (action, request) -> getAssignement(action, currentState, request))); @@ -163,7 +165,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C clusterService.submitStateUpdateTask("remove persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasks.Builder tasksInProgress = builder(currentState); + PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { return update(currentState, tasksInProgress.removeTask(id)); } else { @@ -194,7 +196,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasks.Builder tasksInProgress = builder(currentState); + PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { return update(currentState, tasksInProgress.updateTaskStatus(id, status)); } else { @@ -214,10 +216,14 @@ public class PersistentTaskClusterService extends AbstractComponent implements C }); } - private Assignment getAssignement(String action, ClusterState currentState, Request request) { - TransportPersistentAction persistentAction = registry.getPersistentActionSafe(action); - persistentAction.validate(request, currentState); - return persistentAction.getAssignment(request, currentState); + private Assignment getAssignement(String taskName, ClusterState currentState, Request request) { + PersistentTasksExecutor persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName); + return persistentTasksExecutor.getAssignment(request, currentState); + } + + private void validate(String taskName, ClusterState currentState, Request request) { + PersistentTasksExecutor persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName); + persistentTasksExecutor.validate(request, currentState); } @Override @@ -234,12 +240,12 @@ public class PersistentTaskClusterService extends AbstractComponent implements C } interface ExecutorNodeDecider { - Assignment getAssignment(String action, ClusterState currentState, Request request); + Assignment getAssignment(String action, ClusterState currentState, Request request); } static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecider decider) { - PersistentTasks tasks = event.state().getMetaData().custom(PersistentTasks.TYPE); - PersistentTasks prevTasks = event.previousState().getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + PersistentTasksCustomMetaData prevTasks = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); if (tasks != null && (Objects.equals(tasks, prevTasks) == false || event.nodesChanged() || event.routingTableChanged() || @@ -250,7 +256,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C if (taskInProgress.needsReassignment(event.state().nodes())) { // there is an unassigned task or task with a disappeared node - we need to try assigning it if (Objects.equals(taskInProgress.getAssignment(), - decider.getAssignment(taskInProgress.getAction(), event.state(), taskInProgress.getRequest())) == false) { + decider.getAssignment(taskInProgress.getTaskName(), event.state(), taskInProgress.getRequest())) == false) { // it looks like a assignment for at least one task is possible - let's trigger reassignment reassignmentRequired = true; break; @@ -270,7 +276,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C clusterService.submitStateUpdateTask("reassign persistent tasks", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - return reassignTasks(currentState, logger, PersistentTaskClusterService.this::getAssignement); + return reassignTasks(currentState, logger, PersistentTasksClusterService.this::getAssignement); } @Override @@ -286,7 +292,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C } static ClusterState reassignTasks(ClusterState currentState, Logger logger, ExecutorNodeDecider decider) { - PersistentTasks tasks = currentState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); ClusterState clusterState = currentState; DiscoveryNodes nodes = currentState.nodes(); if (tasks != null) { @@ -295,7 +301,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C for (PersistentTask task : tasks.tasks()) { if (task.needsReassignment(nodes)) { // there is an unassigned task - we need to try assigning it - Assignment assignment = decider.getAssignment(task.getAction(), clusterState, task.getRequest()); + Assignment assignment = decider.getAssignment(task.getTaskName(), clusterState, task.getRequest()); if (Objects.equals(assignment, task.getAssignment()) == false) { logger.trace("reassigning task {} from node {} to node {}", task.getId(), task.getAssignment().getExecutorNode(), assignment.getExecutorNode()); @@ -315,14 +321,14 @@ public class PersistentTaskClusterService extends AbstractComponent implements C return clusterState; } - private static PersistentTasks.Builder builder(ClusterState currentState) { - return PersistentTasks.builder(currentState.getMetaData().custom(PersistentTasks.TYPE)); + private static PersistentTasksCustomMetaData.Builder builder(ClusterState currentState) { + return PersistentTasksCustomMetaData.builder(currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); } - private static ClusterState update(ClusterState currentState, PersistentTasks.Builder tasksInProgress) { + private static ClusterState update(ClusterState currentState, PersistentTasksCustomMetaData.Builder tasksInProgress) { if (tasksInProgress.isChanged()) { return ClusterState.builder(currentState).metaData( - MetaData.builder(currentState.metaData()).putCustom(PersistentTasks.TYPE, tasksInProgress.build()) + MetaData.builder(currentState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasksInProgress.build()) ).build(); } else { return currentState; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasks.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksCustomMetaData.java similarity index 84% rename from plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasks.java rename to plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksCustomMetaData.java index 1205432319c..f8decae706f 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasks.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksCustomMetaData.java @@ -43,7 +43,7 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constru /** * A cluster state record that contains a list of all running persistent tasks */ -public final class PersistentTasks extends AbstractNamedDiffable implements MetaData.Custom { +public final class PersistentTasksCustomMetaData extends AbstractNamedDiffable implements MetaData.Custom { public static final String TYPE = "persistent_tasks"; private static final String API_CONTEXT = MetaData.XContentContext.API.toString(); @@ -53,21 +53,21 @@ public final class PersistentTasks extends AbstractNamedDiffable> tasks) { + public PersistentTasksCustomMetaData(long currentId, Map> tasks) { this.currentId = currentId; this.tasks = tasks; } private static final ObjectParser PERSISTENT_TASKS_PARSER = new ObjectParser<>(TYPE, Builder::new); - private static final ObjectParser, Void> PERSISTENT_TASK_PARSER = + private static final ObjectParser, Void> PERSISTENT_TASK_PARSER = new ObjectParser<>("tasks", TaskBuilder::new); public static final ConstructingObjectParser ASSIGNMENT_PARSER = new ConstructingObjectParser<>("assignment", objects -> new Assignment((String) objects[0], (String) objects[1])); - private static final NamedObjectParser REQUEST_PARSER = - (XContentParser p, Void c, String name) -> p.namedObject(PersistentActionRequest.class, name, null); + private static final NamedObjectParser REQUEST_PARSER = + (XContentParser p, Void c, String name) -> p.namedObject(PersistentTaskRequest.class, name, null); private static final NamedObjectParser STATUS_PARSER = (XContentParser p, Void c, String name) -> p.namedObject(Status.class, name, null); @@ -82,20 +82,20 @@ public final class PersistentTasks extends AbstractNamedDiffable taskBuilder, List objects) -> { + (TaskBuilder taskBuilder, List objects) -> { if (objects.size() != 1) { - throw new IllegalArgumentException("only one action request per task is allowed"); + throw new IllegalArgumentException("only one request per task is allowed"); } taskBuilder.setRequest(objects.get(0)); }, REQUEST_PARSER, new ParseField("request")); PERSISTENT_TASK_PARSER.declareNamedObjects( - (TaskBuilder taskBuilder, List objects) -> { + (TaskBuilder taskBuilder, List objects) -> { if (objects.size() != 1) { throw new IllegalArgumentException("only one status per task is allowed"); } @@ -120,16 +120,16 @@ public final class PersistentTasks extends AbstractNamedDiffable> findTasks(String actionName, Predicate> predicate) { + public Collection> findTasks(String taskName, Predicate> predicate) { return this.tasks().stream() - .filter(p -> actionName.equals(p.getAction())) + .filter(p -> taskName.equals(p.getTaskName())) .filter(predicate) .collect(Collectors.toList()); } - public boolean tasksExist(String actionName, Predicate> predicate) { + public boolean tasksExist(String taskName, Predicate> predicate) { return this.tasks().stream() - .filter(p -> actionName.equals(p.getAction())) + .filter(p -> taskName.equals(p.getTaskName())) .anyMatch(predicate); } @@ -137,7 +137,7 @@ public final class PersistentTasks extends AbstractNamedDiffable action.equals(task.action) && nodeId.equals(task.assignment.executorNode)).count(); + public long getNumberOfTasksOnNode(String nodeId, String taskName) { + return tasks.values().stream().filter( + task -> taskName.equals(task.taskName) && nodeId.equals(task.assignment.executorNode)).count(); } @Override @@ -166,7 +167,7 @@ public final class PersistentTasks extends AbstractNamedDiffable implements Writeable, ToXContent { + public static class PersistentTask implements Writeable, ToXContent { private final long id; private final long allocationId; - private final String action; + private final String taskName; private final Request request; private final boolean stopped; private final boolean removeOnCompletion; @@ -235,25 +236,25 @@ public final class PersistentTasks extends AbstractNamedDiffable task, boolean stopped, Assignment assignment) { - this(task.id, task.allocationId + 1L, task.action, task.request, stopped, task.removeOnCompletion, task.status, + this(task.id, task.allocationId + 1L, task.taskName, task.request, stopped, task.removeOnCompletion, task.status, assignment, task.allocationId); } public PersistentTask(PersistentTask task, Status status) { - this(task.id, task.allocationId, task.action, task.request, task.stopped, task.removeOnCompletion, status, + this(task.id, task.allocationId, task.taskName, task.request, task.stopped, task.removeOnCompletion, status, task.assignment, task.allocationId); } - private PersistentTask(long id, long allocationId, String action, Request request, boolean stopped, boolean removeOnCompletion, + private PersistentTask(long id, long allocationId, String taskName, Request request, boolean stopped, boolean removeOnCompletion, Status status, Assignment assignment, Long allocationIdOnLastStatusUpdate) { this.id = id; this.allocationId = allocationId; - this.action = action; + this.taskName = taskName; this.request = request; this.status = status; this.stopped = stopped; @@ -268,8 +269,8 @@ public final class PersistentTasks extends AbstractNamedDiffable that = (PersistentTask) o; return id == that.id && allocationId == that.allocationId && - Objects.equals(action, that.action) && + Objects.equals(taskName, that.taskName) && Objects.equals(request, that.request) && stopped == that.stopped && removeOnCompletion == that.removeOnCompletion && @@ -309,7 +310,7 @@ public final class PersistentTasks extends AbstractNamedDiffable { + private static class TaskBuilder { private long id; private long allocationId; - private String action; + private String taskName; private Request request; private boolean stopped = true; private boolean removeOnCompletion; @@ -441,8 +442,8 @@ public final class PersistentTasks extends AbstractNamedDiffable setAction(String action) { - this.action = action; + public TaskBuilder setTaskName(String taskName) { + this.taskName = taskName; return this; } @@ -478,7 +479,7 @@ public final class PersistentTasks extends AbstractNamedDiffable build() { - return new PersistentTask<>(id, allocationId, action, request, stopped, removeOnCompletion, status, + return new PersistentTask<>(id, allocationId, taskName, request, stopped, removeOnCompletion, status, assignment, allocationIdOnLastStatusUpdate); } } @@ -488,7 +489,7 @@ public final class PersistentTasks extends AbstractNamedDiffable Builder setTasks(List> tasks) { + private Builder setTasks(List> tasks) { for (TaskBuilder builder : tasks) { PersistentTask task = builder.build(); this.tasks.put(task.getId(), task); @@ -564,11 +565,11 @@ public final class PersistentTasks extends AbstractNamedDiffable * After the task is added its id can be found by calling {{@link #getCurrentId()}} method. */ - public Builder addTask(String action, Request request, boolean stopped, - boolean removeOnCompletion, Assignment assignment) { + public Builder addTask(String taskName, Request request, boolean stopped, + boolean removeOnCompletion, Assignment assignment) { changed = true; currentId++; - tasks.put(currentId, new PersistentTask<>(currentId, action, request, stopped, removeOnCompletion, assignment)); + tasks.put(currentId, new PersistentTask<>(currentId, taskName, request, stopped, removeOnCompletion, assignment)); return this; } @@ -591,11 +592,11 @@ public final class PersistentTasks extends AbstractNamedDiffable Builder assignTask(long taskId, - BiFunction executorNodeFunc) { + public Builder assignTask(long taskId, + BiFunction executorNodeFunc) { PersistentTask taskInProgress = (PersistentTask) tasks.get(taskId); if (taskInProgress != null && taskInProgress.assignment.isAssigned() == false) { // only assign unassigned tasks - Assignment assignment = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); + Assignment assignment = executorNodeFunc.apply(taskInProgress.taskName, taskInProgress.request); if (assignment.isAssigned() || taskInProgress.isStopped()) { changed = true; tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment)); @@ -608,12 +609,12 @@ public final class PersistentTasks extends AbstractNamedDiffable Builder reassignTask(long taskId, - BiFunction executorNodeFunc) { + public Builder reassignTask(long taskId, + BiFunction executorNodeFunc) { PersistentTask taskInProgress = (PersistentTask) tasks.get(taskId); if (taskInProgress != null) { changed = true; - Assignment assignment = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); + Assignment assignment = executorNodeFunc.apply(taskInProgress.taskName, taskInProgress.request); tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment)); } return this; @@ -680,8 +681,8 @@ public final class PersistentTasks extends AbstractNamedDiffable - extends HandledTransportAction { +public abstract class PersistentTasksExecutor extends AbstractComponent { private final String executor; - private final PersistentActionService persistentActionService; + private final String taskName; + private final PersistentTasksService persistentTasksService; - protected TransportPersistentAction(Settings settings, String actionName, boolean canTripCircuitBreaker, ThreadPool threadPool, - TransportService transportService, PersistentActionService persistentActionService, - PersistentActionRegistry persistentActionRegistry, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier requestSupplier, String executor) { - super(settings, actionName, canTripCircuitBreaker, threadPool, transportService, actionFilters, indexNameExpressionResolver, - requestSupplier); + protected PersistentTasksExecutor(Settings settings, String taskName, PersistentTasksService persistentTasksService, + String executor) { + super(settings); + this.taskName = taskName; this.executor = executor; - this.persistentActionService = persistentActionService; - persistentActionRegistry.registerPersistentAction(actionName, this); + this.persistentTasksService = persistentTasksService; + } + + public String getTaskName() { + return taskName; } public static final Assignment NO_NODE_FOUND = new Assignment(null, "no appropriate nodes found for the assignment"); @@ -59,21 +55,20 @@ public abstract class TransportPersistentAction selector) { long minLoad = Long.MAX_VALUE; DiscoveryNode minLoadedNode = null; - PersistentTasks persistentTasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData persistentTasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); for (DiscoveryNode node : clusterState.getNodes()) { if (selector.test(node)) { if (persistentTasks == null) { // We don't have any task running yet, pick the first available node return node; } - long numberOfTasks = persistentTasks.getNumberOfTasksOnNode(node.getId(), actionName); + long numberOfTasks = persistentTasks.getNumberOfTasksOnNode(node.getId(), taskName); if (minLoad > numberOfTasks) { minLoad = numberOfTasks; minLoadedNode = node; @@ -92,11 +87,6 @@ public abstract class TransportPersistentAction listener) { - persistentActionService.sendRequest(actionName, request, listener); - } - /** * Updates the persistent task status in the cluster state. *

@@ -104,10 +94,10 @@ public abstract class TransportPersistentAction listener) { - persistentActionService.updateStatus(task.getPersistentTaskId(), status, - new ActionListener() { + persistentTasksService.updateStatus(task.getPersistentTaskId(), status, + new PersistentTaskOperationListener() { @Override - public void onResponse(UpdatePersistentTaskStatusAction.Response response) { + public void onResponse(long taskId) { listener.onResponse(Empty.INSTANCE); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorRegistry.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorRegistry.java new file mode 100644 index 00000000000..25781c66a7d --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorRegistry.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Components that registers all persistent task executors + */ +public class PersistentTasksExecutorRegistry extends AbstractComponent { + + private final Map> taskExecutors; + + @SuppressWarnings("unchecked") + public PersistentTasksExecutorRegistry(Settings settings, Collection> taskExecutors) { + super(settings); + Map> map = new HashMap<>(); + for (PersistentTasksExecutor executor : taskExecutors) { + map.put(executor.getTaskName(), executor); + } + this.taskExecutors = Collections.unmodifiableMap(map); + } + + @SuppressWarnings("unchecked") + public PersistentTasksExecutor getPersistentTaskExecutorSafe(String taskName) { + PersistentTasksExecutor executor = (PersistentTasksExecutor) taskExecutors.get(taskName); + if (executor == null) { + throw new IllegalStateException("Unknown persistent executor [" + taskName + "]"); + } + return executor; + } +} \ No newline at end of file diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinator.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeService.java similarity index 80% rename from plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinator.java rename to plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeService.java index 34156bb5017..063bce14ca4 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinator.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeService.java @@ -9,7 +9,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.common.Nullable; @@ -19,12 +18,15 @@ import org.elasticsearch.common.inject.Provider; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.HashMap; @@ -37,33 +39,35 @@ import java.util.concurrent.atomic.AtomicReference; import static java.util.Objects.requireNonNull; /** - * This component is responsible for coordination of execution of persistent actions on individual nodes. It runs on all + * This component is responsible for coordination of execution of persistent tasks on individual nodes. It runs on all * non-transport client nodes in the cluster and monitors cluster state changes to detect started commands. */ -public class PersistentActionCoordinator extends AbstractComponent implements ClusterStateListener { +public class PersistentTasksNodeService extends AbstractComponent implements ClusterStateListener { private final Map runningTasks = new HashMap<>(); - private final PersistentActionService persistentActionService; - private final PersistentActionRegistry persistentActionRegistry; + private final PersistentTasksService persistentTasksService; + private final PersistentTasksExecutorRegistry persistentTasksExecutorRegistry; private final TaskManager taskManager; - private final PersistentActionExecutor persistentActionExecutor; + private final ThreadPool threadPool; + private final NodePersistentTasksExecutor nodePersistentTasksExecutor; - public PersistentActionCoordinator(Settings settings, - PersistentActionService persistentActionService, - PersistentActionRegistry persistentActionRegistry, - TaskManager taskManager, - PersistentActionExecutor persistentActionExecutor) { + public PersistentTasksNodeService(Settings settings, + PersistentTasksService persistentTasksService, + PersistentTasksExecutorRegistry persistentTasksExecutorRegistry, + TaskManager taskManager, ThreadPool threadPool, + NodePersistentTasksExecutor nodePersistentTasksExecutor) { super(settings); - this.persistentActionService = persistentActionService; - this.persistentActionRegistry = persistentActionRegistry; + this.persistentTasksService = persistentTasksService; + this.persistentTasksExecutorRegistry = persistentTasksExecutorRegistry; this.taskManager = taskManager; - this.persistentActionExecutor = persistentActionExecutor; + this.threadPool = threadPool; + this.nodePersistentTasksExecutor = nodePersistentTasksExecutor; } @Override public void clusterChanged(ClusterChangedEvent event) { - PersistentTasks tasks = event.state().getMetaData().custom(PersistentTasks.TYPE); - PersistentTasks previousTasks = event.previousState().getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + PersistentTasksCustomMetaData previousTasks = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) { // We have some changes let's check if they are related to our node @@ -111,10 +115,9 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl } - private void startTask(PersistentTask taskInProgress) { - PersistentActionRegistry.PersistentActionHolder holder = - persistentActionRegistry.getPersistentActionHolderSafe(taskInProgress.getAction()); - NodePersistentTask task = (NodePersistentTask) taskManager.register("persistent", taskInProgress.getAction() + "[c]", + private void startTask(PersistentTask taskInProgress) { + PersistentTasksExecutor action = persistentTasksExecutorRegistry.getPersistentTaskExecutorSafe(taskInProgress.getTaskName()); + NodePersistentTask task = (NodePersistentTask) taskManager.register("persistent", taskInProgress.getTaskName() + "[c]", taskInProgress.getRequest()); boolean processed = false; try { @@ -124,7 +127,7 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl PersistentTaskListener listener = new PersistentTaskListener(runningPersistentTask); try { runningTasks.put(new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()), runningPersistentTask); - persistentActionExecutor.executeAction(taskInProgress.getRequest(), task, holder, listener); + nodePersistentTasksExecutor.executeTask(taskInProgress.getRequest(), task, action, listener); } catch (Exception e) { // Submit task failure listener.onFailure(e); @@ -149,9 +152,11 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl RunningPersistentTask task = runningTasks.remove(persistentTaskId); if (task != null && task.getTask() != null) { if (task.markAsCancelled()) { - persistentActionService.sendCancellation(task.getTask().getId(), new ActionListener() { + persistentTasksService.sendCancellation(task.getTask().getId(), new PersistentTaskOperationListener() { @Override - public void onResponse(CancelTasksResponse cancelTasksResponse) { + public void onResponse(long taskId) { + logger.trace("Persistent task with id {} was cancelled", taskId); + } @Override @@ -171,7 +176,24 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl taskManager.unregister(task.getTask()); } else { if (task.restartCompletionNotification()) { - persistentActionService.sendCompletionNotification(task.getId(), task.getFailure(), new PublishedResponseListener(task)); + // Need to fork otherwise: java.lang.AssertionError: should not be called by a cluster state applier. + // reason [the applied cluster state is not yet available]) + PublishedResponseListener listener = new PublishedResponseListener(task); + try { + threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + + @Override + protected void doRun() throws Exception { + persistentTasksService.sendCompletionNotification(task.getId(), task.getFailure(), listener); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } } else { logger.warn("attempt to resend notification for task {} in the {} state", task.getId(), task.getState()); } @@ -184,7 +206,7 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl } else { logger.trace("sending notification for failed task {}", task.getId()); if (task.startNotification(e)) { - persistentActionService.sendCompletionNotification(task.getId(), e, new PublishedResponseListener(task)); + persistentTasksService.sendCompletionNotification(task.getId(), e, new PublishedResponseListener(task)); } else { logger.warn("attempt to send notification for task {} in the {} state", task.getId(), task.getState()); } @@ -223,7 +245,7 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl } } - private class PublishedResponseListener implements ActionListener { + private class PublishedResponseListener implements PersistentTaskOperationListener { private final RunningPersistentTask task; PublishedResponseListener(final RunningPersistentTask task) { @@ -232,7 +254,7 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl @Override - public void onResponse(CompletionPersistentTaskAction.Response response) { + public void onResponse(long taskId) { logger.trace("notification for task {} was successful", task.getId()); if (task.markAsNotified() == false) { logger.warn("attempt to mark task {} in the {} state as NOTIFIED", task.getId(), task.getState()); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksService.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksService.java new file mode 100644 index 00000000000..e636393d751 --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksService.java @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; + +/** + * This service is used by persistent actions to propagate changes in the action state and notify about completion + */ +public class PersistentTasksService extends AbstractComponent { + + private final Client client; + private final ClusterService clusterService; + + public PersistentTasksService(Settings settings, ClusterService clusterService, Client client) { + super(settings); + this.client = client; + this.clusterService = clusterService; + } + + /** + * Creates the specified persistent action and tries to start it immediately, upon completion the task is + * removed from the cluster state + */ + public void createPersistentActionTask(String action, Request request, + PersistentTaskOperationListener listener) { + createPersistentActionTask(action, request, false, true, listener); + } + + /** + * Creates the specified persistent action. The action is started unless the stopped parameter is equal to true. + * If removeOnCompletion parameter is equal to true, the task is removed from the cluster state upon completion. + * Otherwise it will remain there in the stopped state. + */ + public void createPersistentActionTask(String action, Request request, + boolean stopped, + boolean removeOnCompletion, + PersistentTaskOperationListener listener) { + CreatePersistentTaskAction.Request createPersistentActionRequest = new CreatePersistentTaskAction.Request(action, request); + createPersistentActionRequest.setStopped(stopped); + createPersistentActionRequest.setRemoveOnCompletion(removeOnCompletion); + try { + client.execute(CreatePersistentTaskAction.INSTANCE, createPersistentActionRequest, ActionListener.wrap( + o -> listener.onResponse(o.getTaskId()), listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Notifies the PersistentTasksClusterService about successful (failure == null) completion of a task or its failure + * + */ + public void sendCompletionNotification(long taskId, Exception failure, PersistentTaskOperationListener listener) { + CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure); + try { + client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, ActionListener.wrap(o -> listener.onResponse(taskId), + listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Cancels the persistent task. + */ + public void sendCancellation(long taskId, PersistentTaskOperationListener listener) { + DiscoveryNode localNode = clusterService.localNode(); + CancelTasksRequest cancelTasksRequest = new CancelTasksRequest(); + cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId)); + cancelTasksRequest.setReason("persistent action was removed"); + try { + client.admin().cluster().cancelTasks(cancelTasksRequest, ActionListener.wrap(o -> listener.onResponse(taskId), + listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Updates status of the persistent task + */ + public void updateStatus(long taskId, Task.Status status, PersistentTaskOperationListener listener) { + UpdatePersistentTaskStatusAction.Request updateStatusRequest = new UpdatePersistentTaskStatusAction.Request(taskId, status); + try { + client.execute(UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, ActionListener.wrap( + o -> listener.onResponse(taskId), listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Removes a persistent task + */ + public void removeTask(long taskId, PersistentTaskOperationListener listener) { + RemovePersistentTaskAction.Request removeRequest = new RemovePersistentTaskAction.Request(taskId); + try { + client.execute(RemovePersistentTaskAction.INSTANCE, removeRequest, ActionListener.wrap(o -> listener.onResponse(taskId), + listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Starts a persistent task + */ + public void startTask(long taskId, PersistentTaskOperationListener listener) { + StartPersistentTaskAction.Request startRequest = new StartPersistentTaskAction.Request(taskId); + try { + client.execute(StartPersistentTaskAction.INSTANCE, startRequest, ActionListener.wrap(o -> listener.onResponse(taskId), + listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public interface PersistentTaskOperationListener { + void onResponse(long taskId); + void onFailure(Exception e); + } + +} \ No newline at end of file diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/RemovePersistentTaskAction.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/RemovePersistentTaskAction.java index 180defc03b0..d43ab432929 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/RemovePersistentTaskAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/RemovePersistentTaskAction.java @@ -148,16 +148,16 @@ public class RemovePersistentTaskAction extends Action { - private final PersistentTaskClusterService persistentTaskClusterService; + private final PersistentTasksClusterService persistentTasksClusterService; @Inject public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - PersistentTaskClusterService persistentTaskClusterService, + PersistentTasksClusterService persistentTasksClusterService, IndexNameExpressionResolver indexNameExpressionResolver) { super(settings, RemovePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); - this.persistentTaskClusterService = persistentTaskClusterService; + this.persistentTasksClusterService = persistentTasksClusterService; } @Override @@ -178,7 +178,7 @@ public class RemovePersistentTaskAction extends Action listener) { - persistentTaskClusterService.removePersistentTask(request.taskId, new ActionListener() { + persistentTasksClusterService.removePersistentTask(request.taskId, new ActionListener() { @Override public void onResponse(Empty empty) { listener.onResponse(new Response(true)); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/StartPersistentTaskAction.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/StartPersistentTaskAction.java index 90a7b7023a9..d52c6f77693 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/StartPersistentTaskAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/StartPersistentTaskAction.java @@ -31,7 +31,7 @@ import java.io.IOException; import java.util.Objects; /** - * This action can be used to start persistent action previously created using {@link CreatePersistentTaskAction} + * This action can be used to start a persistent task previously created using {@link CreatePersistentTaskAction} */ public class StartPersistentTaskAction extends Action { - private final PersistentTaskClusterService persistentTaskClusterService; + private final PersistentTasksClusterService persistentTasksClusterService; @Inject public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - PersistentTaskClusterService persistentTaskClusterService, + PersistentTasksClusterService persistentTasksClusterService, IndexNameExpressionResolver indexNameExpressionResolver) { super(settings, StartPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); - this.persistentTaskClusterService = persistentTaskClusterService; + this.persistentTasksClusterService = persistentTasksClusterService; } @Override @@ -181,7 +181,7 @@ public class StartPersistentTaskAction extends Action listener) { - persistentTaskClusterService.startPersistentTask(request.taskId, new ActionListener() { + persistentTasksClusterService.startPersistentTask(request.taskId, new ActionListener() { @Override public void onResponse(Empty empty) { listener.onResponse(new Response(true)); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskStatusAction.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskStatusAction.java index d068a647ab0..f40dd323c93 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskStatusAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskStatusAction.java @@ -164,16 +164,16 @@ public class UpdatePersistentTaskStatusAction extends Action { - private final PersistentTaskClusterService persistentTaskClusterService; + private final PersistentTasksClusterService persistentTasksClusterService; @Inject public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - PersistentTaskClusterService persistentTaskClusterService, + PersistentTasksClusterService persistentTasksClusterService, IndexNameExpressionResolver indexNameExpressionResolver) { super(settings, UpdatePersistentTaskStatusAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); - this.persistentTaskClusterService = persistentTaskClusterService; + this.persistentTasksClusterService = persistentTasksClusterService; } @Override @@ -194,7 +194,7 @@ public class UpdatePersistentTaskStatusAction extends Action listener) { - persistentTaskClusterService.updatePersistentTaskStatus(request.taskId, request.status, new ActionListener() { + persistentTasksClusterService.updatePersistentTaskStatus(request.taskId, request.status, new ActionListener() { @Override public void onResponse(Empty empty) { listener.onResponse(new Response(true)); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/persistent/package-info.java b/plugin/src/main/java/org/elasticsearch/xpack/persistent/package-info.java index d97640dee02..8bcd033eb7e 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/persistent/package-info.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/persistent/package-info.java @@ -5,31 +5,30 @@ */ /** - * The Persistent Actions are actions responsible for executing restartable actions that can survive disappearance of a + * The Persistent Tasks Executors are responsible for executing restartable tasks that can survive disappearance of a * coordinating and executor nodes. *

- * In order to be resilient to node restarts, the persistent actions are using the cluster state instead of a transport service to send + * In order to be resilient to node restarts, the persistent tasks are using the cluster state instead of a transport service to send * requests and responses. The execution is done in six phases: *

- * 1. The coordinating node sends an ordinary transport request to the master node to start a new persistent action. This action is handled - * by the {@link org.elasticsearch.xpack.persistent.PersistentActionService}, which is using - * {@link org.elasticsearch.xpack.persistent.PersistentTaskClusterService} to update cluster state with the record about running persistent + * 1. The coordinating node sends an ordinary transport request to the master node to start a new persistent task. This task is handled + * by the {@link org.elasticsearch.xpack.persistent.PersistentTasksService}, which is using + * {@link org.elasticsearch.xpack.persistent.PersistentTasksClusterService} to update cluster state with the record about running persistent * task. *

- * 2. The master node updates the {@link org.elasticsearch.xpack.persistent.PersistentTasks} in the cluster state to indicate that - * there is a new persistent action - * running in the system. + * 2. The master node updates the {@link org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData} in the cluster state to indicate + * that there is a new persistent task is running in the system. *

- * 3. The {@link org.elasticsearch.xpack.persistent.PersistentActionCoordinator} running on every node in the cluster monitors changes in - * the cluster state and starts execution of all new actions assigned to the node it is running on. + * 3. The {@link org.elasticsearch.xpack.persistent.PersistentTasksNodeService} running on every node in the cluster monitors changes in + * the cluster state and starts execution of all new tasks assigned to the node it is running on. *

- * 4. If the action fails to start on the node, the {@link org.elasticsearch.xpack.persistent.PersistentActionCoordinator} uses the - * {@link org.elasticsearch.xpack.persistent.PersistentTasks} to notify the - * {@link org.elasticsearch.xpack.persistent.PersistentActionService}, which reassigns the action to another node in the cluster. + * 4. If the task fails to start on the node, the {@link org.elasticsearch.xpack.persistent.PersistentTasksNodeService} uses the + * {@link org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData} to notify the + * {@link org.elasticsearch.xpack.persistent.PersistentTasksService}, which reassigns the action to another node in the cluster. *

- * 5. If action finishes successfully on the node and calls listener.onResponse(), the corresponding persistent action is removed from the - * cluster state unless . + * 5. If a task finishes successfully on the node and calls listener.onResponse(), the corresponding persistent action is removed from the + * cluster state unless removeOnCompletion flag for this task is set to false. *

- * 6. The {@link org.elasticsearch.xpack.persistent.RemovePersistentTaskAction} action can be also used to remove the persistent action. + * 6. The {@link org.elasticsearch.xpack.persistent.RemovePersistentTaskAction} action can be also used to remove the persistent task. */ package org.elasticsearch.xpack.persistent; \ No newline at end of file diff --git a/plugin/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java b/plugin/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java index 59b982fb4ef..0d8903fdc72 100644 --- a/plugin/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java +++ b/plugin/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java @@ -28,8 +28,7 @@ import org.elasticsearch.xpack.ml.client.MachineLearningClient; import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; -import org.elasticsearch.xpack.persistent.PersistentActionResponse; -import org.elasticsearch.xpack.persistent.PersistentTasks; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; import org.junit.Before; import java.util.Collections; @@ -99,7 +98,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase { // test that license restricted apis do not work try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) { client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress()); - PlainListenableActionFuture listener = new PlainListenableActionFuture<>(client.threadPool()); + PlainListenableActionFuture listener = new PlainListenableActionFuture<>(client.threadPool()); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), listener); listener.actionGet(); fail("open job action should not be enabled!"); @@ -123,9 +122,9 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase { // test that license restricted apis do now work try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) { client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress()); - PlainListenableActionFuture listener = new PlainListenableActionFuture<>(client.threadPool()); + PlainListenableActionFuture listener = new PlainListenableActionFuture<>(client.threadPool()); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), listener); - PersistentActionResponse response = listener.actionGet(); + OpenJobAction.Response response = listener.actionGet(); assertNotNull(response); } } @@ -192,12 +191,12 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase { PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet(); assertNotNull(putDatafeedResponse); // open job - PlainListenableActionFuture openJobListener = new PlainListenableActionFuture<>(client.threadPool()); + PlainListenableActionFuture openJobListener = new PlainListenableActionFuture<>(client.threadPool()); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener); - PersistentActionResponse openJobResponse = openJobListener.actionGet(); + OpenJobAction.Response openJobResponse = openJobListener.actionGet(); assertNotNull(openJobResponse); // start datafeed - PlainListenableActionFuture listener = new PlainListenableActionFuture<>(client.threadPool()); + PlainListenableActionFuture listener = new PlainListenableActionFuture<>(client.threadPool()); new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener); listener.actionGet(); } @@ -218,7 +217,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase { assertEquals(DatafeedState.STOPPED, datafeedState); ClusterState state = client().admin().cluster().prepareState().get().getState(); - PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE); assertEquals(0, tasks.taskMap().size()); }); @@ -228,12 +227,12 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase { try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) { client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress()); // open job - PlainListenableActionFuture openJobListener = new PlainListenableActionFuture<>(client.threadPool()); + PlainListenableActionFuture openJobListener = new PlainListenableActionFuture<>(client.threadPool()); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener); - PersistentActionResponse openJobResponse = openJobListener.actionGet(); + OpenJobAction.Response openJobResponse = openJobListener.actionGet(); assertNotNull(openJobResponse); // start datafeed - PlainListenableActionFuture listener = new PlainListenableActionFuture<>(client.threadPool()); + PlainListenableActionFuture listener = new PlainListenableActionFuture<>(client.threadPool()); new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener); listener.actionGet(); } @@ -246,7 +245,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase { assertEquals(DatafeedState.STARTED, datafeedState); ClusterState state = client().admin().cluster().prepareState().get().getState(); - PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE); assertEquals(2, tasks.taskMap().size()); }); @@ -266,7 +265,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase { assertEquals(DatafeedState.STOPPED, datafeedState); ClusterState state = client().admin().cluster().prepareState().get().getState(); - PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE); assertEquals(0, tasks.taskMap().size()); }); } @@ -287,9 +286,9 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase { new PutDatafeedAction.Request(createDatafeed("foobar", "foo", Collections.singletonList("foo"))), putDatafeedListener); PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet(); assertNotNull(putDatafeedResponse); - PlainListenableActionFuture openJobListener = new PlainListenableActionFuture<>(client.threadPool()); + PlainListenableActionFuture openJobListener = new PlainListenableActionFuture<>(client.threadPool()); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener); - PersistentActionResponse openJobResponse = openJobListener.actionGet(); + OpenJobAction.Response openJobResponse = openJobListener.actionGet(); assertNotNull(openJobResponse); } @@ -303,14 +302,14 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase { JobState jobState = getJobStats("foo").getState(); assertEquals(JobState.CLOSED, jobState); ClusterState state = client().admin().cluster().prepareState().get().getState(); - PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE); assertEquals(0, tasks.taskMap().size()); }); // test that license restricted apis do not work try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) { client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress()); - PlainListenableActionFuture listener = new PlainListenableActionFuture<>(client.threadPool()); + PlainListenableActionFuture listener = new PlainListenableActionFuture<>(client.threadPool()); new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener); listener.actionGet(); fail("start datafeed action should not be enabled!"); @@ -328,14 +327,14 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase { try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) { client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress()); // re-open job now that the license is valid again - PlainListenableActionFuture openJobListener = new PlainListenableActionFuture<>(client.threadPool()); + PlainListenableActionFuture openJobListener = new PlainListenableActionFuture<>(client.threadPool()); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener); - PersistentActionResponse openJobResponse = openJobListener.actionGet(); + OpenJobAction.Response openJobResponse = openJobListener.actionGet(); assertNotNull(openJobResponse); - PlainListenableActionFuture listener = new PlainListenableActionFuture<>(client.threadPool()); + PlainListenableActionFuture listener = new PlainListenableActionFuture<>(client.threadPool()); new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener); - PersistentActionResponse response = listener.actionGet(); + StartDatafeedAction.Response response = listener.actionGet(); assertNotNull(response); } } @@ -356,14 +355,14 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase { new PutDatafeedAction.Request(createDatafeed("foobar", "foo", Collections.singletonList("foo"))), putDatafeedListener); PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet(); assertNotNull(putDatafeedResponse); - PlainListenableActionFuture openJobListener = new PlainListenableActionFuture<>(client.threadPool()); + PlainListenableActionFuture openJobListener = new PlainListenableActionFuture<>(client.threadPool()); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener); - PersistentActionResponse openJobResponse = openJobListener.actionGet(); + OpenJobAction.Response openJobResponse = openJobListener.actionGet(); assertNotNull(openJobResponse); - PlainListenableActionFuture startDatafeedListener = new PlainListenableActionFuture<>( + PlainListenableActionFuture startDatafeedListener = new PlainListenableActionFuture<>( client.threadPool()); new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), startDatafeedListener); - PersistentActionResponse startDatafeedResponse = startDatafeedListener.actionGet(); + StartDatafeedAction.Response startDatafeedResponse = startDatafeedListener.actionGet(); assertNotNull(startDatafeedResponse); } @@ -400,9 +399,9 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase { new MachineLearningClient(client).putJob(new PutJobAction.Request(createJob("foo").build()), putJobListener); PutJobAction.Response putJobResponse = putJobListener.actionGet(); assertNotNull(putJobResponse); - PlainListenableActionFuture openJobListener = new PlainListenableActionFuture<>(client.threadPool()); + PlainListenableActionFuture openJobListener = new PlainListenableActionFuture<>(client.threadPool()); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener); - PersistentActionResponse openJobResponse = openJobListener.actionGet(); + OpenJobAction.Response openJobResponse = openJobListener.actionGet(); assertNotNull(openJobResponse); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java index 3fa8ac92251..8d5f2720cd6 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java @@ -26,8 +26,8 @@ import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobTests; import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; -import org.elasticsearch.xpack.persistent.PersistentTasks; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.Collections; @@ -36,7 +36,7 @@ import static org.elasticsearch.xpack.ml.action.OpenJobActionTests.createJobTask import static org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests.createDatafeedConfig; import static org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests.createDatafeedJob; import static org.elasticsearch.xpack.ml.job.config.JobTests.buildJobBuilder; -import static org.elasticsearch.xpack.persistent.PersistentTasks.INITIAL_ASSIGNMENT; +import static org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; @@ -133,7 +133,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase { assertThat(result.getJobs().get("1"), sameInstance(job1)); assertThat(result.getDatafeeds().get("1"), nullValue()); - builder.deleteJob("1", new PersistentTasks(0L, Collections.emptyMap())); + builder.deleteJob("1", new PersistentTasksCustomMetaData(0L, Collections.emptyMap())); result = builder.build(); assertThat(result.getJobs().get("1"), nullValue()); assertThat(result.getDatafeeds().get("1"), nullValue()); @@ -151,7 +151,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase { PersistentTask task = createJobTask(0L, "1", null, JobState.CLOSED); MlMetadata.Builder builder2 = new MlMetadata.Builder(result); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, - () -> builder2.deleteJob("1", new PersistentTasks(0L, Collections.singletonMap(0L, task)))); + () -> builder2.deleteJob("1", new PersistentTasksCustomMetaData(0L, Collections.singletonMap(0L, task)))); assertThat(e.status(), equalTo(RestStatus.CONFLICT)); } @@ -163,7 +163,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase { builder.putDatafeed(datafeedConfig1); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, - () -> builder.deleteJob(job1.getId(), new PersistentTasks(0L, Collections.emptyMap()))); + () -> builder.deleteJob(job1.getId(), new PersistentTasksCustomMetaData(0L, Collections.emptyMap()))); assertThat(e.status(), equalTo(RestStatus.CONFLICT)); String expectedMsg = "Cannot delete job [" + job1.getId() + "] while datafeed [" + datafeedConfig1.getId() + "] refers to it"; assertThat(e.getMessage(), equalTo(expectedMsg)); @@ -172,7 +172,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase { public void testRemoveJob_failBecauseJobDoesNotExist() { MlMetadata.Builder builder1 = new MlMetadata.Builder(); expectThrows(ResourceNotFoundException.class, - () -> builder1.deleteJob("1", new PersistentTasks(0L, Collections.emptyMap()))); + () -> builder1.deleteJob("1", new PersistentTasksCustomMetaData(0L, Collections.emptyMap()))); } public void testCrudDatafeed() { @@ -187,7 +187,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase { assertThat(result.getDatafeeds().get("datafeed1"), sameInstance(datafeedConfig1)); builder = new MlMetadata.Builder(result); - builder.removeDatafeed("datafeed1", new PersistentTasks(0, Collections.emptyMap())); + builder.removeDatafeed("datafeed1", new PersistentTasksCustomMetaData(0, Collections.emptyMap())); result = builder.build(); assertThat(result.getJobs().get("job_id"), sameInstance(job1)); assertThat(result.getDatafeeds().get("datafeed1"), nullValue()); @@ -271,8 +271,8 @@ public class MlMetadataTests extends AbstractSerializingTestCase { StartDatafeedAction.Request request = new StartDatafeedAction.Request(datafeedConfig1.getId(), 0L); PersistentTask taskInProgress = new PersistentTask<>(0, StartDatafeedAction.NAME, request, false, true, INITIAL_ASSIGNMENT); - PersistentTasks tasksInProgress = - new PersistentTasks(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress)); + PersistentTasksCustomMetaData tasksInProgress = + new PersistentTasksCustomMetaData(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress)); DatafeedUpdate.Builder update = new DatafeedUpdate.Builder(datafeedConfig1.getId()); update.setScrollSize(5000); @@ -333,8 +333,8 @@ public class MlMetadataTests extends AbstractSerializingTestCase { StartDatafeedAction.Request request = new StartDatafeedAction.Request("datafeed1", 0L); PersistentTask taskInProgress = new PersistentTask<>(0, StartDatafeedAction.NAME, request, false, true, INITIAL_ASSIGNMENT); - PersistentTasks tasksInProgress = - new PersistentTasks(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress)); + PersistentTasksCustomMetaData tasksInProgress = + new PersistentTasksCustomMetaData(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress)); MlMetadata.Builder builder2 = new MlMetadata.Builder(result); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/CloseJobActionTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/CloseJobActionTests.java index 0a1e70563da..44014744c62 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/CloseJobActionTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/CloseJobActionTests.java @@ -13,11 +13,11 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ml.MlMetadata; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; -import org.elasticsearch.xpack.persistent.PersistentTasks; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import java.util.Collections; import java.util.HashMap; @@ -35,10 +35,11 @@ public class CloseJobActionTests extends ESTestCase { createJobTask(1L, "job_id", null, randomFrom(JobState.OPENED, JobState.FAILED)); ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) - .putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, Collections.singletonMap(1L, task)))); + .putCustom(PersistentTasksCustomMetaData.TYPE, + new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task)))); ClusterState result = CloseJobAction.moveJobToClosingState("job_id", csBuilder.build()); - PersistentTasks actualTasks = result.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData actualTasks = result.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); assertEquals(JobState.CLOSING, actualTasks.getTask(1L).getStatus()); MlMetadata actualMetadata = result.metaData().custom(MlMetadata.TYPE); @@ -49,7 +50,7 @@ public class CloseJobActionTests extends ESTestCase { MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) - .putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, Collections.emptyMap()))); + .putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(1L, Collections.emptyMap()))); expectThrows(ResourceNotFoundException.class, () -> CloseJobAction.moveJobToClosingState("job_id", csBuilder.build())); } @@ -59,14 +60,15 @@ public class CloseJobActionTests extends ESTestCase { PersistentTask task = createJobTask(1L, "job_id", null, JobState.OPENING); ClusterState.Builder csBuilder1 = ClusterState.builder(new ClusterName("_name")) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) - .putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, Collections.singletonMap(1L, task)))); + .putCustom(PersistentTasksCustomMetaData.TYPE, + new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task)))); ElasticsearchStatusException result = expectThrows(ElasticsearchStatusException.class, () -> CloseJobAction.moveJobToClosingState("job_id", csBuilder1.build())); assertEquals("cannot close job [job_id], expected job state [opened], but got [opening]", result.getMessage()); ClusterState.Builder csBuilder2 = ClusterState.builder(new ClusterName("_name")) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) - .putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, Collections.emptyMap()))); + .putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(1L, Collections.emptyMap()))); result = expectThrows(ElasticsearchStatusException.class, () -> CloseJobAction.moveJobToClosingState("job_id", csBuilder2.build())); assertEquals("cannot close job [job_id], expected job state [opened], but got [closed]", result.getMessage()); } @@ -81,7 +83,7 @@ public class CloseJobActionTests extends ESTestCase { tasks.put(2L, createDatafeedTask(2L, "datafeed_id", 0L, null, DatafeedState.STARTED)); ClusterState cs1 = ClusterState.builder(new ClusterName("_name")) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) - .putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, tasks))).build(); + .putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(1L, tasks))).build(); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> CloseJobAction.validateAndFindTask("job_id", cs1)); @@ -95,7 +97,7 @@ public class CloseJobActionTests extends ESTestCase { } ClusterState cs2 = ClusterState.builder(new ClusterName("_name")) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) - .putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, tasks))).build(); + .putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(1L, tasks))).build(); assertEquals(jobTask, CloseJobAction.validateAndFindTask("job_id", cs2)); } @@ -103,7 +105,7 @@ public class CloseJobActionTests extends ESTestCase { String nodeId, DatafeedState datafeedState) { PersistentTask task = new PersistentTask<>(id, StartDatafeedAction.NAME, new StartDatafeedAction.Request(datafeedId, startTime), false, true, - new PersistentTasks.Assignment(nodeId, "test assignment")); + new PersistentTasksCustomMetaData.Assignment(nodeId, "test assignment")); task = new PersistentTask<>(task, datafeedState); return task; } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java index c954d4156a5..7729b68fab5 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java @@ -24,10 +24,9 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; -import org.elasticsearch.xpack.persistent.PersistentActionCoordinator; -import org.elasticsearch.xpack.persistent.PersistentActionRequest; -import org.elasticsearch.xpack.persistent.PersistentActionResponse; -import org.elasticsearch.xpack.persistent.PersistentTasks; +import org.elasticsearch.xpack.persistent.PersistentTasksNodeService; +import org.elasticsearch.xpack.persistent.PersistentTaskRequest; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.xpack.security.Security; import org.junit.After; @@ -70,12 +69,13 @@ public class DatafeedJobsIT extends SecurityIntegTestCase { List entries = new ArrayList<>(ClusterModule.getNamedWriteables()); entries.addAll(new SearchModule(Settings.EMPTY, true, Collections.emptyList()).getNamedWriteables()); entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new)); - entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new)); - entries.add(new NamedWriteableRegistry.Entry(PersistentActionRequest.class, StartDatafeedAction.NAME, + entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, + PersistentTasksCustomMetaData::new)); + entries.add(new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, StartDatafeedAction.NAME, StartDatafeedAction.Request::new)); - entries.add(new NamedWriteableRegistry.Entry(PersistentActionRequest.class, OpenJobAction.NAME, OpenJobAction.Request::new)); - entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, PersistentActionCoordinator.Status.NAME, - PersistentActionCoordinator.Status::new)); + entries.add(new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, OpenJobAction.NAME, OpenJobAction.Request::new)); + entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, + PersistentTasksNodeService.Status::new)); entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, JobState.NAME, JobState::fromStream)); entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream)); final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries); @@ -187,8 +187,7 @@ public class DatafeedJobsIT extends SecurityIntegTestCase { assertTrue(putDatafeedResponse.isAcknowledged()); StartDatafeedAction.Request startDatafeedRequest = new StartDatafeedAction.Request(datafeedConfig.getId(), 0L); - PersistentActionResponse startDatafeedResponse = - client().execute(StartDatafeedAction.INSTANCE, startDatafeedRequest).get(); + StartDatafeedAction.Response startDatafeedResponse = client().execute(StartDatafeedAction.INSTANCE, startDatafeedRequest).get(); assertBusy(() -> { DataCounts dataCounts = getDataCounts(job.getId()); assertThat(dataCounts.getProcessedRecordCount(), equalTo(numDocs1)); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionTests.java index d34e8e80a06..25700d03dbe 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionTests.java @@ -29,12 +29,11 @@ import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.ml.job.persistence.JobProvider; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; -import org.elasticsearch.xpack.persistent.PersistentTasks; -import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.net.InetAddress; import java.util.ArrayList; @@ -58,14 +57,14 @@ public class OpenJobActionTests extends ESTestCase { PersistentTask task = createJobTask(1L, "job_id", "_node_id", randomFrom(JobState.CLOSED, JobState.FAILED)); - PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task)); + PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task)); OpenJobAction.validate("job_id", mlBuilder.build(), tasks, nodes); - OpenJobAction.validate("job_id", mlBuilder.build(), new PersistentTasks(1L, Collections.emptyMap()), nodes); + OpenJobAction.validate("job_id", mlBuilder.build(), new PersistentTasksCustomMetaData(1L, Collections.emptyMap()), nodes); OpenJobAction.validate("job_id", mlBuilder.build(), null, nodes); task = createJobTask(1L, "job_id", "_other_node_id", JobState.OPENED); - tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task)); + tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task)); OpenJobAction.validate("job_id", mlBuilder.build(), tasks, nodes); } @@ -95,7 +94,7 @@ public class OpenJobActionTests extends ESTestCase { JobState jobState = randomFrom(JobState.OPENING, JobState.OPENED, JobState.CLOSING); PersistentTask task = createJobTask(1L, "job_id", "_node_id", jobState); - PersistentTasks tasks1 = new PersistentTasks(1L, Collections.singletonMap(1L, task)); + PersistentTasksCustomMetaData tasks1 = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task)); Exception e = expectThrows(ElasticsearchStatusException.class, () -> OpenJobAction.validate("job_id", mlBuilder.build(), tasks1, nodes)); @@ -103,7 +102,7 @@ public class OpenJobActionTests extends ESTestCase { jobState = randomFrom(JobState.OPENING, JobState.CLOSING); task = createJobTask(1L, "job_id", "_other_node_id", jobState); - PersistentTasks tasks2 = new PersistentTasks(1L, Collections.singletonMap(1L, task)); + PersistentTasksCustomMetaData tasks2 = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task)); e = expectThrows(ElasticsearchStatusException.class, () -> OpenJobAction.validate("job_id", mlBuilder.build(), tasks2, nodes)); @@ -129,14 +128,14 @@ public class OpenJobActionTests extends ESTestCase { new Assignment("_node_id1", "test assignment"))); taskMap.put(2L, new PersistentTask<>(2L, OpenJobAction.NAME, new OpenJobAction.Request("job_id3"), false, true, new Assignment("_node_id2", "test assignment"))); - PersistentTasks tasks = new PersistentTasks(3L, taskMap); + PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(3L, taskMap); ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); MetaData.Builder metaData = MetaData.builder(); RoutingTable.Builder routingTable = RoutingTable.builder(); addJobAndIndices(metaData, routingTable, "job_id1", "job_id2", "job_id3", "job_id4"); cs.nodes(nodes); - metaData.putCustom(PersistentTasks.TYPE, tasks); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); cs.routingTable(routingTable.build()); Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id4", cs.build(), 2, logger); @@ -160,14 +159,14 @@ public class OpenJobActionTests extends ESTestCase { taskMap.put(id, createJobTask(id, "job_id" + id, nodeId, JobState.OPENED)); } } - PersistentTasks tasks = new PersistentTasks(numNodes * maxRunningJobsPerNode, taskMap); + PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(numNodes * maxRunningJobsPerNode, taskMap); ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); MetaData.Builder metaData = MetaData.builder(); RoutingTable.Builder routingTable = RoutingTable.builder(); addJobAndIndices(metaData, routingTable, "job_id1", "job_id2"); cs.nodes(nodes); - metaData.putCustom(PersistentTasks.TYPE, tasks); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); cs.routingTable(routingTable.build()); Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id2", cs.build(), 2, logger); @@ -187,14 +186,14 @@ public class OpenJobActionTests extends ESTestCase { PersistentTask task = new PersistentTask<>(1L, OpenJobAction.NAME, new OpenJobAction.Request("job_id1"), false, true, new Assignment("_node_id1", "test assignment")); - PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task)); + PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task)); ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); MetaData.Builder metaData = MetaData.builder(); RoutingTable.Builder routingTable = RoutingTable.builder(); addJobAndIndices(metaData, routingTable, "job_id1", "job_id2"); cs.nodes(nodes); - metaData.putCustom(PersistentTasks.TYPE, tasks); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); cs.routingTable(routingTable.build()); Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id2", cs.build(), 2, logger); @@ -220,7 +219,7 @@ public class OpenJobActionTests extends ESTestCase { taskMap.put(2L, createJobTask(2L, "job_id3", "_node_id2", JobState.OPENING)); taskMap.put(3L, createJobTask(3L, "job_id4", "_node_id2", JobState.OPENING)); taskMap.put(4L, createJobTask(4L, "job_id5", "_node_id3", JobState.OPENING)); - PersistentTasks tasks = new PersistentTasks(5L, taskMap); + PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(5L, taskMap); ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); csBuilder.nodes(nodes); @@ -228,7 +227,7 @@ public class OpenJobActionTests extends ESTestCase { RoutingTable.Builder routingTable = RoutingTable.builder(); addJobAndIndices(metaData, routingTable, "job_id1", "job_id2", "job_id3", "job_id4", "job_id5", "job_id6", "job_id7"); csBuilder.routingTable(routingTable.build()); - metaData.putCustom(PersistentTasks.TYPE, tasks); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); csBuilder.metaData(metaData); ClusterState cs = csBuilder.build(); @@ -237,30 +236,30 @@ public class OpenJobActionTests extends ESTestCase { PersistentTask lastTask = createJobTask(5L, "job_id6", "_node_id3", JobState.OPENING); taskMap.put(5L, lastTask); - tasks = new PersistentTasks(6L, taskMap); + tasks = new PersistentTasksCustomMetaData(6L, taskMap); csBuilder = ClusterState.builder(cs); - csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasks.TYPE, tasks)); + csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); result = OpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, logger); assertNull("no node selected, because OPENING state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); taskMap.put(5L, new PersistentTask<>(lastTask, false, new Assignment("_node_id3", "test assignment"))); - tasks = new PersistentTasks(6L, taskMap); + tasks = new PersistentTasksCustomMetaData(6L, taskMap); csBuilder = ClusterState.builder(cs); - csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasks.TYPE, tasks)); + csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); result = OpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, logger); assertNull("no node selected, because stale task", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); taskMap.put(5L, new PersistentTask<>(lastTask, null)); - tasks = new PersistentTasks(6L, taskMap); + tasks = new PersistentTasksCustomMetaData(6L, taskMap); csBuilder = ClusterState.builder(cs); - csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasks.TYPE, tasks)); + csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); result = OpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, logger); assertNull("no node selected, because null state", result.getExecutorNode()); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/StartDatafeedActionTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/StartDatafeedActionTests.java index 91c3b281cf1..951c141b7b3 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/StartDatafeedActionTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/StartDatafeedActionTests.java @@ -23,9 +23,9 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests; import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.JobState; -import org.elasticsearch.xpack.persistent.PersistentTasks; -import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.net.InetAddress; import java.util.Collections; @@ -35,7 +35,7 @@ import java.util.Map; import static org.elasticsearch.xpack.ml.action.OpenJobActionTests.createJobTask; import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createDatafeed; import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createScheduledJob; -import static org.elasticsearch.xpack.persistent.PersistentTasks.INITIAL_ASSIGNMENT; +import static org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT; import static org.hamcrest.Matchers.equalTo; public class StartDatafeedActionTests extends ESTestCase { @@ -48,7 +48,7 @@ public class StartDatafeedActionTests extends ESTestCase { JobState jobState = randomFrom(JobState.FAILED, JobState.CLOSED, JobState.CLOSING, JobState.OPENING); PersistentTask task = createJobTask(0L, job.getId(), "node_id", jobState); - PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task)); + PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task)); DiscoveryNodes nodes = DiscoveryNodes.builder() .add(new DiscoveryNode("node_name", "node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), @@ -57,7 +57,7 @@ public class StartDatafeedActionTests extends ESTestCase { ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name")) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build()) - .putCustom(PersistentTasks.TYPE, tasks)) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasks)) .nodes(nodes); Assignment result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build()); @@ -66,10 +66,10 @@ public class StartDatafeedActionTests extends ESTestCase { "] while state [opened] is required", result.getExplanation()); task = createJobTask(0L, job.getId(), "node_id", JobState.OPENED); - tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task)); + tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task)); cs = ClusterState.builder(new ClusterName("cluster_name")) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build()) - .putCustom(PersistentTasks.TYPE, tasks)) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasks)) .nodes(nodes); result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build()); assertEquals("node_id", result.getExecutorNode()); @@ -83,7 +83,7 @@ public class StartDatafeedActionTests extends ESTestCase { String nodeId = randomBoolean() ? "node_id2" : null; PersistentTask task = createJobTask(0L, job.getId(), nodeId, JobState.OPENED); - PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task)); + PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task)); DiscoveryNodes nodes = DiscoveryNodes.builder() .add(new DiscoveryNode("node_name", "node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), @@ -92,7 +92,7 @@ public class StartDatafeedActionTests extends ESTestCase { ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name")) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build()) - .putCustom(PersistentTasks.TYPE, tasks)) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasks)) .nodes(nodes); Assignment result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build()); @@ -101,10 +101,10 @@ public class StartDatafeedActionTests extends ESTestCase { result.getExplanation()); task = createJobTask(0L, job.getId(), "node_id1", JobState.OPENED); - tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task)); + tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task)); cs = ClusterState.builder(new ClusterName("cluster_name")) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build()) - .putCustom(PersistentTasks.TYPE, tasks)) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasks)) .nodes(nodes); result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build()); assertEquals("node_id1", result.getExecutorNode()); @@ -128,7 +128,7 @@ public class StartDatafeedActionTests extends ESTestCase { PersistentTask task = new PersistentTask<>(0L, OpenJobAction.NAME, new OpenJobAction.Request("job_id"), false, true, INITIAL_ASSIGNMENT); - PersistentTasks tasks = new PersistentTasks(0L, Collections.singletonMap(0L, task)); + PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(0L, Collections.singletonMap(0L, task)); DatafeedConfig datafeedConfig1 = DatafeedJobRunnerTests.createDatafeedConfig("foo-datafeed", "job_id").build(); MlMetadata mlMetadata2 = new MlMetadata.Builder(mlMetadata1) .putDatafeed(datafeedConfig1) @@ -158,7 +158,7 @@ public class StartDatafeedActionTests extends ESTestCase { Map> taskMap = new HashMap<>(); taskMap.put(0L, jobTask); taskMap.put(1L, datafeedTask); - PersistentTasks tasks = new PersistentTasks(2L, taskMap); + PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(2L, taskMap); Exception e = expectThrows(ElasticsearchStatusException.class, () -> StartDatafeedAction.validate("datafeed_id", mlMetadata1, tasks, nodes)); @@ -186,7 +186,7 @@ public class StartDatafeedActionTests extends ESTestCase { Map> taskMap = new HashMap<>(); taskMap.put(0L, jobTask); taskMap.put(1L, datafeedTask); - PersistentTasks tasks = new PersistentTasks(2L, taskMap); + PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(2L, taskMap); StartDatafeedAction.validate("datafeed_id", mlMetadata1, tasks, nodes); datafeedTask = new PersistentTask<>(0L, StartDatafeedAction.NAME, new StartDatafeedAction.Request("datafeed_id", 0L), diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/StopDatafeedActionRequestTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/StopDatafeedActionRequestTests.java index 785870f7430..7b114e8cf38 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/StopDatafeedActionRequestTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/StopDatafeedActionRequestTests.java @@ -15,9 +15,9 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; -import org.elasticsearch.xpack.persistent.PersistentActionRequest; -import org.elasticsearch.xpack.persistent.PersistentTasks; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.xpack.persistent.PersistentTaskRequest; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.util.Collections; @@ -47,10 +47,10 @@ public class StopDatafeedActionRequestTests extends AbstractStreamableXContentTe } public void testValidate() { - PersistentTask task = new PersistentTask(1L, StartDatafeedAction.NAME, - new StartDatafeedAction.Request("foo", 0L), false, false, new PersistentTasks.Assignment("node_id", "")); + PersistentTask task = new PersistentTask(1L, StartDatafeedAction.NAME, + new StartDatafeedAction.Request("foo", 0L), false, false, new PersistentTasksCustomMetaData.Assignment("node_id", "")); task = new PersistentTask<>(task, DatafeedState.STARTED); - PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task)); + PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task)); Job job = createDatafeedJob().build(); MlMetadata mlMetadata1 = new MlMetadata.Builder().putJob(job, false).build(); @@ -66,14 +66,14 @@ public class StopDatafeedActionRequestTests extends AbstractStreamableXContentTe } public void testValidate_alreadyStopped() { - PersistentTasks tasks; + PersistentTasksCustomMetaData tasks; if (randomBoolean()) { - PersistentTask task = new PersistentTask(1L, StartDatafeedAction.NAME, - new StartDatafeedAction.Request("foo", 0L), false, false, new PersistentTasks.Assignment("node_id", "")); + PersistentTask task = new PersistentTask(1L, StartDatafeedAction.NAME, + new StartDatafeedAction.Request("foo", 0L), false, false, new PersistentTasksCustomMetaData.Assignment("node_id", "")); task = new PersistentTask<>(task, DatafeedState.STOPPED); - tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task)); + tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task)); } else { - tasks = randomBoolean() ? null : new PersistentTasks(0L, Collections.emptyMap()); + tasks = randomBoolean() ? null : new PersistentTasksCustomMetaData(0L, Collections.emptyMap()); } Job job = createDatafeedJob().build(); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunnerTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunnerTests.java index f3d5d8b6378..d4b56abf3a2 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunnerTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunnerTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.ml.datafeed; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -41,10 +40,10 @@ import org.elasticsearch.xpack.ml.job.persistence.MockClientBuilder; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.notifications.AuditMessage; import org.elasticsearch.xpack.ml.notifications.Auditor; -import org.elasticsearch.xpack.persistent.PersistentTasks; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; -import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction; -import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction.Response; +import org.elasticsearch.xpack.persistent.PersistentTasksService; +import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -84,6 +83,7 @@ public class DatafeedJobRunnerTests extends ESTestCase { private DatafeedJobRunner datafeedJobRunner; private long currentTime = 120000; private Auditor auditor; + private PersistentTasksService persistentTasksService; @Before @SuppressWarnings("unchecked") @@ -93,14 +93,14 @@ public class DatafeedJobRunnerTests extends ESTestCase { mlMetadata.putJob(job, false); mlMetadata.putDatafeed(createDatafeedConfig("datafeed_id", job.getId()).build()); PersistentTask task = createJobTask(0L, job.getId(), "node_id", JobState.OPENED); - PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task)); + PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task)); DiscoveryNodes nodes = DiscoveryNodes.builder() .add(new DiscoveryNode("node_name", "node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Collections.emptyMap(), Collections.emptySet(), Version.CURRENT)) .build(); ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name")) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build()) - .putCustom(PersistentTasks.TYPE, tasks)) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasks)) .nodes(nodes); clusterService = mock(ClusterService.class); @@ -140,7 +140,9 @@ public class DatafeedJobRunnerTests extends ESTestCase { when(client.execute(same(PostDataAction.INSTANCE), any())).thenReturn(jobDataFuture); when(client.execute(same(FlushJobAction.INSTANCE), any())).thenReturn(flushJobFuture); - datafeedJobRunner = new DatafeedJobRunner(threadPool, client, clusterService, jobProvider, () -> currentTime, auditor) { + persistentTasksService = mock(PersistentTasksService.class); + datafeedJobRunner = new DatafeedJobRunner(threadPool, client, clusterService, jobProvider, () -> currentTime, + persistentTasksService, auditor) { @Override DataExtractorFactory createDataExtractorFactory(DatafeedConfig datafeedConfig, Job job) { return dataExtractorFactory; @@ -153,13 +155,12 @@ public class DatafeedJobRunnerTests extends ESTestCase { consumer.accept(new ResourceNotFoundException("dummy")); return null; }).when(jobProvider).bucketsViaInternalClient(any(), any(), any(), any()); - doAnswer(invocationOnMock -> { @SuppressWarnings("rawtypes") - ActionListener listener = (ActionListener) invocationOnMock.getArguments()[2]; - listener.onResponse(new Response(true)); + PersistentTaskOperationListener listener = (PersistentTaskOperationListener) invocationOnMock.getArguments()[2]; + listener.onResponse(0L); return null; - }).when(client).execute(same(UpdatePersistentTaskStatusAction.INSTANCE), any(), any()); + }).when(persistentTasksService).updateStatus(anyLong(), any(), any()); } public void testLookbackOnly_WarnsWhenNoDataIsRetrieved() throws Exception { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index c06b2c7a691..d94f1134a7a 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -26,8 +26,8 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; -import org.elasticsearch.xpack.persistent.PersistentTasks; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.Collection; @@ -165,7 +165,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase { client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet(); assertBusy(() -> { ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); - PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); PersistentTask task = tasks.taskMap().values().iterator().next(); DiscoveryNode node = clusterState.nodes().resolveNode(task.getExecutorNode()); @@ -216,7 +216,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase { // Sample each cs update and keep track each time a node holds more than `maxConcurrentJobAllocations` opening jobs. List violations = new CopyOnWriteArrayList<>(); internalCluster().clusterService(nonMlNode).addListener(event -> { - PersistentTasks tasks = event.state().metaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = event.state().metaData().custom(PersistentTasksCustomMetaData.TYPE); if (tasks == null) { return; } @@ -247,7 +247,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase { assertBusy(() -> { ClusterState state = client().admin().cluster().prepareState().get().getState(); - PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE); assertEquals(numJobs, tasks.taskMap().size()); for (PersistentTask task : tasks.taskMap().values()) { assertNotNull(task.getExecutorNode()); @@ -271,7 +271,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase { ensureStableCluster(1, nonMlNode); assertBusy(() -> { ClusterState state = client(nonMlNode).admin().cluster().prepareState().get().getState(); - PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE); assertEquals(numJobs, tasks.taskMap().size()); for (PersistentTask task : tasks.taskMap().values()) { assertNull(task.getExecutorNode()); @@ -287,7 +287,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase { ensureStableCluster(1 + numMlNodes); assertBusy(() -> { ClusterState state = client().admin().cluster().prepareState().get().getState(); - PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE); assertEquals(numJobs, tasks.taskMap().size()); for (PersistentTask task : tasks.taskMap().values()) { assertNotNull(task.getExecutorNode()); @@ -332,7 +332,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase { client().execute(CloseJobAction.INSTANCE, closeJobRequest); assertBusy(() -> { ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); - PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); assertEquals(0, tasks.taskMap().size()); }); logger.info("Stop data node"); @@ -357,7 +357,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase { private void assertJobTask(String jobId, JobState expectedState, boolean hasExecutorNode) { ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); - PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); assertEquals(1, tasks.taskMap().size()); PersistentTask task = tasks.findTasks(OpenJobAction.NAME, p -> { return p.getRequest() instanceof OpenJobAction.Request && diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index 118801be3ea..323df5207d6 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -24,8 +24,8 @@ import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; -import org.elasticsearch.xpack.persistent.PersistentTasks; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.util.Collections; import java.util.concurrent.TimeUnit; @@ -125,7 +125,7 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase { disrupt.run(); assertBusy(() -> { ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); - PersistentTasks tasks = clusterState.metaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); assertNotNull(tasks); assertEquals(2, tasks.taskMap().size()); for (PersistentTask task : tasks.tasks()) { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java index 224657b22b6..17b7b495847 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java @@ -17,7 +17,7 @@ import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; -import org.elasticsearch.xpack.persistent.PersistentTasks; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; public class TooManyJobsIT extends BaseMlIntegTestCase { @@ -49,10 +49,10 @@ public class TooManyJobsIT extends BaseMlIntegTestCase { client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request("2")).actionGet(); assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.CLOSED); ClusterState state = client().admin().cluster().prepareState().get().getState(); - PersistentTasks tasks = state.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); assertEquals(1, tasks.taskMap().size()); // now just double check that the first job is still opened: - PersistentTasks.PersistentTask task = tasks.taskMap().values().iterator().next(); + PersistentTasksCustomMetaData.PersistentTask task = tasks.taskMap().values().iterator().next(); assertEquals(JobState.OPENED, task.getStatus()); OpenJobAction.Request openJobRequest = (OpenJobAction.Request) task.getRequest(); assertEquals("1", openJobRequest.getJobId()); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java index 69d2358fdbb..0a7f09de5e4 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java @@ -34,7 +34,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory; -import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction; +import org.elasticsearch.xpack.persistent.PersistentTasksService; import org.junit.Before; import org.mockito.Mockito; @@ -122,16 +122,15 @@ public class AutodetectProcessManagerTests extends ESTestCase { public void testOpenJob() { Client client = mock(Client.class); + PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); when(jobManager.getJobOrThrowIfUnknown("foo")).thenReturn(createJobDetails("foo")); - AutodetectProcessManager manager = createManager(communicator, client); + AutodetectProcessManager manager = createManager(communicator, client, persistentTasksService); manager.openJob("foo", 1L, false, e -> {}); assertEquals(1, manager.numberOfOpenJobs()); assertTrue(manager.jobHasActiveAutodetectProcess("foo")); - UpdatePersistentTaskStatusAction.Request expectedRequest = - new UpdatePersistentTaskStatusAction.Request(1L, JobState.OPENED); - verify(client).execute(eq(UpdatePersistentTaskStatusAction.INSTANCE), eq(expectedRequest), any()); + verify(persistentTasksService).updateStatus(eq(1L), eq(JobState.OPENED), any()); } public void testOpenJob_exceedMaxNumJobs() { @@ -149,6 +148,7 @@ public class AutodetectProcessManagerTests extends ESTestCase { when(jobManager.getJobOrThrowIfUnknown("foobar")).thenReturn(createJobDetails("foobar")); Client client = mock(Client.class); + PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); ThreadPool threadPool = mock(ThreadPool.class); ThreadPool.Cancellable cancellable = mock(ThreadPool.Cancellable.class); when(threadPool.scheduleWithFixedDelay(any(), any(), any())).thenReturn(cancellable); @@ -161,7 +161,7 @@ public class AutodetectProcessManagerTests extends ESTestCase { Settings.Builder settings = Settings.builder(); settings.put(AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE.getKey(), 3); AutodetectProcessManager manager = spy(new AutodetectProcessManager(settings.build(), client, threadPool, jobManager, jobProvider, - jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory)); + jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory, persistentTasksService)); DataCounts dataCounts = new DataCounts("foo"); ModelSnapshot modelSnapshot = new ModelSnapshot.Builder("foo").build(); @@ -319,11 +319,11 @@ public class AutodetectProcessManagerTests extends ESTestCase { handler.accept(new DataCounts(jobId)); return null; }).when(jobProvider).dataCounts(eq("my_id"), any(), any()); - + PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); AutodetectProcess autodetectProcess = mock(AutodetectProcess.class); AutodetectProcessFactory autodetectProcessFactory = (j, modelSnapshot, quantiles, filters, i, e) -> autodetectProcess; AutodetectProcessManager manager = new AutodetectProcessManager(Settings.EMPTY, client, threadPool, jobManager, jobProvider, - jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory); + jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory, persistentTasksService); expectThrows(EsRejectedExecutionException.class, () -> manager.create("my_id", 1L, dataCounts, modelSnapshot, quantiles, filters, false, e -> {})); @@ -332,14 +332,16 @@ public class AutodetectProcessManagerTests extends ESTestCase { private AutodetectProcessManager createManager(AutodetectCommunicator communicator) { Client client = mock(Client.class); - return createManager(communicator, client); + PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); + return createManager(communicator, client, persistentTasksService); } - private AutodetectProcessManager createManager(AutodetectCommunicator communicator, Client client) { + private AutodetectProcessManager createManager(AutodetectCommunicator communicator, Client client, + PersistentTasksService persistentTasksService) { ThreadPool threadPool = mock(ThreadPool.class); AutodetectProcessFactory autodetectProcessFactory = mock(AutodetectProcessFactory.class); AutodetectProcessManager manager = new AutodetectProcessManager(Settings.EMPTY, client, threadPool, jobManager, jobProvider, - jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory); + jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory, persistentTasksService); manager = spy(manager); doReturn(communicator).when(manager) .create(any(), anyLong(), eq(dataCounts), eq(modelSnapshot), eq(quantiles), eq(filters), anyBoolean(), any()); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionRegistryTests.java b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionRegistryTests.java deleted file mode 100644 index 4a4d3ac9403..00000000000 --- a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionRegistryTests.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.persistent; - -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.threadpool.ThreadPool; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class PersistentActionRegistryTests extends ESTestCase { - - public void testActionLookup() { - PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); - TransportPersistentAction action1 = mock(TransportPersistentAction.class); - when(action1.getExecutor()).thenReturn(ThreadPool.Names.MANAGEMENT); - TransportPersistentAction action2 = mock(TransportPersistentAction.class); - when(action2.getExecutor()).thenReturn(ThreadPool.Names.GENERIC); - registry.registerPersistentAction("test1", action1); - registry.registerPersistentAction("test2", action2); - - assertEquals(registry.getPersistentActionHolderSafe("test1").getAction(), "test1"); - assertEquals(registry.getPersistentActionHolderSafe("test1").getExecutor(), ThreadPool.Names.MANAGEMENT); - assertEquals(registry.getPersistentActionHolderSafe("test1").getPersistentAction(), action1); - assertEquals(registry.getPersistentActionSafe("test1"), action1); - - assertEquals(registry.getPersistentActionHolderSafe("test2").getAction(), "test2"); - assertEquals(registry.getPersistentActionHolderSafe("test2").getExecutor(), ThreadPool.Names.GENERIC); - assertEquals(registry.getPersistentActionHolderSafe("test2").getPersistentAction(), action2); - assertEquals(registry.getPersistentActionSafe("test2"), action2); - - try { - registry.getPersistentActionHolderSafe("test3"); - fail("Should have failed"); - } catch (IllegalStateException ex) { - assertEquals(ex.getMessage(), "Unknown persistent action [test3]"); - } - - try { - registry.getPersistentActionSafe("test3"); - fail("Should have failed"); - } catch (IllegalStateException ex) { - assertEquals(ex.getMessage(), "Unknown persistent action [test3]"); - } - } -} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterServiceTests.java b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksClusterServiceTests.java similarity index 84% rename from plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterServiceTests.java rename to plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksClusterServiceTests.java index e63b6baabb3..456bdbbfe5a 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterServiceTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksClusterServiceTests.java @@ -18,9 +18,9 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; -import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; -import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest; import java.util.ArrayList; import java.util.Arrays; @@ -29,13 +29,13 @@ import java.util.HashSet; import java.util.List; import static java.util.Collections.emptyMap; -import static org.elasticsearch.xpack.persistent.TransportPersistentAction.NO_NODE_FOUND; +import static org.elasticsearch.xpack.persistent.PersistentTasksExecutor.NO_NODE_FOUND; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class PersistentTaskClusterServiceTests extends ESTestCase { +public class PersistentTasksClusterServiceTests extends ESTestCase { public void testReassignmentRequired() { int numberOfIterations = randomIntBetween(1, 30); @@ -50,10 +50,10 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { clusterState = insignificantChange(clusterState); } ClusterChangedEvent event = new ClusterChangedEvent("test", clusterState, previousState); - assertThat(dumpEvent(event), PersistentTaskClusterService.reassignmentRequired(event, - new PersistentTaskClusterService.ExecutorNodeDecider() { + assertThat(dumpEvent(event), PersistentTasksClusterService.reassignmentRequired(event, + new PersistentTasksClusterService.ExecutorNodeDecider() { @Override - public Assignment getAssignment( + public Assignment getAssignment( String action, ClusterState currentState, Request request) { if ("never_assign".equals(((TestRequest) request).getTestParam())) { return NO_NODE_FOUND; @@ -66,14 +66,14 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { public void testReassignTasksWithNoTasks() { ClusterState clusterState = initialState(); - assertThat(reassign(clusterState).metaData().custom(PersistentTasks.TYPE), nullValue()); + assertThat(reassign(clusterState).metaData().custom(PersistentTasksCustomMetaData.TYPE), nullValue()); } public void testReassignConsidersClusterStateUpdates() { ClusterState clusterState = initialState(); ClusterState.Builder builder = ClusterState.builder(clusterState); - PersistentTasks.Builder tasks = PersistentTasks.builder( - clusterState.metaData().custom(PersistentTasks.TYPE)); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder( + clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE)); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); addTestNodes(nodes, randomIntBetween(1, 10)); int numberOfTasks = randomIntBetween(2, 40); @@ -81,11 +81,11 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { addTask(tasks, "should_assign", "assign_one", randomBoolean() ? null : "no_longer_exits", false); } - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, tasks.build()); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); clusterState = builder.metaData(metaData).nodes(nodes).build(); ClusterState newClusterState = reassign(clusterState); - PersistentTasks tasksInProgress = newClusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress, notNullValue()); } @@ -93,8 +93,8 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { public void testReassignTasks() { ClusterState clusterState = initialState(); ClusterState.Builder builder = ClusterState.builder(clusterState); - PersistentTasks.Builder tasks = PersistentTasks.builder( - clusterState.metaData().custom(PersistentTasks.TYPE)); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder( + clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE)); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); addTestNodes(nodes, randomIntBetween(1, 10)); int numberOfTasks = randomIntBetween(0, 40); @@ -118,11 +118,11 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { } } - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, tasks.build()); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); clusterState = builder.metaData(metaData).nodes(nodes).build(); ClusterState newClusterState = reassign(clusterState); - PersistentTasks tasksInProgress = newClusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress, notNullValue()); assertThat("number of tasks shouldn't change as a result or reassignment", @@ -133,15 +133,15 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { for (PersistentTask task : tasksInProgress.tasks()) { if (task.isStopped()) { assertThat("stopped tasks should be never assigned", task.getExecutorNode(), nullValue()); - assertThat(task.getAssignment().getExplanation(), equalTo("explanation: " + task.getAction())); + assertThat(task.getAssignment().getExplanation(), equalTo("explanation: " + task.getTaskName())); } else { // explanation should correspond to the action name - switch (task.getAction()) { + switch (task.getTaskName()) { case "should_assign": assertThat(task.getExecutorNode(), notNullValue()); assertThat(task.isAssigned(), equalTo(true)); if (clusterState.nodes().nodeExists(task.getExecutorNode()) == false) { - logger.info(clusterState.metaData().custom(PersistentTasks.TYPE).toString()); + logger.info(clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE).toString()); } assertThat("task should be assigned to a node that is in the cluster, was assigned to " + task.getExecutorNode(), clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true)); @@ -162,7 +162,7 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { } break; default: - fail("Unknown action " + task.getAction()); + fail("Unknown action " + task.getTaskName()); } } } @@ -176,10 +176,10 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { } private ClusterState reassign(ClusterState clusterState) { - return PersistentTaskClusterService.reassignTasks(clusterState, logger, - new PersistentTaskClusterService.ExecutorNodeDecider() { + return PersistentTasksClusterService.reassignTasks(clusterState, logger, + new PersistentTasksClusterService.ExecutorNodeDecider() { @Override - public Assignment getAssignment( + public Assignment getAssignment( String action, ClusterState currentState, Request request) { TestRequest testRequest = (TestRequest) request; switch (testRequest.getTestParam()) { @@ -203,7 +203,7 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { private Assignment assignOnlyOneTaskAtATime(ClusterState clusterState) { DiscoveryNodes nodes = clusterState.nodes(); - PersistentTasks tasksInProgress = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasksInProgress = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); if (tasksInProgress.findTasks("assign_one", task -> task.isStopped() == false && nodes.nodeExists(task.getExecutorNode())).isEmpty()) { return randomNodeAssignment(clusterState.nodes()); @@ -232,12 +232,12 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { return "nodes_changed: " + event.nodesChanged() + " nodes_removed:" + event.nodesRemoved() + " routing_table_changed:" + event.routingTableChanged() + - " tasks: " + event.state().metaData().custom(PersistentTasks.TYPE); + " tasks: " + event.state().metaData().custom(PersistentTasksCustomMetaData.TYPE); } private ClusterState significantChange(ClusterState clusterState) { ClusterState.Builder builder = ClusterState.builder(clusterState); - PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); if (tasks != null) { if (randomBoolean()) { for (PersistentTask task : tasks.tasks()) { @@ -255,11 +255,12 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { // we don't have any unassigned tasks - add some if (randomBoolean()) { logger.info("added random task"); - addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasks.builder(tasks), null, false); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks), null, + false); tasksOrNodesChanged = true; } else { logger.info("added unassignable task with custom assignment message"); - addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasks.builder(tasks), + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks), new Assignment(null, "change me"), "never_assign", false); tasksOrNodesChanged = true; } @@ -282,10 +283,10 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { return builder.build(); } - private PersistentTasks removeTasksWithChangingAssignment(PersistentTasks tasks) { + private PersistentTasksCustomMetaData removeTasksWithChangingAssignment(PersistentTasksCustomMetaData tasks) { if (tasks != null) { boolean changed = false; - PersistentTasks.Builder tasksBuilder = PersistentTasks.builder(tasks); + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); for (PersistentTask task : tasks.tasks()) { // Remove all unassigned tasks that cause changing assignments they might trigger a significant change if ("never_assign".equals(((TestRequest) task.getRequest()).getTestParam()) && @@ -304,9 +305,9 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { private ClusterState insignificantChange(ClusterState clusterState) { ClusterState.Builder builder = ClusterState.builder(clusterState); - PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); tasks = removeTasksWithChangingAssignment(tasks); - PersistentTasks.Builder tasksBuilder = PersistentTasks.builder(tasks); + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); if (randomBoolean()) { if (hasAssignableTasks(tasks, clusterState.nodes()) == false) { @@ -328,7 +329,7 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { } else { logger.info("changed routing table"); MetaData.Builder metaData = MetaData.builder(clusterState.metaData()); - metaData.putCustom(PersistentTasks.TYPE, tasksBuilder.build()); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()); RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable()); changeRoutingTable(metaData, routingTable); builder.metaData(metaData).routingTable(routingTable.build()); @@ -350,12 +351,12 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { // clear the task if (randomBoolean()) { logger.info("removed all tasks"); - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, - PersistentTasks.builder().build()); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, + PersistentTasksCustomMetaData.builder().build()); return builder.metaData(metaData).build(); } else { logger.info("set task custom to null"); - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).removeCustom(PersistentTasks.TYPE); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).removeCustom(PersistentTasksCustomMetaData.TYPE); return builder.metaData(metaData).build(); } } @@ -374,11 +375,11 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { .numberOfReplicas(1) .build(); MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).put(indexMetaData, false) - .putCustom(PersistentTasks.TYPE, tasksBuilder.build()); + .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()); return builder.metaData(metaData).build(); } - private boolean hasAssignableTasks(PersistentTasks tasks, DiscoveryNodes discoveryNodes) { + private boolean hasAssignableTasks(PersistentTasksCustomMetaData tasks, DiscoveryNodes discoveryNodes) { if (tasks == null || tasks.tasks().isEmpty()) { return false; } @@ -393,26 +394,26 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { }); } - private boolean hasTasksAssignedTo(PersistentTasks tasks, String nodeId) { + private boolean hasTasksAssignedTo(PersistentTasksCustomMetaData tasks, String nodeId) { return tasks != null && tasks.tasks().stream().anyMatch( task -> nodeId.equals(task.getExecutorNode())) == false; } private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, - MetaData.Builder metaData, PersistentTasks.Builder tasks, + MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, String node, boolean stopped) { return addRandomTask(clusterStateBuilder, metaData, tasks, new Assignment(node, randomAsciiOfLength(10)), randomAsciiOfLength(10), stopped); } private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, - MetaData.Builder metaData, PersistentTasks.Builder tasks, + MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, Assignment assignment, String param, boolean stopped) { - return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasks.TYPE, + return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks.addTask(randomAsciiOfLength(10), new TestRequest(param), stopped, randomBoolean(), assignment).build())); } - private void addTask(PersistentTasks.Builder tasks, String action, String param, String node, boolean stopped) { + private void addTask(PersistentTasksCustomMetaData.Builder tasks, String action, String param, String node, boolean stopped) { tasks.addTask(action, new TestRequest(param), stopped, randomBoolean(), new Assignment(node, "explanation: " + action)); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksTests.java b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksCustomMetaDataTests.java similarity index 81% rename from plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksTests.java rename to plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksCustomMetaDataTests.java index ea2ac1577fb..4a89dd4d246 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksCustomMetaDataTests.java @@ -21,12 +21,12 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractDiffableSerializationTestCase; -import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; -import org.elasticsearch.xpack.persistent.PersistentTasks.Builder; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; -import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.Status; -import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestPersistentAction; -import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Builder; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.Status; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest; import java.io.IOException; import java.util.ArrayList; @@ -35,17 +35,17 @@ import java.util.Collections; import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_GATEWAY; import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_SNAPSHOT; -import static org.elasticsearch.xpack.persistent.TransportPersistentAction.NO_NODE_FOUND; +import static org.elasticsearch.xpack.persistent.PersistentTasksExecutor.NO_NODE_FOUND; -public class PersistentTasksTests extends AbstractDiffableSerializationTestCase { +public class PersistentTasksCustomMetaDataTests extends AbstractDiffableSerializationTestCase { @Override - protected PersistentTasks createTestInstance() { + protected PersistentTasksCustomMetaData createTestInstance() { int numberOfTasks = randomInt(10); - PersistentTasks.Builder tasks = PersistentTasks.builder(); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); for (int i = 0; i < numberOfTasks; i++) { boolean stopped = randomBoolean(); - tasks.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), + tasks.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(), stopped ? new Assignment(null, "stopped") : randomAssignment()); if (randomBoolean()) { // From time to time update status @@ -57,22 +57,22 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase< @Override protected Writeable.Reader instanceReader() { - return PersistentTasks::new; + return PersistentTasksCustomMetaData::new; } @Override protected NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(Arrays.asList( - new Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new), - new Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom), - new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new), + new Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::new), + new Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::readDiffFrom), + new Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new), new Entry(Task.Status.class, Status.NAME, Status::new) )); } @Override protected Custom makeTestChanges(Custom testInstance) { - PersistentTasks tasksInProgress = (PersistentTasks) testInstance; + PersistentTasksCustomMetaData tasksInProgress = (PersistentTasksCustomMetaData) testInstance; Builder builder = new Builder(); switch (randomInt(3)) { case 0: @@ -105,12 +105,12 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase< @Override protected Writeable.Reader> diffReader() { - return PersistentTasks::readDiffFrom; + return PersistentTasksCustomMetaData::readDiffFrom; } @Override - protected PersistentTasks doParseInstance(XContentParser parser) throws IOException { - return PersistentTasks.fromXContent(parser); + protected PersistentTasksCustomMetaData doParseInstance(XContentParser parser) throws IOException { + return PersistentTasksCustomMetaData.fromXContent(parser); } @Override @@ -137,19 +137,19 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase< private Builder addRandomTask(Builder builder) { boolean stopped = randomBoolean(); - builder.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(), + builder.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(), stopped ? new Assignment(null, "stopped") : randomAssignment()); return builder; } - private long pickRandomTask(PersistentTasks testInstance) { + private long pickRandomTask(PersistentTasksCustomMetaData testInstance) { return randomFrom(new ArrayList<>(testInstance.tasks())).getId(); } @Override protected NamedXContentRegistry xContentRegistry() { return new NamedXContentRegistry(Arrays.asList( - new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(TestPersistentAction.NAME), + new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(TestPersistentTasksExecutor.NAME), TestRequest::fromXContent), new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) )); @@ -157,9 +157,9 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase< @SuppressWarnings("unchecked") public void testSerializationContext() throws Exception { - PersistentTasks testInstance = createTestInstance(); + PersistentTasksCustomMetaData testInstance = createTestInstance(); for (int i = 0; i < randomInt(10); i++) { - testInstance = (PersistentTasks) makeTestChanges(testInstance); + testInstance = (PersistentTasksCustomMetaData) makeTestChanges(testInstance); } ToXContent.MapParams params = new ToXContent.MapParams( @@ -170,7 +170,7 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase< XContentBuilder shuffled = shuffleXContent(builder); XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled.bytes()); - PersistentTasks newInstance = doParseInstance(parser); + PersistentTasksCustomMetaData newInstance = doParseInstance(parser); assertNotSame(newInstance, testInstance); assertEquals(testInstance.tasks().size(), newInstance.tasks().size()); @@ -179,7 +179,7 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase< assertNotNull(newTask); // Things that should be serialized - assertEquals(testTask.getAction(), newTask.getAction()); + assertEquals(testTask.getTaskName(), newTask.getTaskName()); assertEquals(testTask.getId(), newTask.getId()); assertEquals(testTask.getStatus(), newTask.getStatus()); assertEquals(testTask.getRequest(), newTask.getRequest()); @@ -192,7 +192,7 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase< } public void testBuilder() { - PersistentTasks persistentTasks = null; + PersistentTasksCustomMetaData persistentTasks = null; long lastKnownTask = -1; for (int i = 0; i < randomIntBetween(10, 100); i++) { final Builder builder; diff --git a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionFullRestartIT.java b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorFullRestartIT.java similarity index 59% rename from plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionFullRestartIT.java rename to plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorFullRestartIT.java index d784f1cc6ae..e105678cc71 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionFullRestartIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorFullRestartIT.java @@ -8,14 +8,16 @@ package org.elasticsearch.xpack.persistent; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; -import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestPersistentAction; -import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest; +import org.elasticsearch.xpack.persistent.PersistentTasksExecutorIT.PersistentTaskOperationFuture; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.List; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -23,10 +25,10 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, minNumDataNodes = 1) -public class PersistentActionFullRestartIT extends ESIntegTestCase { +public class PersistentTasksExecutorFullRestartIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singletonList(TestPersistentActionPlugin.class); + return Collections.singletonList(TestPersistentTasksPlugin.class); } @Override @@ -40,35 +42,38 @@ public class PersistentActionFullRestartIT extends ESIntegTestCase { @TestLogging("org.elasticsearch.xpack.persistent:TRACE,org.elasticsearch.cluster.service:DEBUG") public void testFullClusterRestart() throws Exception { + PersistentTasksService service = internalCluster().getInstance(PersistentTasksService.class); int numberOfTasks = randomIntBetween(1, 10); long[] taskIds = new long[numberOfTasks]; + List futures = new ArrayList<>(numberOfTasks); + boolean[] stopped = new boolean[numberOfTasks]; int runningTasks = 0; for (int i = 0; i < numberOfTasks; i++) { - if (randomBoolean()) { + stopped[i] = randomBoolean(); + if (stopped[i] == false) { runningTasks++; - taskIds[i] = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId(); - stopped[i] = false; - } else { - taskIds[i] = CreatePersistentTaskAction.INSTANCE.newRequestBuilder(client()) - .setAction(TestPersistentAction.NAME) - .setRequest(new TestRequest("Blah")) - .setStopped(true) - .get().getTaskId(); - stopped[i] = true; } + PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + futures.add(future); + service.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), stopped[i], true, future); } + + for (int i = 0; i < numberOfTasks; i++) { + taskIds[i] = futures.get(i).get(); + } + final int numberOfRunningTasks = runningTasks; - PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); if (numberOfRunningTasks > 0) { // Make sure that at least one of the tasks is running assertBusy(() -> { // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() - .getTasks().size(), greaterThan(0)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), greaterThan(0)); }); } @@ -76,7 +81,7 @@ public class PersistentActionFullRestartIT extends ESIntegTestCase { internalCluster().fullRestart(); ensureYellow(); - tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasks.TYPE); + tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); // Check that cluster state is correct for (int i = 0; i < numberOfTasks; i++) { @@ -88,12 +93,13 @@ public class PersistentActionFullRestartIT extends ESIntegTestCase { logger.info("Waiting for {} original tasks to start", numberOfRunningTasks); assertBusy(() -> { // Wait for the running task to start automatically - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), - equalTo(numberOfRunningTasks)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(numberOfRunningTasks)); }); // Start all other tasks - tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasks.TYPE); + tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + service = internalCluster().getInstance(PersistentTasksService.class); for (int i = 0; i < numberOfTasks; i++) { PersistentTask task = tasksInProgress.getTask(taskIds[i]); assertNotNull(task); @@ -101,26 +107,28 @@ public class PersistentActionFullRestartIT extends ESIntegTestCase { assertThat(task.isStopped(), equalTo(stopped[i])); assertThat(task.getExecutorNode(), stopped[i] ? nullValue() : notNullValue()); if (stopped[i]) { - assertAcked(StartPersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(task.getId()).get()); + PersistentTaskOperationFuture startFuture = new PersistentTaskOperationFuture(); + service.startTask(task.getId(), startFuture); + assertEquals(startFuture.get(), (Long) task.getId()); } } logger.info("Waiting for {} tasks to start", numberOfTasks); assertBusy(() -> { // Wait for all tasks to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), - equalTo(numberOfTasks)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(numberOfTasks)); }); logger.info("Complete all tasks"); // Complete the running task and make sure it finishes properly - assertThat(new TestPersistentActionPlugin.TestTasksRequestBuilder(client()).setOperation("finish").get().getTasks().size(), + assertThat(new TestPersistentTasksPlugin.TestTasksRequestBuilder(client()).setOperation("finish").get().getTasks().size(), equalTo(numberOfTasks)); assertBusy(() -> { // Make sure the task is removed from the cluster state - assertThat(((PersistentTasks) internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE)).tasks(), empty()); + assertThat(((PersistentTasksCustomMetaData) internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE)).tasks(), empty()); }); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionIT.java b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorIT.java similarity index 61% rename from plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionIT.java rename to plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorIT.java index 3a37a5d92c7..e25d31d9a18 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorIT.java @@ -6,12 +6,15 @@ package org.elasticsearch.xpack.persistent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.BaseFuture; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestPersistentAction; -import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestTasksRequestBuilder; +import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestTasksRequestBuilder; import org.junit.After; import java.util.Collection; @@ -26,11 +29,11 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, minNumDataNodes = 2) -public class PersistentActionIT extends ESIntegTestCase { +public class PersistentTasksExecutorIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singletonList(TestPersistentActionPlugin.class); + return Collections.singletonList(TestPersistentTasksPlugin.class); } @Override @@ -47,14 +50,30 @@ public class PersistentActionIT extends ESIntegTestCase { assertNoRunningTasks(); } + public static class PersistentTaskOperationFuture extends BaseFuture implements PersistentTaskOperationListener { + + @Override + public void onResponse(long taskId) { + set(taskId); + } + + @Override + public void onFailure(Exception e) { + setException(e); + } + } + public void testPersistentActionRestart() throws Exception { - long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId(); + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + long taskId = future.get(); assertBusy(() -> { // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), - equalTo(1)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(1)); }); - TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId()); // Verifying parent @@ -68,7 +87,7 @@ public class PersistentActionIT extends ESIntegTestCase { assertBusy(() -> { // Wait for the task to restart - List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() + List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() .getTasks(); logger.info("Found {} tasks", tasks.size()); assertThat(tasks.size(), equalTo(1)); @@ -78,24 +97,29 @@ public class PersistentActionIT extends ESIntegTestCase { logger.info("Removing persistent task with id {}", firstRunningTask.getId()); // Remove the persistent task - assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); + PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture(); + persistentTasksService.removeTask(taskId, removeFuture); + assertEquals(removeFuture.get(), (Long) taskId); logger.info("Waiting for persistent task with id {} to disappear", firstRunningTask.getId()); assertBusy(() -> { // Wait for the task to disappear completely - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(), + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(), empty()); }); } public void testPersistentActionCompletion() throws Exception { - long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId(); + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + long taskId = future.get(); assertBusy(() -> { // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), - equalTo(1)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(1)); }); - TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId()); // Verifying parent @@ -106,14 +130,14 @@ public class PersistentActionIT extends ESIntegTestCase { public void testPersistentActionCompletionWithoutRemoval() throws Exception { boolean stopped = randomBoolean(); - long taskId = CreatePersistentTaskAction.INSTANCE.newRequestBuilder(client()) - .setAction(TestPersistentAction.NAME) - .setRequest(new TestPersistentActionPlugin.TestRequest("Blah")) - .setRemoveOnCompletion(false) - .setStopped(stopped).get().getTaskId(); + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), stopped, false, + future); + long taskId = future.get(); - PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(1)); assertThat(tasksInProgress.getTask(taskId).isStopped(), equalTo(stopped)); assertThat(tasksInProgress.getTask(taskId).getExecutorNode(), stopped ? nullValue() : notNullValue()); @@ -124,24 +148,26 @@ public class PersistentActionIT extends ESIntegTestCase { for (int i = 0; i < numberOfIters; i++) { logger.info("iteration {}", i); if (stopped) { - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(), - empty()); - assertAcked(StartPersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks(), empty()); + PersistentTaskOperationFuture startFuture = new PersistentTaskOperationFuture(); + persistentTasksService.startTask(taskId, startFuture); + assertEquals(startFuture.get(), (Long) taskId); } assertBusy(() -> { // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks() - .size(), equalTo(1)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(1)); }); - TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); stopOrCancelTask(firstRunningTask.getTaskId()); assertBusy(() -> { // Wait for the task to finish - List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() - .getTasks(); + List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") + .get().getTasks(); logger.info("Found {} tasks", tasks.size()); assertThat(tasks.size(), equalTo(0)); }); @@ -150,30 +176,36 @@ public class PersistentActionIT extends ESIntegTestCase { assertBusy(() -> { // Wait for the task to be marked as stopped - PersistentTasks tasks = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasks.tasks().size(), equalTo(1)); assertThat(tasks.getTask(taskId).isStopped(), equalTo(true)); assertThat(tasks.getTask(taskId).shouldRemoveOnCompletion(), equalTo(false)); }); logger.info("Removing action record from cluster state"); - assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); + PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture(); + persistentTasksService.removeTask(taskId, removeFuture); + assertEquals(removeFuture.get(), (Long) taskId); } public void testPersistentActionWithNoAvailableNode() throws Exception { - long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah") - .executorNodeAttr("test").get().getTaskId(); + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + TestRequest testRequest = new TestRequest("Blah"); + testRequest.setExecutorNodeAttr("test"); + persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, testRequest, future); + long taskId = future.get(); Settings nodeSettings = Settings.builder().put(nodeSettings(0)).put("node.attr.test_attr", "test").build(); String newNode = internalCluster().startNode(nodeSettings); String newNodeId = internalCluster().clusterService(newNode).localNode().getId(); assertBusy(() -> { // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), - equalTo(1)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks() + .size(), equalTo(1)); }); - TaskInfo taskInfo = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + TaskInfo taskInfo = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); // Verifying the the task runs on the new node @@ -183,27 +215,32 @@ public class PersistentActionIT extends ESIntegTestCase { assertBusy(() -> { // Wait for the task to disappear completely - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(), + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(), empty()); }); // Remove the persistent task - assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); - + PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture(); + persistentTasksService.removeTask(taskId, removeFuture); + assertEquals(removeFuture.get(), (Long) taskId); } public void testPersistentActionStatusUpdate() throws Exception { - TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get(); + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + future.get(); + assertBusy(() -> { // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), - equalTo(1)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks() + .size(), equalTo(1)); }); - TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); - PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(1)); assertThat(tasksInProgress.tasks().iterator().next().getStatus(), nullValue()); @@ -216,8 +253,8 @@ public class PersistentActionIT extends ESIntegTestCase { int finalI = i; assertBusy(() -> { - PersistentTasks tasks = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasks.tasks().size(), equalTo(1)); assertThat(tasks.tasks().iterator().next().getStatus(), notNullValue()); assertThat(tasks.tasks().iterator().next().getStatus().toString(), equalTo("{\"phase\":\"phase " + (finalI + 1) + "\"}")); @@ -252,14 +289,14 @@ public class PersistentActionIT extends ESIntegTestCase { private void assertNoRunningTasks() throws Exception { assertBusy(() -> { // Wait for the task to finish - List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() + List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() .getTasks(); logger.info("Found {} tasks", tasks.size()); assertThat(tasks.size(), equalTo(0)); // Make sure the task is removed from the cluster state - assertThat(((PersistentTasks) internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE)).tasks(), empty()); + assertThat(((PersistentTasksCustomMetaData) internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE)).tasks(), empty()); }); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionResponseTests.java b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorResponseTests.java similarity index 53% rename from plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionResponseTests.java rename to plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorResponseTests.java index afb50ea725a..28f078cd8b5 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionResponseTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorResponseTests.java @@ -7,15 +7,15 @@ package org.elasticsearch.xpack.persistent; import org.elasticsearch.test.AbstractStreamableTestCase; -public class PersistentActionResponseTests extends AbstractStreamableTestCase { +public class PersistentTasksExecutorResponseTests extends AbstractStreamableTestCase { @Override - protected PersistentActionResponse createTestInstance() { - return new PersistentActionResponse(randomLong()); + protected PersistentTaskResponse createTestInstance() { + return new PersistentTaskResponse(randomLong()); } @Override - protected PersistentActionResponse createBlankInstance() { - return new PersistentActionResponse(); + protected PersistentTaskResponse createBlankInstance() { + return new PersistentTaskResponse(); } } \ No newline at end of file diff --git a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinatorStatusTests.java b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeServiceStatusTests.java similarity index 75% rename from plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinatorStatusTests.java rename to plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeServiceStatusTests.java index 05d193f6eb1..4e7e5d57135 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinatorStatusTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeServiceStatusTests.java @@ -7,12 +7,12 @@ package org.elasticsearch.xpack.persistent; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.persistent.PersistentActionCoordinator.State; -import org.elasticsearch.xpack.persistent.PersistentActionCoordinator.Status; +import org.elasticsearch.xpack.persistent.PersistentTasksNodeService.State; +import org.elasticsearch.xpack.persistent.PersistentTasksNodeService.Status; import static org.hamcrest.Matchers.containsString; -public class PersistentActionCoordinatorStatusTests extends AbstractWireSerializingTestCase { +public class PersistentTasksNodeServiceStatusTests extends AbstractWireSerializingTestCase { @Override protected Status createTestInstance() { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinatorTests.java b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeServiceTests.java similarity index 58% rename from plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinatorTests.java rename to plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeServiceTests.java index 53ef7ca6301..d8cae0cbb4b 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinatorTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeServiceTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.persistent; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; @@ -22,12 +21,13 @@ import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; -import org.elasticsearch.xpack.persistent.CompletionPersistentTaskAction.Response; -import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; -import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest; +import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -37,7 +37,7 @@ import static org.hamcrest.core.IsEqual.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class PersistentActionCoordinatorTests extends ESTestCase { +public class PersistentTasksNodeServiceTests extends ESTestCase { private ClusterService createClusterService() { ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); @@ -58,21 +58,21 @@ public class PersistentActionCoordinatorTests extends ESTestCase { public void testStartTask() throws Exception { ClusterService clusterService = createClusterService(); - PersistentActionService persistentActionService = mock(PersistentActionService.class); - PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); - @SuppressWarnings("unchecked") TransportPersistentAction action = mock(TransportPersistentAction.class); + PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); + @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); - registry.registerPersistentAction("test", action); + when(action.getTaskName()).thenReturn("test"); + PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); int nonLocalNodesCount = randomInt(10); MockExecutor executor = new MockExecutor(); - PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService, - registry, new TaskManager(Settings.EMPTY), executor); + PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, + registry, new TaskManager(Settings.EMPTY), mock(ThreadPool.class), executor); ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) .build(); - PersistentTasks.Builder tasks = PersistentTasks.builder(); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); boolean added = false; if (nonLocalNodesCount > 0) { for (int i = 0; i < randomInt(5); i++) { @@ -91,7 +91,7 @@ public class PersistentActionCoordinatorTests extends ESTestCase { } MetaData.Builder metaData = MetaData.builder(state.metaData()); - metaData.putCustom(PersistentTasks.TYPE, tasks.build()); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); ClusterState newClusterState = ClusterState.builder(state).metaData(metaData).build(); coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); @@ -149,24 +149,24 @@ public class PersistentActionCoordinatorTests extends ESTestCase { public void testTaskCancellation() { ClusterService clusterService = createClusterService(); AtomicLong capturedTaskId = new AtomicLong(); - AtomicReference> capturedListener = new AtomicReference<>(); - PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, null, null, null) { + AtomicReference capturedListener = new AtomicReference<>(); + PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, null, null) { @Override - public void sendCancellation(long taskId, ActionListener listener) { + public void sendCancellation(long taskId, PersistentTaskOperationListener listener) { capturedTaskId.set(taskId); capturedListener.set(listener); } }; - PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); - @SuppressWarnings("unchecked") TransportPersistentAction action = mock(TransportPersistentAction.class); + @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); - registry.registerPersistentAction("test", action); + when(action.getTaskName()).thenReturn("test"); + PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); int nonLocalNodesCount = randomInt(10); MockExecutor executor = new MockExecutor(); TaskManager taskManager = new TaskManager(Settings.EMPTY); - PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService, - registry, taskManager, executor); + PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, + registry, taskManager, mock(ThreadPool.class), executor); ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) .build(); @@ -203,7 +203,7 @@ public class PersistentActionCoordinatorTests extends ESTestCase { // That should trigger cancellation request assertThat(capturedTaskId.get(), equalTo(localId)); // Notify successful cancellation - capturedListener.get().onResponse(new CancelTasksResponse()); + capturedListener.get().onResponse(localId); // finish or fail task if (randomBoolean()) { @@ -217,106 +217,118 @@ public class PersistentActionCoordinatorTests extends ESTestCase { } - public void testNotificationFailure() { - ClusterService clusterService = createClusterService(); - AtomicLong capturedTaskId = new AtomicLong(-1L); - AtomicReference capturedException = new AtomicReference<>(); - AtomicReference> capturedListener = new AtomicReference<>(); - PersistentActionService persistentActionService = - new PersistentActionService(Settings.EMPTY, mock(ThreadPool.class), clusterService, null) { - @Override - public void sendCompletionNotification(long taskId, Exception failure, ActionListener listener) { - capturedTaskId.set(taskId); - capturedException.set(failure); - capturedListener.set(listener); - } - }; - PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); - @SuppressWarnings("unchecked") TransportPersistentAction action = mock(TransportPersistentAction.class); - when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); - registry.registerPersistentAction("test", action); - - int nonLocalNodesCount = randomInt(10); - MockExecutor executor = new MockExecutor(); - TaskManager taskManager = new TaskManager(Settings.EMPTY); - PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService, - registry, taskManager, executor); - - ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) + public void testNotificationFailure() throws Exception { + Settings settings = Settings.builder() + .put("node.name", PersistentTasksNodeServiceTests.class.getSimpleName()) .build(); + ThreadPool threadPool = new ThreadPool(settings); + try { + ClusterService clusterService = createClusterService(); + AtomicLong capturedTaskId = new AtomicLong(-1L); + AtomicReference capturedException = new AtomicReference<>(); + AtomicReference capturedListener = new AtomicReference<>(); + PersistentTasksService persistentTasksService = + new PersistentTasksService(Settings.EMPTY, clusterService, null) { + @Override + public void sendCompletionNotification(long taskId, Exception failure, PersistentTaskOperationListener listener) { + capturedTaskId.set(taskId); + capturedException.set(failure); + capturedListener.set(listener); + } + }; + @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); + when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + when(action.getTaskName()).thenReturn("test"); + PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, + Collections.singletonList(action)); - ClusterState newClusterState = state; - // Allocate first task - state = newClusterState; - newClusterState = addTask(state, "test", new TestRequest(), "this_node"); - coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + int nonLocalNodesCount = randomInt(10); + MockExecutor executor = new MockExecutor(); + TaskManager taskManager = new TaskManager(Settings.EMPTY); + PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, + registry, taskManager, threadPool, executor); - // Fail the task - executor.get(0).listener.onFailure(new RuntimeException("test failure")); + ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) + .build(); - // Check that notification was sent - assertThat(capturedException.get().getMessage(), equalTo("test failure")); - capturedException.set(null); + ClusterState newClusterState = state; + // Allocate first task + state = newClusterState; + newClusterState = addTask(state, "test", new TestRequest(), "this_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); - // Simulate failure to notify - capturedListener.get().onFailure(new IOException("simulated notification failure")); + // Fail the task + executor.get(0).listener.onFailure(new RuntimeException("test failure")); - // Allocate another task - state = newClusterState; - newClusterState = addTask(state, "test", new TestRequest(), "other_node"); - coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + // Check that notification was sent + assertThat(capturedException.get().getMessage(), equalTo("test failure")); + capturedException.set(null); - // Check that notification was sent again - assertThat(capturedException.get().getMessage(), equalTo("test failure")); + // Simulate failure to notify + capturedListener.get().onFailure(new IOException("simulated notification failure")); - // Check the the task is still known by the task manager - assertThat(taskManager.getTasks().size(), equalTo(1)); - long id = taskManager.getTasks().values().iterator().next().getParentTaskId().getId(); + // Allocate another task + state = newClusterState; + newClusterState = addTask(state, "test", new TestRequest(), "other_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); - // This time acknowledge notification - capturedListener.get().onResponse(new Response()); + // Check that notification was sent again + assertBusy(() -> { + assertNotNull(capturedException.get()); + assertThat(capturedException.get().getMessage(), equalTo("test failure")); + }); - // Reallocate failed task to another node - state = newClusterState; - newClusterState = reallocateTask(state, id, "other_node"); - coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + // Check the the task is still known by the task manager + assertThat(taskManager.getTasks().size(), equalTo(1)); + long id = taskManager.getTasks().values().iterator().next().getParentTaskId().getId(); - // Check the the task is now removed from task manager - assertThat(taskManager.getTasks().values(), empty()); + // This time acknowledge notification + capturedListener.get().onResponse(id); + + // Reallocate failed task to another node + state = newClusterState; + newClusterState = reallocateTask(state, id, "other_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Check the the task is now removed from task manager + assertThat(taskManager.getTasks().values(), empty()); + } finally { + assertTrue(ESTestCase.terminate(threadPool)); + } } - private ClusterState addTask(ClusterState state, String action, Request request, - String node) { - PersistentTasks.Builder builder = - PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE)); - return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE, + private ClusterState addTask(ClusterState state, String action, Request request, + String node) { + PersistentTasksCustomMetaData.Builder builder = + PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, builder.addTask(action, request, false, true, new Assignment(node, "test assignment")).build())).build(); } private ClusterState reallocateTask(ClusterState state, long taskId, String node) { - PersistentTasks.Builder builder = - PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE)); + PersistentTasksCustomMetaData.Builder builder = + PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); assertTrue(builder.hasTask(taskId)); - return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE, + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, builder.reassignTask(taskId, new Assignment(node, "test assignment")).build())).build(); } private ClusterState removeTask(ClusterState state, long taskId) { - PersistentTasks.Builder builder = - PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE)); + PersistentTasksCustomMetaData.Builder builder = + PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); assertTrue(builder.hasTask(taskId)); - return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE, + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, builder.removeTask(taskId).build())).build(); } private class Execution { - private final PersistentActionRequest request; + private final PersistentTaskRequest request; private final NodePersistentTask task; - private final PersistentActionRegistry.PersistentActionHolder holder; + private final PersistentTasksExecutor holder; private final ActionListener listener; - Execution(PersistentActionRequest request, NodePersistentTask task, PersistentActionRegistry.PersistentActionHolder holder, + Execution(PersistentTaskRequest request, NodePersistentTask task, PersistentTasksExecutor holder, ActionListener listener) { this.request = request; this.task = task; @@ -325,7 +337,7 @@ public class PersistentActionCoordinatorTests extends ESTestCase { } } - private class MockExecutor extends PersistentActionExecutor { + private class MockExecutor extends NodePersistentTasksExecutor { private List executions = new ArrayList<>(); MockExecutor() { @@ -333,10 +345,10 @@ public class PersistentActionCoordinatorTests extends ESTestCase { } @Override - public void executeAction(Request request, NodePersistentTask task, - PersistentActionRegistry.PersistentActionHolder holder, - ActionListener listener) { - executions.add(new Execution(request, task, holder, listener)); + public void executeTask(Request request, NodePersistentTask task, + PersistentTasksExecutor action, + ActionListener listener) { + executions.add(new Execution(request, task, action, listener)); } public Execution get(int i) { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/persistent/StartPersistentActionRequestTests.java b/plugin/src/test/java/org/elasticsearch/xpack/persistent/StartPersistentActionRequestTests.java index d3a6f54d73e..1943ce84003 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/persistent/StartPersistentActionRequestTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/persistent/StartPersistentActionRequestTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.xpack.persistent; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.xpack.persistent.CreatePersistentTaskAction.Request; -import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestPersistentAction; -import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest; import org.elasticsearch.test.AbstractStreamableTestCase; import java.util.Collections; @@ -39,7 +39,7 @@ public class StartPersistentActionRequestTests extends AbstractStreamableTestCas @Override protected NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(Collections.singletonList( - new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new) + new Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new) )); } } \ No newline at end of file diff --git a/plugin/src/test/java/org/elasticsearch/xpack/persistent/TestPersistentActionPlugin.java b/plugin/src/test/java/org/elasticsearch/xpack/persistent/TestPersistentTasksPlugin.java similarity index 82% rename from plugin/src/test/java/org/elasticsearch/xpack/persistent/TestPersistentActionPlugin.java rename to plugin/src/test/java/org/elasticsearch/xpack/persistent/TestPersistentTasksPlugin.java index 7389f2f4b09..fe2d6d16432 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/persistent/TestPersistentActionPlugin.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/persistent/TestPersistentTasksPlugin.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.persistent; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.FailedNodeException; @@ -49,7 +48,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; +import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment; import java.io.IOException; import java.util.ArrayList; @@ -72,12 +71,11 @@ import static org.junit.Assert.fail; /** * A plugin that adds a test persistent task. */ -public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { +public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin { @Override public List> getActions() { return Arrays.asList( - new ActionHandler<>(TestPersistentAction.INSTANCE, TransportTestPersistentAction.class), new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class), new ActionHandler<>(CreatePersistentTaskAction.INSTANCE, CreatePersistentTaskAction.TransportAction.class), new ActionHandler<>(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class), @@ -91,24 +89,28 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry) { - - PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, threadPool, clusterService, client); - PersistentActionRegistry persistentActionRegistry = new PersistentActionRegistry(Settings.EMPTY); + PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, client); + TestPersistentTasksExecutor testPersistentAction = new TestPersistentTasksExecutor(Settings.EMPTY, persistentTasksService, + clusterService); + PersistentTasksExecutorRegistry persistentTasksExecutorRegistry = new PersistentTasksExecutorRegistry(Settings.EMPTY, + Collections.singletonList(testPersistentAction)); return Arrays.asList( - persistentActionService, - persistentActionRegistry, - new PersistentTaskClusterService(Settings.EMPTY, persistentActionRegistry, clusterService) + persistentTasksService, + persistentTasksExecutorRegistry, + new PersistentTasksClusterService(Settings.EMPTY, persistentTasksExecutorRegistry, clusterService) ); } @Override public List getNamedWriteables() { return Arrays.asList( - new NamedWriteableRegistry.Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new), + new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new), new NamedWriteableRegistry.Entry(Task.Status.class, - PersistentActionCoordinator.Status.NAME, PersistentActionCoordinator.Status::new), - new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new), - new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom), + PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new), + new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, + PersistentTasksCustomMetaData::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE, + PersistentTasksCustomMetaData::readDiffFrom), new NamedWriteableRegistry.Entry(Task.Status.class, Status.NAME, Status::new) ); } @@ -116,18 +118,18 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { @Override public List getNamedXContent() { return Arrays.asList( - new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasks.TYPE), - PersistentTasks::fromXContent), - new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(TestPersistentAction.NAME), + new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksCustomMetaData.TYPE), + PersistentTasksCustomMetaData::fromXContent), + new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(TestPersistentTasksExecutor.NAME), TestRequest::fromXContent), new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) ); } - public static class TestRequest extends PersistentActionRequest { + public static class TestRequest extends PersistentTaskRequest { public static final ConstructingObjectParser REQUEST_PARSER = - new ConstructingObjectParser<>(TestPersistentAction.NAME, args -> new TestRequest((String) args[0])); + new ConstructingObjectParser<>(TestPersistentTasksExecutor.NAME, args -> new TestRequest((String) args[0])); static { REQUEST_PARSER.declareString(constructorArg(), new ParseField("param")); @@ -158,7 +160,7 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { @Override public String getWriteableName() { - return TestPersistentAction.NAME; + return TestPersistentTasksExecutor.NAME; } public void setExecutorNodeAttr(String executorNodeAttr) { @@ -226,53 +228,13 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { } } - public static class TestPersistentTaskRequestBuilder extends - ActionRequestBuilder { - - protected TestPersistentTaskRequestBuilder(ElasticsearchClient client, Action action, TestRequest request) { - super(client, action, request); - } - - public TestPersistentTaskRequestBuilder testParam(String testParam) { - request.setTestParam(testParam); - return this; - } - - public TestPersistentTaskRequestBuilder executorNodeAttr(String targetNode) { - request.setExecutorNodeAttr(targetNode); - return this; - } - - } - - public static class TestPersistentAction extends Action { - - public static final TestPersistentAction INSTANCE = new TestPersistentAction(); - public static final String NAME = "cluster:admin/persistent/test"; - - private TestPersistentAction() { - super(NAME); - } - - @Override - public PersistentActionResponse newResponse() { - return new PersistentActionResponse(); - } - - @Override - public TestPersistentTaskRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new TestPersistentTaskRequestBuilder(client, this, new TestRequest()); - } - } - public static class Status implements Task.Status { public static final String NAME = "test"; private final String phase; public static final ConstructingObjectParser STATUS_PARSER = - new ConstructingObjectParser<>(TestPersistentAction.NAME, args -> new Status((String) args[0])); + new ConstructingObjectParser<>(TestPersistentTasksExecutor.NAME, args -> new Status((String) args[0])); static { STATUS_PARSER.declareString(constructorArg(), new ParseField("phase")); @@ -336,19 +298,15 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { } - public static class TransportTestPersistentAction extends TransportPersistentAction { + public static class TestPersistentTasksExecutor extends PersistentTasksExecutor { - private final TransportService transportService; + public static final String NAME = "cluster:admin/persistent/test"; + private final ClusterService clusterService; - @Inject - public TransportTestPersistentAction(Settings settings, ThreadPool threadPool, TransportService transportService, - PersistentActionService persistentActionService, - PersistentActionRegistry persistentActionRegistry, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver) { - super(settings, TestPersistentAction.NAME, false, threadPool, transportService, persistentActionService, - persistentActionRegistry, actionFilters, indexNameExpressionResolver, TestRequest::new, - ThreadPool.Names.GENERIC); - this.transportService = transportService; + public TestPersistentTasksExecutor(Settings settings, PersistentTasksService persistentTasksService, + ClusterService clusterService) { + super(settings, NAME, persistentTasksService, ThreadPool.Names.GENERIC); + this.clusterService = clusterService; } @Override @@ -377,9 +335,9 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { // wait for something to happen assertTrue(awaitBusy(() -> testTask.isCancelled() || testTask.getOperation() != null || - transportService.lifecycleState() != Lifecycle.State.STARTED, // speedup finishing on closed nodes + clusterService.lifecycleState() != Lifecycle.State.STARTED, // speedup finishing on closed nodes 30, TimeUnit.SECONDS)); // This can take a while during large cluster restart - if (transportService.lifecycleState() != Lifecycle.State.STARTED) { + if (clusterService.lifecycleState() != Lifecycle.State.STARTED) { return; } if ("finish".equals(testTask.getOperation())) { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskRequestTests.java b/plugin/src/test/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskRequestTests.java index 33b8be1f659..c11a3123e31 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskRequestTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskRequestTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.persistent; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractStreamableTestCase; -import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.Status; +import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.Status; import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction.Request; import java.util.Collections; diff --git a/plugin/src/test/resources/org/elasticsearch/transport/actions b/plugin/src/test/resources/org/elasticsearch/transport/actions index 9b022aeb1d7..91da1ecd097 100644 --- a/plugin/src/test/resources/org/elasticsearch/transport/actions +++ b/plugin/src/test/resources/org/elasticsearch/transport/actions @@ -143,4 +143,4 @@ cluster:admin/persistent/create cluster:admin/persistent/start cluster:admin/persistent/completion cluster:admin/persistent/update_status -cluster:admin/persistent/remove +cluster:admin/persistent/remove \ No newline at end of file