honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make PersistentAction independent from TransportActions (elastic/x-pack-elasticsearch#742) 

Removes the transport layer dependency from PersistentActions, makes PersistentActionRegistry immutable and rename actions into tasks in class and variable names.

Original commit: elastic/x-pack-elasticsearch@e3e5b79c28
This commit is contained in:
Igor Motov 2017-03-22 14:02:30 -04:00 committed by GitHub
parent d779bf66a5
commit 5c4933f5ea
62 changed files with 1315 additions and 1165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
dev-tools/checkstyle_suppressions.xml
View File

@ -376,16 +376,17 @@
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]CompletionPersistentTaskAction.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]CompletionPersistentTaskAction.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]CreatePersistentTaskAction.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]CreatePersistentTaskAction.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]NodePersistentTask.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]NodePersistentTask.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionCoordinator.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]NodePersistentTasksExecutor.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionExecutor.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTaskRequest.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionRegistry.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTaskResponse.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionRequest.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksClusterService.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionService.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksCustomMetaData.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTaskClusterService.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksExecutor.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasks.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksExecutorRegistry.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksNodeService.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksService.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]RemovePersistentTaskAction.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]RemovePersistentTaskAction.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]StartPersistentTaskAction.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]StartPersistentTaskAction.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]TransportPersistentAction.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]UpdatePersistentTaskStatusAction.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]UpdatePersistentTaskStatusAction.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]package-info.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]package-info.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]rest[/\\]XPackRestHandler.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]rest[/\\]XPackRestHandler.java" checks="LineLength" />
@ -1089,16 +1090,15 @@
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]notification[/\\]slack[/\\]SlackAccountsTests.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]notification[/\\]slack[/\\]SlackAccountsTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]notification[/\\]slack[/\\]message[/\\]SlackMessageDefaultsTests.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]notification[/\\]slack[/\\]message[/\\]SlackMessageDefaultsTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]notification[/\\]slack[/\\]message[/\\]SlackMessageTests.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]notification[/\\]slack[/\\]message[/\\]SlackMessageTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionCoordinatorStatusTests.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksClusterServiceTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionCoordinatorTests.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksCustomMetaDataTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionFullRestartIT.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksExecutorFullRestartIT.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionIT.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksExecutorIT.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionRegistryTests.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksExecutorResponseTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentActionResponseTests.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksNodeServiceTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTaskClusterServiceTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksTests.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]PersistentTasksTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]StartPersistentActionRequestTests.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]StartPersistentActionRequestTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]TestPersistentActionPlugin.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]persistent[/\\]TestPersistentTasksPlugin.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]security[/\\]InternalClientIntegTests.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]security[/\\]InternalClientIntegTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]security[/\\]InternalClientTests.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]security[/\\]InternalClientTests.java" checks="LineLength" />
<suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]security[/\\]SecurityContextTests.java" checks="LineLength" /> <suppress files="plugin[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]security[/\\]SecurityContextTests.java" checks="LineLength" />

55
plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
View File

@ -123,12 +123,12 @@ import org.elasticsearch.xpack.ml.rest.validate.RestValidateDetectorAction;
import org.elasticsearch.xpack.ml.rest.validate.RestValidateJobConfigAction; import org.elasticsearch.xpack.ml.rest.validate.RestValidateJobConfigAction;
import org.elasticsearch.xpack.persistent.CompletionPersistentTaskAction; import org.elasticsearch.xpack.persistent.CompletionPersistentTaskAction;
import org.elasticsearch.xpack.persistent.CreatePersistentTaskAction; import org.elasticsearch.xpack.persistent.CreatePersistentTaskAction;
import org.elasticsearch.xpack.persistent.PersistentActionCoordinator; import org.elasticsearch.xpack.persistent.PersistentTasksClusterService;
import org.elasticsearch.xpack.persistent.PersistentActionRegistry; import org.elasticsearch.xpack.persistent.PersistentTasksNodeService;
import org.elasticsearch.xpack.persistent.PersistentActionRequest; import org.elasticsearch.xpack.persistent.PersistentTasksExecutorRegistry;
import org.elasticsearch.xpack.persistent.PersistentActionService; import org.elasticsearch.xpack.persistent.PersistentTaskRequest;
import org.elasticsearch.xpack.persistent.PersistentTaskClusterService; import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.RemovePersistentTaskAction; import org.elasticsearch.xpack.persistent.RemovePersistentTaskAction;
import org.elasticsearch.xpack.persistent.StartPersistentTaskAction; import org.elasticsearch.xpack.persistent.StartPersistentTaskAction;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction; import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction;
@ -208,16 +208,18 @@ public class MachineLearning implements ActionPlugin {
// Custom metadata // Custom metadata
new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new), new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new),
new NamedWriteableRegistry.Entry(NamedDiff.class, "ml", MlMetadata.MlMetadataDiff::new), new NamedWriteableRegistry.Entry(NamedDiff.class, "ml", MlMetadata.MlMetadataDiff::new),
new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new), new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE,
new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom), PersistentTasksCustomMetaData::new),
new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE,
PersistentTasksCustomMetaData::readDiffFrom),
// Persistent action requests // Persistent action requests
new NamedWriteableRegistry.Entry(PersistentActionRequest.class, StartDatafeedAction.NAME, StartDatafeedAction.Request::new), new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, StartDatafeedAction.NAME, StartDatafeedAction.Request::new),
new NamedWriteableRegistry.Entry(PersistentActionRequest.class, OpenJobAction.NAME, OpenJobAction.Request::new), new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, OpenJobAction.NAME, OpenJobAction.Request::new),
// Task statuses // Task statuses
new NamedWriteableRegistry.Entry(Task.Status.class, PersistentActionCoordinator.Status.NAME, new NamedWriteableRegistry.Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME,
PersistentActionCoordinator.Status::new), PersistentTasksNodeService.Status::new),
new NamedWriteableRegistry.Entry(Task.Status.class, JobState.NAME, JobState::fromStream), new NamedWriteableRegistry.Entry(Task.Status.class, JobState.NAME, JobState::fromStream),
new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream) new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream)
); );
@ -228,13 +230,13 @@ public class MachineLearning implements ActionPlugin {
// Custom metadata // Custom metadata
new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField("ml"), new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField("ml"),
parser -> MlMetadata.ML_METADATA_PARSER.parse(parser, null).build()), parser -> MlMetadata.ML_METADATA_PARSER.parse(parser, null).build()),
new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasks.TYPE), new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksCustomMetaData.TYPE),
PersistentTasks::fromXContent), PersistentTasksCustomMetaData::fromXContent),
// Persistent action requests // Persistent action requests
new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(StartDatafeedAction.NAME), new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(StartDatafeedAction.NAME),
StartDatafeedAction.Request::fromXContent), StartDatafeedAction.Request::fromXContent),
new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(OpenJobAction.NAME), new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(OpenJobAction.NAME),
OpenJobAction.Request::fromXContent), OpenJobAction.Request::fromXContent),
// Task statuses // Task statuses
@ -290,17 +292,22 @@ public class MachineLearning implements ActionPlugin {
} }
NormalizerFactory normalizerFactory = new NormalizerFactory(normalizerProcessFactory, NormalizerFactory normalizerFactory = new NormalizerFactory(normalizerProcessFactory,
threadPool.executor(MachineLearning.THREAD_POOL_NAME)); threadPool.executor(MachineLearning.THREAD_POOL_NAME));
PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, internalClient);
AutodetectProcessManager autodetectProcessManager = new AutodetectProcessManager(settings, internalClient, threadPool, AutodetectProcessManager autodetectProcessManager = new AutodetectProcessManager(settings, internalClient, threadPool,
jobManager, jobProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, jobManager, jobProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory,
normalizerFactory); normalizerFactory, persistentTasksService);
DatafeedJobRunner datafeedJobRunner = new DatafeedJobRunner(threadPool, internalClient, clusterService, jobProvider, DatafeedJobRunner datafeedJobRunner = new DatafeedJobRunner(threadPool, internalClient, clusterService, jobProvider,
System::currentTimeMillis, auditor); System::currentTimeMillis, persistentTasksService, auditor);
PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, threadPool, clusterService,
internalClient);
PersistentActionRegistry persistentActionRegistry = new PersistentActionRegistry(Settings.EMPTY);
InvalidLicenseEnforcer invalidLicenseEnforcer = InvalidLicenseEnforcer invalidLicenseEnforcer =
new InvalidLicenseEnforcer(settings, licenseState, threadPool, datafeedJobRunner, autodetectProcessManager); new InvalidLicenseEnforcer(settings, licenseState, threadPool, datafeedJobRunner, autodetectProcessManager);
PersistentTasksExecutorRegistry persistentTasksExecutorRegistry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Arrays.asList(
new OpenJobAction.OpenJobPersistentTasksExecutor(settings, threadPool, licenseState, persistentTasksService, clusterService,
autodetectProcessManager, auditor),
new StartDatafeedAction.StartDatafeedPersistentTasksExecutor(settings, threadPool, licenseState, persistentTasksService,
datafeedJobRunner, auditor)
));
return Arrays.asList( return Arrays.asList(
mlLifeCycleService, mlLifeCycleService,
jobProvider, jobProvider,
@ -310,9 +317,9 @@ public class MachineLearning implements ActionPlugin {
new MlInitializationService(settings, threadPool, clusterService, internalClient), new MlInitializationService(settings, threadPool, clusterService, internalClient),
jobDataCountsPersister, jobDataCountsPersister,
datafeedJobRunner, datafeedJobRunner,
persistentActionService, persistentTasksService,
persistentActionRegistry, persistentTasksExecutorRegistry,
new PersistentTaskClusterService(Settings.EMPTY, persistentActionRegistry, clusterService), new PersistentTasksClusterService(Settings.EMPTY, persistentTasksExecutorRegistry, clusterService),
auditor, auditor,
new CloseJobService(internalClient, threadPool, clusterService), new CloseJobService(internalClient, threadPool, clusterService),
invalidLicenseEnforcer invalidLicenseEnforcer

22
plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java
View File

@ -32,8 +32,8 @@ import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.messages.Messages; import org.elasticsearch.xpack.ml.job.messages.Messages;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException; import java.io.IOException;
import java.util.Collection; import java.util.Collection;
@ -240,7 +240,7 @@ public class MlMetadata implements MetaData.Custom {
return this; return this;
} }
public Builder deleteJob(String jobId, PersistentTasks tasks) { public Builder deleteJob(String jobId, PersistentTasksCustomMetaData tasks) {
Optional<DatafeedConfig> datafeed = getDatafeedByJobId(jobId); Optional<DatafeedConfig> datafeed = getDatafeedByJobId(jobId);
if (datafeed.isPresent()) { if (datafeed.isPresent()) {
throw ExceptionsHelper.conflictStatusException("Cannot delete job [" + jobId + "] while datafeed [" throw ExceptionsHelper.conflictStatusException("Cannot delete job [" + jobId + "] while datafeed ["
@ -285,7 +285,7 @@ public class MlMetadata implements MetaData.Custom {
} }
} }
public Builder updateDatafeed(DatafeedUpdate update, PersistentTasks persistentTasks) { public Builder updateDatafeed(DatafeedUpdate update, PersistentTasksCustomMetaData persistentTasks) {
String datafeedId = update.getId(); String datafeedId = update.getId();
DatafeedConfig oldDatafeedConfig = datafeeds.get(datafeedId); DatafeedConfig oldDatafeedConfig = datafeeds.get(datafeedId);
if (oldDatafeedConfig == null) { if (oldDatafeedConfig == null) {
@ -303,7 +303,7 @@ public class MlMetadata implements MetaData.Custom {
return this; return this;
} }
public Builder removeDatafeed(String datafeedId, PersistentTasks persistentTasks) { public Builder removeDatafeed(String datafeedId, PersistentTasksCustomMetaData persistentTasks) {
DatafeedConfig datafeed = datafeeds.get(datafeedId); DatafeedConfig datafeed = datafeeds.get(datafeedId);
if (datafeed == null) { if (datafeed == null) {
throw ExceptionsHelper.missingDatafeedException(datafeedId); throw ExceptionsHelper.missingDatafeedException(datafeedId);
@ -318,7 +318,7 @@ public class MlMetadata implements MetaData.Custom {
return datafeeds.values().stream().filter(s -> s.getJobId().equals(jobId)).findFirst(); return datafeeds.values().stream().filter(s -> s.getJobId().equals(jobId)).findFirst();
} }
private void checkDatafeedIsStopped(Supplier<String> msg, String datafeedId, PersistentTasks persistentTasks) { private void checkDatafeedIsStopped(Supplier<String> msg, String datafeedId, PersistentTasksCustomMetaData persistentTasks) {
if (persistentTasks != null) { if (persistentTasks != null) {
Predicate<PersistentTask<?>> predicate = t -> { Predicate<PersistentTask<?>> predicate = t -> {
StartDatafeedAction.Request storedRequest = (StartDatafeedAction.Request) t.getRequest(); StartDatafeedAction.Request storedRequest = (StartDatafeedAction.Request) t.getRequest();
@ -348,7 +348,7 @@ public class MlMetadata implements MetaData.Custom {
return new MlMetadata(jobs, datafeeds); return new MlMetadata(jobs, datafeeds);
} }
public void markJobAsDeleted(String jobId, PersistentTasks tasks) { public void markJobAsDeleted(String jobId, PersistentTasksCustomMetaData tasks) {
Job job = jobs.get(jobId); Job job = jobs.get(jobId);
if (job == null) { if (job == null) {
throw ExceptionsHelper.missingJobException(jobId); throw ExceptionsHelper.missingJobException(jobId);
@ -375,7 +375,7 @@ public class MlMetadata implements MetaData.Custom {
} }
@Nullable @Nullable
public static PersistentTask<?> getJobTask(String jobId, @Nullable PersistentTasks tasks) { public static PersistentTask<?> getJobTask(String jobId, @Nullable PersistentTasksCustomMetaData tasks) {
if (tasks != null) { if (tasks != null) {
Predicate<PersistentTask<?>> p = t -> { Predicate<PersistentTask<?>> p = t -> {
OpenJobAction.Request storedRequest = (OpenJobAction.Request) t.getRequest(); OpenJobAction.Request storedRequest = (OpenJobAction.Request) t.getRequest();
@ -389,7 +389,7 @@ public class MlMetadata implements MetaData.Custom {
} }
@Nullable @Nullable
public static PersistentTask<?> getDatafeedTask(String datafeedId, @Nullable PersistentTasks tasks) { public static PersistentTask<?> getDatafeedTask(String datafeedId, @Nullable PersistentTasksCustomMetaData tasks) {
if (tasks != null) { if (tasks != null) {
Predicate<PersistentTask<?>> p = t -> { Predicate<PersistentTask<?>> p = t -> {
StartDatafeedAction.Request storedRequest = (StartDatafeedAction.Request) t.getRequest(); StartDatafeedAction.Request storedRequest = (StartDatafeedAction.Request) t.getRequest();
@ -402,7 +402,7 @@ public class MlMetadata implements MetaData.Custom {
return null; return null;
} }
public static JobState getJobState(String jobId, @Nullable PersistentTasks tasks) { public static JobState getJobState(String jobId, @Nullable PersistentTasksCustomMetaData tasks) {
PersistentTask<?> task = getJobTask(jobId, tasks); PersistentTask<?> task = getJobTask(jobId, tasks);
if (task != null && task.getStatus() != null) { if (task != null && task.getStatus() != null) {
JobState jobTaskState = (JobState) task.getStatus(); JobState jobTaskState = (JobState) task.getStatus();
@ -414,7 +414,7 @@ public class MlMetadata implements MetaData.Custom {
return JobState.CLOSED; return JobState.CLOSED;
} }
public static DatafeedState getDatafeedState(String datafeedId, @Nullable PersistentTasks tasks) { public static DatafeedState getDatafeedState(String datafeedId, @Nullable PersistentTasksCustomMetaData tasks) {
PersistentTask<?> task = getDatafeedTask(datafeedId, tasks); PersistentTask<?> task = getDatafeedTask(datafeedId, tasks);
if (task != null && task.getStatus() != null) { if (task != null && task.getStatus() != null) {
return (DatafeedState) task.getStatus(); return (DatafeedState) task.getStatus();

12
plugin/src/main/java/org/elasticsearch/xpack/ml/action/CloseJobAction.java
View File

@ -43,8 +43,8 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException; import java.io.IOException;
import java.util.Date; import java.util.Date;
@ -286,7 +286,7 @@ public class CloseJobAction extends Action<CloseJobAction.Request, CloseJobActio
throw ExceptionsHelper.missingJobException(jobId); throw ExceptionsHelper.missingJobException(jobId);
} }
PersistentTasks tasks = state.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
Optional<DatafeedConfig> datafeed = mlMetadata.getDatafeedByJobId(jobId); Optional<DatafeedConfig> datafeed = mlMetadata.getDatafeedByJobId(jobId);
if (datafeed.isPresent()) { if (datafeed.isPresent()) {
DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeed.get().getId(), tasks); DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeed.get().getId(), tasks);
@ -311,12 +311,12 @@ public class CloseJobAction extends Action<CloseJobAction.Request, CloseJobActio
static ClusterState moveJobToClosingState(String jobId, ClusterState currentState) { static ClusterState moveJobToClosingState(String jobId, ClusterState currentState) {
PersistentTask<?> task = validateAndFindTask(jobId, currentState); PersistentTask<?> task = validateAndFindTask(jobId, currentState);
PersistentTasks currentTasks = currentState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData currentTasks = currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
Map<Long, PersistentTask<?>> updatedTasks = new HashMap<>(currentTasks.taskMap()); Map<Long, PersistentTask<?>> updatedTasks = new HashMap<>(currentTasks.taskMap());
PersistentTask<?> taskToUpdate = currentTasks.getTask(task.getId()); PersistentTask<?> taskToUpdate = currentTasks.getTask(task.getId());
taskToUpdate = new PersistentTask<>(taskToUpdate, JobState.CLOSING); taskToUpdate = new PersistentTask<>(taskToUpdate, JobState.CLOSING);
updatedTasks.put(taskToUpdate.getId(), taskToUpdate); updatedTasks.put(taskToUpdate.getId(), taskToUpdate);
PersistentTasks newTasks = new PersistentTasks(currentTasks.getCurrentId(), updatedTasks); PersistentTasksCustomMetaData newTasks = new PersistentTasksCustomMetaData(currentTasks.getCurrentId(), updatedTasks);
MlMetadata mlMetadata = currentState.metaData().custom(MlMetadata.TYPE); MlMetadata mlMetadata = currentState.metaData().custom(MlMetadata.TYPE);
Job.Builder jobBuilder = new Job.Builder(mlMetadata.getJobs().get(jobId)); Job.Builder jobBuilder = new Job.Builder(mlMetadata.getJobs().get(jobId));
@ -328,7 +328,7 @@ public class CloseJobAction extends Action<CloseJobAction.Request, CloseJobActio
return builder return builder
.metaData(new MetaData.Builder(currentState.metaData()) .metaData(new MetaData.Builder(currentState.metaData())
.putCustom(MlMetadata.TYPE, mlMetadataBuilder.build()) .putCustom(MlMetadata.TYPE, mlMetadataBuilder.build())
.putCustom(PersistentTasks.TYPE, newTasks)) .putCustom(PersistentTasksCustomMetaData.TYPE, newTasks))
.build(); .build();
} }
} }

6
plugin/src/main/java/org/elasticsearch/xpack/ml/action/DeleteDatafeedAction.java
View File

@ -32,7 +32,7 @@ import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.io.IOException; import java.io.IOException;
import java.util.Objects; import java.util.Objects;
@ -170,8 +170,8 @@ public class DeleteDatafeedAction extends Action<DeleteDatafeedAction.Request, D
@Override @Override
public ClusterState execute(ClusterState currentState) throws Exception { public ClusterState execute(ClusterState currentState) throws Exception {
MlMetadata currentMetadata = state.getMetaData().custom(MlMetadata.TYPE); MlMetadata currentMetadata = state.getMetaData().custom(MlMetadata.TYPE);
PersistentTasks persistentTasks = PersistentTasksCustomMetaData persistentTasks =
state.getMetaData().custom(PersistentTasks.TYPE); state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata) MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata)
.removeDatafeed(request.getDatafeedId(), persistentTasks).build(); .removeDatafeed(request.getDatafeedId(), persistentTasks).build();
return ClusterState.builder(state).metaData( return ClusterState.builder(state).metaData(

8
plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetDatafeedsStatsAction.java
View File

@ -38,8 +38,8 @@ import org.elasticsearch.xpack.ml.action.util.QueryPage;
import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@ -318,7 +318,7 @@ public class GetDatafeedsStatsAction extends Action<GetDatafeedsStatsAction.Requ
.map(d -> d.getId()).collect(Collectors.toList()) .map(d -> d.getId()).collect(Collectors.toList())
: Collections.singletonList(request.getDatafeedId()); : Collections.singletonList(request.getDatafeedId());
PersistentTasks tasksInProgress = state.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasksInProgress = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
List<DatafeedStats> results = expandedDatafeedsIds.stream() List<DatafeedStats> results = expandedDatafeedsIds.stream()
.map(datafeedId -> getDatafeedStats(datafeedId, state, tasksInProgress)) .map(datafeedId -> getDatafeedStats(datafeedId, state, tasksInProgress))
.collect(Collectors.toList()); .collect(Collectors.toList());
@ -328,7 +328,7 @@ public class GetDatafeedsStatsAction extends Action<GetDatafeedsStatsAction.Requ
} }
private static DatafeedStats getDatafeedStats(String datafeedId, ClusterState state, private static DatafeedStats getDatafeedStats(String datafeedId, ClusterState state,
PersistentTasks tasks) { PersistentTasksCustomMetaData tasks) {
PersistentTask<?> task = MlMetadata.getDatafeedTask(datafeedId, tasks); PersistentTask<?> task = MlMetadata.getDatafeedTask(datafeedId, tasks);
DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeedId, tasks); DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeedId, tasks);
DiscoveryNode node = null; DiscoveryNode node = null;

10
plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsStatsAction.java
View File

@ -45,7 +45,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManage
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.io.IOException; import java.io.IOException;
import java.time.Duration; import java.time.Duration;
@ -410,10 +410,10 @@ public class GetJobsStatsAction extends Action<GetJobsStatsAction.Request, GetJo
String jobId = task.getJobId(); String jobId = task.getJobId();
logger.debug("Get stats for job [{}]", jobId); logger.debug("Get stats for job [{}]", jobId);
ClusterState state = clusterService.state(); ClusterState state = clusterService.state();
PersistentTasks tasks = state.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
Optional<Tuple<DataCounts, ModelSizeStats>> stats = processManager.getStatistics(jobId); Optional<Tuple<DataCounts, ModelSizeStats>> stats = processManager.getStatistics(jobId);
if (stats.isPresent()) { if (stats.isPresent()) {
PersistentTasks.PersistentTask<?> pTask = MlMetadata.getJobTask(jobId, tasks); PersistentTasksCustomMetaData.PersistentTask<?> pTask = MlMetadata.getJobTask(jobId, tasks);
DiscoveryNode node = state.nodes().get(pTask.getExecutorNode()); DiscoveryNode node = state.nodes().get(pTask.getExecutorNode());
JobState jobState = MlMetadata.getJobState(jobId, tasks); JobState jobState = MlMetadata.getJobState(jobId, tasks);
String assignmentExplanation = pTask.getAssignment().getExplanation(); String assignmentExplanation = pTask.getAssignment().getExplanation();
@ -437,13 +437,13 @@ public class GetJobsStatsAction extends Action<GetJobsStatsAction.Request, GetJo
AtomicInteger counter = new AtomicInteger(jobIds.size()); AtomicInteger counter = new AtomicInteger(jobIds.size());
AtomicArray<Response.JobStats> jobStats = new AtomicArray<>(jobIds.size()); AtomicArray<Response.JobStats> jobStats = new AtomicArray<>(jobIds.size());
PersistentTasks tasks = clusterService.state().getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
for (int i = 0; i < jobIds.size(); i++) { for (int i = 0; i < jobIds.size(); i++) {
int slot = i; int slot = i;
String jobId = jobIds.get(i); String jobId = jobIds.get(i);
gatherDataCountsAndModelSizeStats(jobId, (dataCounts, modelSizeStats) -> { gatherDataCountsAndModelSizeStats(jobId, (dataCounts, modelSizeStats) -> {
JobState jobState = MlMetadata.getJobState(request.jobId, tasks); JobState jobState = MlMetadata.getJobState(request.jobId, tasks);
PersistentTasks.PersistentTask<?> pTask = MlMetadata.getJobTask(jobId, tasks); PersistentTasksCustomMetaData.PersistentTask<?> pTask = MlMetadata.getJobTask(jobId, tasks);
String assignmentExplanation = null; String assignmentExplanation = null;
if (pTask != null) { if (pTask != null) {
assignmentExplanation = pTask.getAssignment().getExplanation(); assignmentExplanation = pTask.getAssignment().getExplanation();

164
plugin/src/main/java/org/elasticsearch/xpack/ml/action/OpenJobAction.java
View File

@ -12,6 +12,8 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@ -50,14 +52,13 @@ import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.ml.utils.JobStateObserver; import org.elasticsearch.xpack.ml.utils.JobStateObserver;
import org.elasticsearch.xpack.persistent.NodePersistentTask; import org.elasticsearch.xpack.persistent.NodePersistentTask;
import org.elasticsearch.xpack.persistent.PersistentActionRegistry; import org.elasticsearch.xpack.persistent.PersistentTasksExecutor;
import org.elasticsearch.xpack.persistent.PersistentActionRequest; import org.elasticsearch.xpack.persistent.PersistentTaskRequest;
import org.elasticsearch.xpack.persistent.PersistentActionResponse; import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.elasticsearch.xpack.persistent.PersistentActionService; import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.xpack.persistent.TransportPersistentAction;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@ -68,7 +69,7 @@ import java.util.Objects;
import static org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE; import static org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE;
public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActionResponse, OpenJobAction.RequestBuilder> { public class OpenJobAction extends Action<OpenJobAction.Request, OpenJobAction.Response, OpenJobAction.RequestBuilder> {
public static final OpenJobAction INSTANCE = new OpenJobAction(); public static final OpenJobAction INSTANCE = new OpenJobAction();
public static final String NAME = "cluster:admin/ml/job/open"; public static final String NAME = "cluster:admin/ml/job/open";
@ -83,11 +84,11 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
} }
@Override @Override
public PersistentActionResponse newResponse() { public Response newResponse() {
return new PersistentActionResponse(); return new Response();
} }
public static class Request extends PersistentActionRequest { public static class Request extends PersistentTaskRequest {
public static final ParseField IGNORE_DOWNTIME = new ParseField("ignore_downtime"); public static final ParseField IGNORE_DOWNTIME = new ParseField("ignore_downtime");
public static final ParseField TIMEOUT = new ParseField("timeout"); public static final ParseField TIMEOUT = new ParseField("timeout");
@ -124,7 +125,8 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
readFrom(in); readFrom(in);
} }
Request() {} Request() {
}
public String getJobId() { public String getJobId() {
return jobId; return jobId;
@ -216,6 +218,40 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
} }
} }
public static class Response extends AcknowledgedResponse {
public Response() {
super();
}
public Response(boolean acknowledged) {
super(acknowledged);
}
@Override
public void readFrom(StreamInput in) throws IOException {
readAcknowledged(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
writeAcknowledged(out);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AcknowledgedResponse that = (AcknowledgedResponse) o;
return isAcknowledged() == that.isAcknowledged();
}
@Override
public int hashCode() {
return Objects.hash(isAcknowledged());
}
}
public static class JobTask extends NodePersistentTask { public static class JobTask extends NodePersistentTask {
private final String jobId; private final String jobId;
@ -243,70 +279,81 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
} }
static class RequestBuilder extends ActionRequestBuilder<Request, PersistentActionResponse, RequestBuilder> { static class RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder> {
RequestBuilder(ElasticsearchClient client, OpenJobAction action) { RequestBuilder(ElasticsearchClient client, OpenJobAction action) {
super(client, action, new Request()); super(client, action, new Request());
} }
} }
public static class TransportAction extends TransportPersistentAction<Request> { public static class TransportAction extends HandledTransportAction<Request, Response> {
private final JobStateObserver observer; private final JobStateObserver observer;
private final ClusterService clusterService;
private final AutodetectProcessManager autodetectProcessManager;
private final XPackLicenseState licenseState; private final XPackLicenseState licenseState;
private final Auditor auditor; private final PersistentTasksService persistentTasksService;
private volatile int maxConcurrentJobAllocations;
@Inject @Inject
public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, XPackLicenseState licenseState, public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, XPackLicenseState licenseState,
PersistentActionService persistentActionService, PersistentActionRegistry persistentActionRegistry, PersistentTasksService persistentTasksService, ActionFilters actionFilters,
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService) {
ClusterService clusterService, AutodetectProcessManager autodetectProcessManager, Auditor auditor) { super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new);
super(settings, OpenJobAction.NAME, false, threadPool, transportService, persistentActionService,
persistentActionRegistry, actionFilters, indexNameExpressionResolver, Request::new, ThreadPool.Names.MANAGEMENT);
this.licenseState = licenseState; this.licenseState = licenseState;
this.clusterService = clusterService; this.persistentTasksService = persistentTasksService;
this.autodetectProcessManager = autodetectProcessManager;
this.auditor = auditor;
this.observer = new JobStateObserver(threadPool, clusterService); this.observer = new JobStateObserver(threadPool, clusterService);
this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings);
clusterService.getClusterSettings()
.addSettingsUpdateConsumer(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, this::setMaxConcurrentJobAllocations);
} }
@Override @Override
protected void doExecute(Request request, ActionListener<PersistentActionResponse> listener) { protected void doExecute(Request request, ActionListener<Response> listener) {
if (licenseState.isMachineLearningAllowed()) { if (licenseState.isMachineLearningAllowed()) {
// If we already know that we can't find an ml node because all ml nodes are running at capacity or PersistentTaskOperationListener finalListener = new PersistentTaskOperationListener() {
// simply because there are no ml nodes in the cluster then we fail quickly here: @Override
ClusterState clusterState = clusterService.state(); public void onResponse(long taskId) {
validate(request, clusterState); waitForJobStarted(request, listener);
Assignment assignment = selectLeastLoadedMlNode(request.getJobId(), clusterState, maxConcurrentJobAllocations, logger); }
if (assignment.getExecutorNode() == null) {
throw new ElasticsearchStatusException("cannot open job [" + request.getJobId() + "], no suitable nodes found, " +
"allocation explanation [{}]", RestStatus.TOO_MANY_REQUESTS, assignment.getExplanation());
}
ActionListener<PersistentActionResponse> finalListener = @Override
ActionListener.wrap(response -> waitForJobStarted(request, response, listener), listener::onFailure); public void onFailure(Exception e) {
super.doExecute(request, finalListener); listener.onFailure(e);
}
};
persistentTasksService.createPersistentActionTask(NAME, request, finalListener);
} else { } else {
listener.onFailure(LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING)); listener.onFailure(LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING));
} }
} }
void waitForJobStarted(Request request, PersistentActionResponse response, ActionListener<PersistentActionResponse> listener) { void waitForJobStarted(Request request, ActionListener<Response> listener) {
observer.waitForState(request.getJobId(), request.timeout, JobState.OPENED, e -> { observer.waitForState(request.getJobId(), request.timeout, JobState.OPENED, e -> {
if (e != null) { if (e != null) {
listener.onFailure(e); listener.onFailure(e);
} else { } else {
listener.onResponse(response); listener.onResponse(new Response(true));
} }
}); });
} }
}
public static class OpenJobPersistentTasksExecutor extends PersistentTasksExecutor<Request> {
private final AutodetectProcessManager autodetectProcessManager;
private final XPackLicenseState licenseState;
private final Auditor auditor;
private final ThreadPool threadPool;
private volatile int maxConcurrentJobAllocations;
public OpenJobPersistentTasksExecutor(Settings settings, ThreadPool threadPool, XPackLicenseState licenseState,
PersistentTasksService persistentTasksService, ClusterService clusterService,
AutodetectProcessManager autodetectProcessManager, Auditor auditor) {
super(settings, NAME, persistentTasksService, ThreadPool.Names.MANAGEMENT);
this.licenseState = licenseState;
this.autodetectProcessManager = autodetectProcessManager;
this.auditor = auditor;
this.threadPool = threadPool;
this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings);
clusterService.getClusterSettings()
.addSettingsUpdateConsumer(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, this::setMaxConcurrentJobAllocations);
}
@Override @Override
public Assignment getAssignment(Request request, ClusterState clusterState) { public Assignment getAssignment(Request request, ClusterState clusterState) {
@ -317,9 +364,20 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
@Override @Override
public void validate(Request request, ClusterState clusterState) { public void validate(Request request, ClusterState clusterState) {
MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE); if (licenseState.isMachineLearningAllowed()) {
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); // If we already know that we can't find an ml node because all ml nodes are running at capacity or
OpenJobAction.validate(request.getJobId(), mlMetadata, tasks, clusterState.nodes()); // simply because there are no ml nodes in the cluster then we fail quickly here:
MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE);
PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
OpenJobAction.validate(request.getJobId(), mlMetadata, tasks, clusterState.nodes());
Assignment assignment = selectLeastLoadedMlNode(request.getJobId(), clusterState, maxConcurrentJobAllocations, logger);
if (assignment.getExecutorNode() == null) {
throw new ElasticsearchStatusException("cannot open job [" + request.getJobId() + "], no suitable nodes found, " +
"allocation explanation [{}]", RestStatus.TOO_MANY_REQUESTS, assignment.getExplanation());
}
} else {
throw LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING);
}
} }
@Override @Override
@ -385,7 +443,7 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
* Fail fast before trying to update the job state on master node if the job doesn't exist or its state * Fail fast before trying to update the job state on master node if the job doesn't exist or its state
* is not what it should be. * is not what it should be.
*/ */
static void validate(String jobId, MlMetadata mlMetadata, @Nullable PersistentTasks tasks, DiscoveryNodes nodes) { static void validate(String jobId, MlMetadata mlMetadata, @Nullable PersistentTasksCustomMetaData tasks, DiscoveryNodes nodes) {
Job job = mlMetadata.getJobs().get(jobId); Job job = mlMetadata.getJobs().get(jobId);
if (job == null) { if (job == null) {
throw ExceptionsHelper.missingJobException(jobId); throw ExceptionsHelper.missingJobException(jobId);
@ -410,7 +468,7 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
} }
if (jobState.isAnyOf(JobState.CLOSED, JobState.FAILED) == false) { if (jobState.isAnyOf(JobState.CLOSED, JobState.FAILED) == false) {
throw new ElasticsearchStatusException("[" + jobId + "] expected state [" + JobState.CLOSED throw new ElasticsearchStatusException("[" + jobId + "] expected state [" + JobState.CLOSED
+ "] or [" + JobState.FAILED + "], but got [" + jobState +"]", RestStatus.CONFLICT); + "] or [" + JobState.FAILED + "], but got [" + jobState + "]", RestStatus.CONFLICT);
} }
} }
@ -427,7 +485,7 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
long maxAvailable = Long.MIN_VALUE; long maxAvailable = Long.MIN_VALUE;
List<String> reasons = new LinkedList<>(); List<String> reasons = new LinkedList<>();
DiscoveryNode minLoadedNode = null; DiscoveryNode minLoadedNode = null;
PersistentTasks persistentTasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData persistentTasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
for (DiscoveryNode node : clusterState.getNodes()) { for (DiscoveryNode node : clusterState.getNodes()) {
Map<String, String> nodeAttributes = node.getAttributes(); Map<String, String> nodeAttributes = node.getAttributes();
String maxNumberOfOpenJobsStr = nodeAttributes.get(AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE.getKey()); String maxNumberOfOpenJobsStr = nodeAttributes.get(AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE.getKey());
@ -450,7 +508,7 @@ public class OpenJobAction extends Action<OpenJobAction.Request, PersistentActio
return jobTaskState == null || // executor node didn't have the chance to set job status to OPENING return jobTaskState == null || // executor node didn't have the chance to set job status to OPENING
jobTaskState == JobState.OPENING || // executor node is busy starting the cpp process jobTaskState == JobState.OPENING || // executor node is busy starting the cpp process
task.isCurrentStatus() == false; // previous executor node failed and task.isCurrentStatus() == false; // previous executor node failed and
// current executor node didn't have the chance to set job status to OPENING // current executor node didn't have the chance to set job status to OPENING
}).size(); }).size();
} else { } else {
numberOfAssignedJobs = 0; numberOfAssignedJobs = 0;

133
plugin/src/main/java/org/elasticsearch/xpack/ml/action/StartDatafeedAction.java
View File

@ -14,6 +14,8 @@ import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.ValidateActions;
import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@ -55,21 +57,20 @@ import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.ml.utils.DatafeedStateObserver; import org.elasticsearch.xpack.ml.utils.DatafeedStateObserver;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.NodePersistentTask; import org.elasticsearch.xpack.persistent.NodePersistentTask;
import org.elasticsearch.xpack.persistent.PersistentActionRegistry; import org.elasticsearch.xpack.persistent.PersistentTaskRequest;
import org.elasticsearch.xpack.persistent.PersistentActionRequest; import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.elasticsearch.xpack.persistent.PersistentActionResponse; import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentActionService; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksExecutor;
import org.elasticsearch.xpack.persistent.TransportPersistentAction;
import java.io.IOException; import java.io.IOException;
import java.util.Objects; import java.util.Objects;
import java.util.function.LongSupplier; import java.util.function.LongSupplier;
public class StartDatafeedAction public class StartDatafeedAction
extends Action<StartDatafeedAction.Request, PersistentActionResponse, StartDatafeedAction.RequestBuilder> { extends Action<StartDatafeedAction.Request, StartDatafeedAction.Response, StartDatafeedAction.RequestBuilder> {
public static final ParseField START_TIME = new ParseField("start"); public static final ParseField START_TIME = new ParseField("start");
public static final ParseField END_TIME = new ParseField("end"); public static final ParseField END_TIME = new ParseField("end");
@ -88,11 +89,11 @@ public class StartDatafeedAction
} }
@Override @Override
public PersistentActionResponse newResponse() { public Response newResponse() {
return new PersistentActionResponse(); return new Response();
} }
public static class Request extends PersistentActionRequest implements ToXContent { public static class Request extends PersistentTaskRequest implements ToXContent {
public static ObjectParser<Request, Void> PARSER = new ObjectParser<>(NAME, Request::new); public static ObjectParser<Request, Void> PARSER = new ObjectParser<>(NAME, Request::new);
@ -250,7 +251,41 @@ public class StartDatafeedAction
} }
} }
static class RequestBuilder extends ActionRequestBuilder<Request, PersistentActionResponse, RequestBuilder> { public static class Response extends AcknowledgedResponse {
public Response() {
super();
}
public Response(boolean acknowledged) {
super(acknowledged);
}
@Override
public void readFrom(StreamInput in) throws IOException {
readAcknowledged(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
writeAcknowledged(out);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AcknowledgedResponse that = (AcknowledgedResponse) o;
return isAcknowledged() == that.isAcknowledged();
}
@Override
public int hashCode() {
return Objects.hash(isAcknowledged());
}
}
static class RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder> {
RequestBuilder(ElasticsearchClient client, StartDatafeedAction action) { RequestBuilder(ElasticsearchClient client, StartDatafeedAction action) {
super(client, action, new Request()); super(client, action, new Request());
@ -303,48 +338,68 @@ public class StartDatafeedAction
} }
} }
public static class TransportAction extends TransportPersistentAction<Request> { public static class TransportAction extends HandledTransportAction<Request, Response> {
private final DatafeedStateObserver observer; private final DatafeedStateObserver observer;
private final DatafeedJobRunner datafeedJobRunner;
private final XPackLicenseState licenseState; private final XPackLicenseState licenseState;
private final Auditor auditor; private final PersistentTasksService persistentTasksService;
@Inject @Inject
public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, XPackLicenseState licenseState, public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, XPackLicenseState licenseState,
PersistentActionService persistentActionService, PersistentActionRegistry persistentActionRegistry, PersistentTasksService persistentTasksService, ActionFilters actionFilters,
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService) {
ClusterService clusterService, DatafeedJobRunner datafeedJobRunner, Auditor auditor) { super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new);
super(settings, NAME, false, threadPool, transportService, persistentActionService, persistentActionRegistry,
actionFilters, indexNameExpressionResolver, Request::new, ThreadPool.Names.MANAGEMENT);
this.licenseState = licenseState; this.licenseState = licenseState;
this.datafeedJobRunner = datafeedJobRunner; this.persistentTasksService = persistentTasksService;
this.auditor = auditor;
this.observer = new DatafeedStateObserver(threadPool, clusterService); this.observer = new DatafeedStateObserver(threadPool, clusterService);
} }
@Override @Override
protected void doExecute(Request request, ActionListener<PersistentActionResponse> listener) { protected void doExecute(Request request, ActionListener<Response> listener) {
if (licenseState.isMachineLearningAllowed()) { if (licenseState.isMachineLearningAllowed()) {
ActionListener<PersistentActionResponse> finalListener = ActionListener PersistentTaskOperationListener finalListener = new PersistentTaskOperationListener() {
.wrap(response -> waitForDatafeedStarted(request, response, listener), listener::onFailure); @Override
super.doExecute(request, finalListener); public void onResponse(long taskId) {
waitForDatafeedStarted(request, listener);
}
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
};
persistentTasksService.createPersistentActionTask(NAME, request, finalListener);
} else { } else {
listener.onFailure(LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING)); listener.onFailure(LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING));
} }
} }
void waitForDatafeedStarted(Request request, void waitForDatafeedStarted(Request request, ActionListener<Response> listener) {
PersistentActionResponse response,
ActionListener<PersistentActionResponse> listener) {
observer.waitForState(request.getDatafeedId(), request.timeout, DatafeedState.STARTED, e -> { observer.waitForState(request.getDatafeedId(), request.timeout, DatafeedState.STARTED, e -> {
if (e != null) { if (e != null) {
listener.onFailure(e); listener.onFailure(e);
} else { } else {
listener.onResponse(response); listener.onResponse(new Response(true));
} }
}); });
} }
}
public static class StartDatafeedPersistentTasksExecutor extends PersistentTasksExecutor<Request> {
private final DatafeedJobRunner datafeedJobRunner;
private final XPackLicenseState licenseState;
private final Auditor auditor;
private final ThreadPool threadPool;
public StartDatafeedPersistentTasksExecutor(Settings settings, ThreadPool threadPool, XPackLicenseState licenseState,
PersistentTasksService persistentTasksService, DatafeedJobRunner datafeedJobRunner,
Auditor auditor) {
super(settings, NAME, persistentTasksService, ThreadPool.Names.MANAGEMENT);
this.licenseState = licenseState;
this.datafeedJobRunner = datafeedJobRunner;
this.auditor = auditor;
this.threadPool = threadPool;
}
@Override @Override
public Assignment getAssignment(Request request, ClusterState clusterState) { public Assignment getAssignment(Request request, ClusterState clusterState) {
@ -355,10 +410,14 @@ public class StartDatafeedAction
@Override @Override
public void validate(Request request, ClusterState clusterState) { public void validate(Request request, ClusterState clusterState) {
MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE); if (licenseState.isMachineLearningAllowed()) {
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE);
DiscoveryNodes nodes = clusterState.getNodes(); PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
StartDatafeedAction.validate(request.getDatafeedId(), mlMetadata, tasks, nodes); DiscoveryNodes nodes = clusterState.getNodes();
StartDatafeedAction.validate(request.getDatafeedId(), mlMetadata, tasks, nodes);
} else {
throw LicenseUtils.newComplianceException(XPackPlugin.MACHINE_LEARNING);
}
} }
@Override @Override
@ -403,7 +462,7 @@ public class StartDatafeedAction
} }
static void validate(String datafeedId, MlMetadata mlMetadata, PersistentTasks tasks, DiscoveryNodes nodes) { static void validate(String datafeedId, MlMetadata mlMetadata, PersistentTasksCustomMetaData tasks, DiscoveryNodes nodes) {
DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId); DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId);
if (datafeed == null) { if (datafeed == null) {
throw ExceptionsHelper.missingDatafeedException(datafeedId); throw ExceptionsHelper.missingDatafeedException(datafeedId);
@ -441,7 +500,7 @@ public class StartDatafeedAction
public static Assignment selectNode(Logger logger, String datafeedId, ClusterState clusterState) { public static Assignment selectNode(Logger logger, String datafeedId, ClusterState clusterState) {
MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE); MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE);
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId); DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId);
DiscoveryNodes nodes = clusterState.getNodes(); DiscoveryNodes nodes = clusterState.getNodes();

8
plugin/src/main/java/org/elasticsearch/xpack/ml/action/StopDatafeedAction.java
View File

@ -44,8 +44,8 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.messages.Messages; import org.elasticsearch.xpack.ml.job.messages.Messages;
import org.elasticsearch.xpack.ml.utils.DatafeedStateObserver; import org.elasticsearch.xpack.ml.utils.DatafeedStateObserver;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
@ -217,7 +217,7 @@ public class StopDatafeedAction
ClusterState state = clusterService.state(); ClusterState state = clusterService.state();
MetaData metaData = state.metaData(); MetaData metaData = state.metaData();
MlMetadata mlMetadata = metaData.custom(MlMetadata.TYPE); MlMetadata mlMetadata = metaData.custom(MlMetadata.TYPE);
PersistentTasks tasks = metaData.custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = metaData.custom(PersistentTasksCustomMetaData.TYPE);
String nodeId = validateAndReturnNodeId(request.getDatafeedId(), mlMetadata, tasks); String nodeId = validateAndReturnNodeId(request.getDatafeedId(), mlMetadata, tasks);
request.setNodes(nodeId); request.setNodes(nodeId);
ActionListener<Response> finalListener = ActionListener<Response> finalListener =
@ -262,7 +262,7 @@ public class StopDatafeedAction
} }
} }
static String validateAndReturnNodeId(String datafeedId, MlMetadata mlMetadata, PersistentTasks tasks) { static String validateAndReturnNodeId(String datafeedId, MlMetadata mlMetadata, PersistentTasksCustomMetaData tasks) {
DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId); DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId);
if (datafeed == null) { if (datafeed == null) {
throw new ResourceNotFoundException(Messages.getMessage(Messages.DATAFEED_NOT_FOUND, datafeedId)); throw new ResourceNotFoundException(Messages.getMessage(Messages.DATAFEED_NOT_FOUND, datafeedId));

8
plugin/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java
View File

@ -30,7 +30,7 @@ import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
@ -62,8 +62,8 @@ public abstract class TransportJobTaskAction<OperationTask extends Task, Request
// node running the job task. // node running the job task.
ClusterState state = clusterService.state(); ClusterState state = clusterService.state();
JobManager.getJobOrThrowIfUnknown(state, jobId); JobManager.getJobOrThrowIfUnknown(state, jobId);
PersistentTasks tasks = clusterService.state().getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
PersistentTasks.PersistentTask<?> jobTask = MlMetadata.getJobTask(jobId, tasks); PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlMetadata.getJobTask(jobId, tasks);
if (jobTask == null || jobTask.isAssigned() == false) { if (jobTask == null || jobTask.isAssigned() == false) {
listener.onFailure( new ElasticsearchStatusException("job [" + jobId + "] state is [" + JobState.CLOSED + listener.onFailure( new ElasticsearchStatusException("job [" + jobId + "] state is [" + JobState.CLOSED +
"], but must be [" + JobState.OPENED + "] to perform requested action", RestStatus.CONFLICT)); "], but must be [" + JobState.OPENED + "] to perform requested action", RestStatus.CONFLICT));
@ -76,7 +76,7 @@ public abstract class TransportJobTaskAction<OperationTask extends Task, Request
@Override @Override
protected final void taskOperation(Request request, OperationTask task, ActionListener<Response> listener) { protected final void taskOperation(Request request, OperationTask task, ActionListener<Response> listener) {
ClusterState state = clusterService.state(); ClusterState state = clusterService.state();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
JobState jobState = MlMetadata.getJobState(request.getJobId(), tasks); JobState jobState = MlMetadata.getJobState(request.getJobId(), tasks);
if (jobState == JobState.OPENED) { if (jobState == JobState.OPENED) {
innerTaskOperation(request, task, listener); innerTaskOperation(request, task, listener);

6
plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateDatafeedAction.java
View File

@ -32,7 +32,7 @@ import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.ml.datafeed.DatafeedUpdate; import org.elasticsearch.xpack.ml.datafeed.DatafeedUpdate;
import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.io.IOException; import java.io.IOException;
import java.util.Objects; import java.util.Objects;
@ -161,8 +161,8 @@ public class UpdateDatafeedAction extends Action<UpdateDatafeedAction.Request, P
public ClusterState execute(ClusterState currentState) throws Exception { public ClusterState execute(ClusterState currentState) throws Exception {
DatafeedUpdate update = request.getUpdate(); DatafeedUpdate update = request.getUpdate();
MlMetadata currentMetadata = state.getMetaData().custom(MlMetadata.TYPE); MlMetadata currentMetadata = state.getMetaData().custom(MlMetadata.TYPE);
PersistentTasks persistentTasks = PersistentTasksCustomMetaData persistentTasks =
state.getMetaData().custom(PersistentTasks.TYPE); state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata) MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata)
.updateDatafeed(update, persistentTasks).build(); .updateDatafeed(update, persistentTasks).build();
updatedDatafeed = newMetadata.getDatafeed(update.getId()); updatedDatafeed = newMetadata.getDatafeed(update.getId());

5
plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateJobAction.java
View File

@ -35,7 +35,8 @@ import org.elasticsearch.xpack.ml.job.JobManager;
import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.config.JobUpdate; import org.elasticsearch.xpack.ml.job.config.JobUpdate;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.io.IOException; import java.io.IOException;
import java.util.Objects; import java.util.Objects;
@ -172,7 +173,7 @@ public class UpdateJobAction extends Action<UpdateJobAction.Request, PutJobActio
throw new IllegalArgumentException("Job Id " + Job.ALL + " cannot be for update"); throw new IllegalArgumentException("Job Id " + Job.ALL + " cannot be for update");
} }
PersistentTasks tasks = clusterService.state().getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
boolean jobIsOpen = MlMetadata.getJobState(request.getJobId(), tasks) == JobState.OPENED; boolean jobIsOpen = MlMetadata.getJobState(request.getJobId(), tasks) == JobState.OPENED;
semaphoreByJob.computeIfAbsent(request.getJobId(), id -> new Semaphore(1)).acquire(); semaphoreByJob.computeIfAbsent(request.getJobId(), id -> new Semaphore(1)).acquire();

5
plugin/src/main/java/org/elasticsearch/xpack/ml/client/MachineLearningClient.java
View File

@ -34,7 +34,6 @@ import org.elasticsearch.xpack.ml.action.StopDatafeedAction;
import org.elasticsearch.xpack.ml.action.UpdateDatafeedAction; import org.elasticsearch.xpack.ml.action.UpdateDatafeedAction;
import org.elasticsearch.xpack.ml.action.UpdateJobAction; import org.elasticsearch.xpack.ml.action.UpdateJobAction;
import org.elasticsearch.xpack.ml.action.UpdateModelSnapshotAction; import org.elasticsearch.xpack.ml.action.UpdateModelSnapshotAction;
import org.elasticsearch.xpack.persistent.PersistentActionResponse;
public class MachineLearningClient { public class MachineLearningClient {
@ -109,7 +108,7 @@ public class MachineLearningClient {
client.execute(GetRecordsAction.INSTANCE, request, listener); client.execute(GetRecordsAction.INSTANCE, request, listener);
} }
public void openJob(OpenJobAction.Request request, ActionListener<PersistentActionResponse> listener) { public void openJob(OpenJobAction.Request request, ActionListener<OpenJobAction.Response> listener) {
client.execute(OpenJobAction.INSTANCE, request, listener); client.execute(OpenJobAction.INSTANCE, request, listener);
} }
@ -134,7 +133,7 @@ public class MachineLearningClient {
client.execute(RevertModelSnapshotAction.INSTANCE, request, listener); client.execute(RevertModelSnapshotAction.INSTANCE, request, listener);
} }
public void startDatafeed(StartDatafeedAction.Request request, ActionListener<PersistentActionResponse> listener) { public void startDatafeed(StartDatafeedAction.Request request, ActionListener<StartDatafeedAction.Response> listener) {
client.execute(StartDatafeedAction.INSTANCE, request, listener); client.execute(StartDatafeedAction.INSTANCE, request, listener);
} }

24
plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunner.java
View File

@ -34,9 +34,10 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.job.results.Bucket; import org.elasticsearch.xpack.ml.job.results.Bucket;
import org.elasticsearch.xpack.ml.job.results.Result; import org.elasticsearch.xpack.ml.job.results.Result;
import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.ml.utils.DatafeedStateObserver; import org.elasticsearch.xpack.ml.utils.DatafeedStateObserver;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction;
import java.time.Duration; import java.time.Duration;
import java.util.Collections; import java.util.Collections;
@ -60,17 +61,19 @@ public class DatafeedJobRunner extends AbstractComponent {
private final JobProvider jobProvider; private final JobProvider jobProvider;
private final ThreadPool threadPool; private final ThreadPool threadPool;
private final Supplier<Long> currentTimeSupplier; private final Supplier<Long> currentTimeSupplier;
private final PersistentTasksService persistentTasksService;
private final Auditor auditor; private final Auditor auditor;
private final ConcurrentMap<String, Holder> runningDatafeeds = new ConcurrentHashMap<>(); private final ConcurrentMap<String, Holder> runningDatafeeds = new ConcurrentHashMap<>();
public DatafeedJobRunner(ThreadPool threadPool, Client client, ClusterService clusterService, JobProvider jobProvider, public DatafeedJobRunner(ThreadPool threadPool, Client client, ClusterService clusterService, JobProvider jobProvider,
Supplier<Long> currentTimeSupplier, Auditor auditor) { Supplier<Long> currentTimeSupplier, PersistentTasksService persistentTasksService, Auditor auditor) {
super(Settings.EMPTY); super(Settings.EMPTY);
this.client = Objects.requireNonNull(client); this.client = Objects.requireNonNull(client);
this.clusterService = Objects.requireNonNull(clusterService); this.clusterService = Objects.requireNonNull(clusterService);
this.jobProvider = Objects.requireNonNull(jobProvider); this.jobProvider = Objects.requireNonNull(jobProvider);
this.threadPool = threadPool; this.threadPool = threadPool;
this.currentTimeSupplier = Objects.requireNonNull(currentTimeSupplier); this.currentTimeSupplier = Objects.requireNonNull(currentTimeSupplier);
this.persistentTasksService = persistentTasksService;
this.auditor = auditor; this.auditor = auditor;
} }
@ -268,10 +271,17 @@ public class DatafeedJobRunner extends AbstractComponent {
} }
private void updateDatafeedState(long persistentTaskId, DatafeedState datafeedState, Consumer<Exception> handler) { private void updateDatafeedState(long persistentTaskId, DatafeedState datafeedState, Consumer<Exception> handler) {
UpdatePersistentTaskStatusAction.Request request = new UpdatePersistentTaskStatusAction.Request(persistentTaskId, datafeedState); persistentTasksService.updateStatus(persistentTaskId, datafeedState, new PersistentTaskOperationListener() {
client.execute(UpdatePersistentTaskStatusAction.INSTANCE, request, ActionListener.wrap(r -> { @Override
handler.accept(null); public void onResponse(long taskId) {
}, handler)); handler.accept(null);
}
@Override
public void onFailure(Exception e) {
handler.accept(e);
}
});
} }
private static Duration getFrequencyOrDefault(DatafeedConfig datafeed, Job job) { private static Duration getFrequencyOrDefault(DatafeedConfig datafeed, Job job) {

9
plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
View File

@ -19,7 +19,6 @@ import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.MachineLearningTemplateRegistry;
import org.elasticsearch.xpack.ml.action.DeleteJobAction; import org.elasticsearch.xpack.ml.action.DeleteJobAction;
import org.elasticsearch.xpack.ml.action.PutJobAction; import org.elasticsearch.xpack.ml.action.PutJobAction;
import org.elasticsearch.xpack.ml.action.RevertModelSnapshotAction; import org.elasticsearch.xpack.ml.action.RevertModelSnapshotAction;
@ -34,7 +33,7 @@ import org.elasticsearch.xpack.ml.job.persistence.JobStorageDeletionTask;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
@ -129,7 +128,7 @@ public class JobManager extends AbstractComponent {
} }
public JobState getJobState(String jobId) { public JobState getJobState(String jobId) {
PersistentTasks tasks = clusterService.state().getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
return MlMetadata.getJobState(jobId, tasks); return MlMetadata.getJobState(jobId, tasks);
} }
@ -281,7 +280,7 @@ public class JobManager extends AbstractComponent {
@Override @Override
public ClusterState execute(ClusterState currentState) throws Exception { public ClusterState execute(ClusterState currentState) throws Exception {
MlMetadata.Builder builder = createMlMetadataBuilder(currentState); MlMetadata.Builder builder = createMlMetadataBuilder(currentState);
builder.deleteJob(jobId, currentState.getMetaData().custom(PersistentTasks.TYPE)); builder.deleteJob(jobId, currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE));
return buildNewClusterState(currentState, builder); return buildNewClusterState(currentState, builder);
} }
}); });
@ -307,7 +306,7 @@ public class JobManager extends AbstractComponent {
@Override @Override
public ClusterState execute(ClusterState currentState) throws Exception { public ClusterState execute(ClusterState currentState) throws Exception {
MlMetadata currentMlMetadata = currentState.metaData().custom(MlMetadata.TYPE); MlMetadata currentMlMetadata = currentState.metaData().custom(MlMetadata.TYPE);
PersistentTasks tasks = currentState.metaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = currentState.metaData().custom(PersistentTasksCustomMetaData.TYPE);
MlMetadata.Builder builder = new MlMetadata.Builder(currentMlMetadata); MlMetadata.Builder builder = new MlMetadata.Builder(currentMlMetadata);
builder.markJobAsDeleted(jobId, tasks); builder.markJobAsDeleted(jobId, tasks);
return buildNewClusterState(currentState, builder); return buildNewClusterState(currentState, builder);

75
plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java
View File

@ -7,7 +7,6 @@ package org.elasticsearch.xpack.ml.job.process.autodetect;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.client.Client; import org.elasticsearch.client.Client;
import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.collect.Tuple;
@ -41,7 +40,8 @@ import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer;
import org.elasticsearch.xpack.ml.job.process.normalizer.ScoresUpdater; import org.elasticsearch.xpack.ml.job.process.normalizer.ScoresUpdater;
import org.elasticsearch.xpack.ml.job.process.normalizer.ShortCircuitingRenormalizer; import org.elasticsearch.xpack.ml.job.process.normalizer.ShortCircuitingRenormalizer;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction; import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
@ -74,6 +74,7 @@ public class AutodetectProcessManager extends AbstractComponent {
private final JobResultsPersister jobResultsPersister; private final JobResultsPersister jobResultsPersister;
private final JobDataCountsPersister jobDataCountsPersister; private final JobDataCountsPersister jobDataCountsPersister;
private final PersistentTasksService persistentTasksService;
private final ConcurrentMap<String, AutodetectCommunicator> autoDetectCommunicatorByJob; private final ConcurrentMap<String, AutodetectCommunicator> autoDetectCommunicatorByJob;
@ -82,7 +83,8 @@ public class AutodetectProcessManager extends AbstractComponent {
public AutodetectProcessManager(Settings settings, Client client, ThreadPool threadPool, JobManager jobManager, public AutodetectProcessManager(Settings settings, Client client, ThreadPool threadPool, JobManager jobManager,
JobProvider jobProvider, JobResultsPersister jobResultsPersister, JobProvider jobProvider, JobResultsPersister jobResultsPersister,
JobDataCountsPersister jobDataCountsPersister, JobDataCountsPersister jobDataCountsPersister,
AutodetectProcessFactory autodetectProcessFactory, NormalizerFactory normalizerFactory) { AutodetectProcessFactory autodetectProcessFactory, NormalizerFactory normalizerFactory,
PersistentTasksService persistentTasksService) {
super(settings); super(settings);
this.client = client; this.client = client;
this.threadPool = threadPool; this.threadPool = threadPool;
@ -94,6 +96,7 @@ public class AutodetectProcessManager extends AbstractComponent {
this.jobResultsPersister = jobResultsPersister; this.jobResultsPersister = jobResultsPersister;
this.jobDataCountsPersister = jobDataCountsPersister; this.jobDataCountsPersister = jobDataCountsPersister;
this.persistentTasksService = persistentTasksService;
this.autoDetectCommunicatorByJob = new ConcurrentHashMap<>(); this.autoDetectCommunicatorByJob = new ConcurrentHashMap<>();
} }
@ -113,18 +116,18 @@ public class AutodetectProcessManager extends AbstractComponent {
* Passes data to the native process. * Passes data to the native process.
* This is a blocking call that won't return until all the data has been * This is a blocking call that won't return until all the data has been
* written to the process. * written to the process.
* * <p>
* An ElasticsearchStatusException will be thrown is any of these error conditions occur: * An ElasticsearchStatusException will be thrown is any of these error conditions occur:
* <ol> * <ol>
* <li>If a configured field is missing from the CSV header</li> * <li>If a configured field is missing from the CSV header</li>
* <li>If JSON data is malformed and we cannot recover parsing</li> * <li>If JSON data is malformed and we cannot recover parsing</li>
* <li>If a high proportion of the records the timestamp field that cannot be parsed</li> * <li>If a high proportion of the records the timestamp field that cannot be parsed</li>
* <li>If a high proportion of the records chronologically out of order</li> * <li>If a high proportion of the records chronologically out of order</li>
* </ol> * </ol>
* *
* @param jobId the jobId * @param jobId the jobId
* @param input Data input stream * @param input Data input stream
* @param params Data processing parameters * @param params Data processing parameters
* @return Count of records, fields, bytes, etc written * @return Count of records, fields, bytes, etc written
*/ */
public DataCounts processData(String jobId, InputStream input, DataLoadParams params) { public DataCounts processData(String jobId, InputStream input, DataLoadParams params) {
@ -152,9 +155,9 @@ public class AutodetectProcessManager extends AbstractComponent {
* opportunity to process all data previously sent to it with none left * opportunity to process all data previously sent to it with none left
* sitting in buffers. * sitting in buffers.
* *
* @param jobId The job to flush * @param jobId The job to flush
* @param params Parameters about whether interim results calculation * @param params Parameters about whether interim results calculation
* should occur and for which period of time * should occur and for which period of time
*/ */
public void flushJob(String jobId, InterimResultsParams params) { public void flushJob(String jobId, InterimResultsParams params) {
logger.debug("Flushing job {}", jobId); logger.debug("Flushing job {}", jobId);
@ -278,9 +281,10 @@ public class AutodetectProcessManager extends AbstractComponent {
/** /**
* Stop the running job and mark it as finished.<br> * Stop the running job and mark it as finished.<br>
* @param jobId The job to stop *
* @param restart Whether the job should be restarted by persistent tasks * @param jobId The job to stop
* @param reason The reason for closing the job * @param restart Whether the job should be restarted by persistent tasks
* @param reason The reason for closing the job
*/ */
public void closeJob(String jobId, boolean restart, String reason) { public void closeJob(String jobId, boolean restart, String reason) {
logger.debug("Attempting to close job [{}], because [{}]", jobId, reason); logger.debug("Attempting to close job [{}], because [{}]", jobId, reason);
@ -321,34 +325,39 @@ public class AutodetectProcessManager extends AbstractComponent {
} }
private void setJobState(long taskId, String jobId, JobState state) { private void setJobState(long taskId, String jobId, JobState state) {
UpdatePersistentTaskStatusAction.Request request = new UpdatePersistentTaskStatusAction.Request(taskId, state); persistentTasksService.updateStatus(taskId, state, new PersistentTaskOperationListener() {
client.execute(UpdatePersistentTaskStatusAction.INSTANCE, request, new ActionListener<UpdatePersistentTaskStatusAction.Response>() {
@Override @Override
public void onResponse(UpdatePersistentTaskStatusAction.Response response) { public void onResponse(long taskId) {
if (response.isAcknowledged()) { logger.info("Successfully set job state to [{}] for job [{}]", state, jobId);
logger.info("Successfully set job state to [{}] for job [{}]", state, jobId);
} else {
logger.info("Changing job state to [{}] for job [{}] wasn't acked", state, jobId);
}
} }
@Override @Override
public void onFailure(Exception e) { public void onFailure(Exception e) {
logger.error("Could not set job state to [" + state + "] for job [" + jobId +"]", e); logger.error("Could not set job state to [" + state + "] for job [" + jobId + "]", e);
} }
}); });
} }
public void setJobState(long taskId, JobState state, CheckedConsumer<Exception, IOException> handler) { public void setJobState(long taskId, JobState state, CheckedConsumer<Exception, IOException> handler) {
UpdatePersistentTaskStatusAction.Request request = new UpdatePersistentTaskStatusAction.Request(taskId, state); persistentTasksService.updateStatus(taskId, state, new PersistentTaskOperationListener() {
client.execute(UpdatePersistentTaskStatusAction.INSTANCE, request, @Override
ActionListener.wrap(r -> handler.accept(null), e -> { public void onResponse(long taskId) {
try { try {
handler.accept(e); handler.accept(null);
} catch (IOException e1) { } catch (IOException e1) {
logger.warn("Error while delagating exception [" + e.getMessage() + "]", e1); logger.warn("Error while delegating response", e1);
}
} }
}));
@Override
public void onFailure(Exception e) {
try {
handler.accept(e);
} catch (IOException e1) {
logger.warn("Error while delegating exception [" + e.getMessage() + "]", e1);
}
}
});
} }
public Optional<Tuple<DataCounts, ModelSizeStats>> getStatistics(String jobId) { public Optional<Tuple<DataCounts, ModelSizeStats>> getStatistics(String jobId) {

7
plugin/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java
View File

@ -20,7 +20,6 @@ import org.elasticsearch.rest.action.RestBuilderListener;
import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.ml.action.StartDatafeedAction;
import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.persistent.PersistentActionResponse;
import java.io.IOException; import java.io.IOException;
@ -55,12 +54,12 @@ public class RestStartDatafeedAction extends BaseRestHandler {
} }
return channel -> { return channel -> {
client.execute(StartDatafeedAction.INSTANCE, jobDatafeedRequest, client.execute(StartDatafeedAction.INSTANCE, jobDatafeedRequest,
new RestBuilderListener<PersistentActionResponse>(channel) { new RestBuilderListener<StartDatafeedAction.Response>(channel) {
@Override @Override
public RestResponse buildResponse(PersistentActionResponse r, XContentBuilder builder) throws Exception { public RestResponse buildResponse(StartDatafeedAction.Response r, XContentBuilder builder) throws Exception {
builder.startObject(); builder.startObject();
builder.field("started", true); builder.field("started", r.isAcknowledged());
builder.endObject(); builder.endObject();
return new BytesRestResponse(RestStatus.OK, builder); return new BytesRestResponse(RestStatus.OK, builder);
} }

8
plugin/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java
View File

@ -19,7 +19,6 @@ import org.elasticsearch.rest.action.RestBuilderListener;
import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.action.OpenJobAction; import org.elasticsearch.xpack.ml.action.OpenJobAction;
import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.persistent.PersistentActionResponse;
import java.io.IOException; import java.io.IOException;
@ -45,12 +44,11 @@ public class RestOpenJobAction extends BaseRestHandler {
} }
} }
return channel -> { return channel -> {
client.execute(OpenJobAction.INSTANCE, request, new RestBuilderListener<PersistentActionResponse>(channel) { client.execute(OpenJobAction.INSTANCE, request, new RestBuilderListener<OpenJobAction.Response>(channel) {
@Override @Override
public RestResponse buildResponse(PersistentActionResponse r, XContentBuilder builder) throws Exception { public RestResponse buildResponse(OpenJobAction.Response r, XContentBuilder builder) throws Exception {
builder.startObject(); builder.startObject();
builder.field("opened", true); builder.field("opened", r.isAcknowledged());
builder.endObject(); builder.endObject();
return new BytesRestResponse(RestStatus.OK, builder); return new BytesRestResponse(RestStatus.OK, builder);
} }

4
plugin/src/main/java/org/elasticsearch/xpack/ml/utils/DatafeedStateObserver.java
View File

@ -14,7 +14,7 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.function.Predicate; import java.util.function.Predicate;
@ -35,7 +35,7 @@ public class DatafeedStateObserver {
ClusterStateObserver observer = ClusterStateObserver observer =
new ClusterStateObserver(clusterService, LOGGER, threadPool.getThreadContext()); new ClusterStateObserver(clusterService, LOGGER, threadPool.getThreadContext());
Predicate<ClusterState> predicate = (newState) -> { Predicate<ClusterState> predicate = (newState) -> {
PersistentTasks tasks = newState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeedId, tasks); DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeedId, tasks);
return datafeedState == expectedState; return datafeedState == expectedState;
}; };

6
plugin/src/main/java/org/elasticsearch/xpack/ml/utils/JobStateObserver.java
View File

@ -16,7 +16,7 @@ import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.function.Predicate; import java.util.function.Predicate;
@ -66,7 +66,7 @@ public class JobStateObserver {
handler.accept(null); handler.accept(null);
} }
} else { } else {
PersistentTasks tasks = state.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
JobState actual = MlMetadata.getJobState(jobId, tasks); JobState actual = MlMetadata.getJobState(jobId, tasks);
Exception e = new IllegalArgumentException("Timeout expired while waiting for job state [" + actual + Exception e = new IllegalArgumentException("Timeout expired while waiting for job state [" + actual +
"] to change to [" + expectedState + "]"); "] to change to [" + expectedState + "]");
@ -90,7 +90,7 @@ public class JobStateObserver {
@Override @Override
public boolean test(ClusterState newState) { public boolean test(ClusterState newState) {
PersistentTasks tasks = newState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
JobState jobState = MlMetadata.getJobState(jobId, tasks); JobState jobState = MlMetadata.getJobState(jobId, tasks);
if (jobState == JobState.FAILED) { if (jobState == JobState.FAILED) {
failed = true; failed = true;

38
plugin/src/main/java/org/elasticsearch/xpack/persistent/CompletionPersistentTaskAction.java
View File

@ -105,6 +105,36 @@ public class CompletionPersistentTaskAction extends Action<CompletionPersistentT
} }
public static class Response extends AcknowledgedResponse { public static class Response extends AcknowledgedResponse {
public Response() {
super();
}
public Response(boolean acknowledged) {
super(acknowledged);
}
@Override
public void readFrom(StreamInput in) throws IOException {
readAcknowledged(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
writeAcknowledged(out);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AcknowledgedResponse that = (AcknowledgedResponse) o;
return isAcknowledged() == that.isAcknowledged();
}
@Override
public int hashCode() {
return Objects.hash(isAcknowledged());
}
} }
@ -118,16 +148,16 @@ public class CompletionPersistentTaskAction extends Action<CompletionPersistentT
public static class TransportAction extends TransportMasterNodeAction<Request, Response> { public static class TransportAction extends TransportMasterNodeAction<Request, Response> {
private final PersistentTaskClusterService persistentTaskClusterService; private final PersistentTasksClusterService persistentTasksClusterService;
@Inject @Inject
public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters, ThreadPool threadPool, ActionFilters actionFilters,
PersistentTaskClusterService persistentTaskClusterService, PersistentTasksClusterService persistentTasksClusterService,
IndexNameExpressionResolver indexNameExpressionResolver) { IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, CompletionPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, super(settings, CompletionPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, Request::new); indexNameExpressionResolver, Request::new);
this.persistentTaskClusterService = persistentTaskClusterService; this.persistentTasksClusterService = persistentTasksClusterService;
} }
@Override @Override
@ -148,7 +178,7 @@ public class CompletionPersistentTaskAction extends Action<CompletionPersistentT
@Override @Override
protected final void masterOperation(final Request request, ClusterState state, final ActionListener<Response> listener) { protected final void masterOperation(final Request request, ClusterState state, final ActionListener<Response> listener) {
persistentTaskClusterService.completeOrRestartPersistentTask(request.taskId, request.exception, new ActionListener<Empty>() { persistentTasksClusterService.completeOrRestartPersistentTask(request.taskId, request.exception, new ActionListener<Empty>() {
@Override @Override
public void onResponse(Empty empty) { public void onResponse(Empty empty) {
listener.onResponse(newResponse()); listener.onResponse(newResponse());

50
plugin/src/main/java/org/elasticsearch/xpack/persistent/CreatePersistentTaskAction.java
View File

@ -34,7 +34,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
* This action can be used to add the record for the persistent action to the cluster state. * This action can be used to add the record for the persistent action to the cluster state.
*/ */
public class CreatePersistentTaskAction extends Action<CreatePersistentTaskAction.Request, public class CreatePersistentTaskAction extends Action<CreatePersistentTaskAction.Request,
PersistentActionResponse, PersistentTaskResponse,
CreatePersistentTaskAction.RequestBuilder> { CreatePersistentTaskAction.RequestBuilder> {
public static final CreatePersistentTaskAction INSTANCE = new CreatePersistentTaskAction(); public static final CreatePersistentTaskAction INSTANCE = new CreatePersistentTaskAction();
@ -50,15 +50,15 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
} }
@Override @Override
public PersistentActionResponse newResponse() { public PersistentTaskResponse newResponse() {
return new PersistentActionResponse(); return new PersistentTaskResponse();
} }
public static class Request extends MasterNodeRequest<Request> { public static class Request extends MasterNodeRequest<Request> {
private String action; private String action;
private PersistentActionRequest request; private PersistentTaskRequest request;
private boolean stopped; private boolean stopped;
@ -68,7 +68,7 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
} }
public Request(String action, PersistentActionRequest request) { public Request(String action, PersistentTaskRequest request) {
this.action = action; this.action = action;
this.request = request; this.request = request;
this.stopped = false; this.stopped = false;
@ -79,7 +79,7 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
public void readFrom(StreamInput in) throws IOException { public void readFrom(StreamInput in) throws IOException {
super.readFrom(in); super.readFrom(in);
action = in.readString(); action = in.readString();
request = in.readNamedWriteable(PersistentActionRequest.class); request = in.readNamedWriteable(PersistentTaskRequest.class);
stopped = in.readBoolean(); stopped = in.readBoolean();
removeOnCompletion = in.readBoolean(); removeOnCompletion = in.readBoolean();
} }
@ -129,11 +129,11 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
this.action = action; this.action = action;
} }
public PersistentActionRequest getRequest() { public PersistentTaskRequest getRequest() {
return request; return request;
} }
public void setRequest(PersistentActionRequest request) { public void setRequest(PersistentTaskRequest request) {
this.request = request; this.request = request;
} }
@ -155,7 +155,7 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
} }
public static class RequestBuilder extends MasterNodeOperationRequestBuilder<CreatePersistentTaskAction.Request, public static class RequestBuilder extends MasterNodeOperationRequestBuilder<CreatePersistentTaskAction.Request,
PersistentActionResponse, CreatePersistentTaskAction.RequestBuilder> { PersistentTaskResponse, CreatePersistentTaskAction.RequestBuilder> {
protected RequestBuilder(ElasticsearchClient client, CreatePersistentTaskAction action) { protected RequestBuilder(ElasticsearchClient client, CreatePersistentTaskAction action) {
super(client, action, new Request()); super(client, action, new Request());
@ -166,8 +166,8 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
return this; return this;
} }
public RequestBuilder setRequest(PersistentActionRequest persistentActionRequest) { public RequestBuilder setRequest(PersistentTaskRequest persistentTaskRequest) {
request.setRequest(persistentActionRequest); request.setRequest(persistentTaskRequest);
return this; return this;
} }
@ -188,23 +188,23 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
} }
} }
public static class TransportAction extends TransportMasterNodeAction<Request, PersistentActionResponse> { public static class TransportAction extends TransportMasterNodeAction<Request, PersistentTaskResponse> {
private final PersistentTaskClusterService persistentTaskClusterService; private final PersistentTasksClusterService persistentTasksClusterService;
@Inject @Inject
public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters, ThreadPool threadPool, ActionFilters actionFilters,
PersistentTaskClusterService persistentTaskClusterService, PersistentTasksClusterService persistentTasksClusterService,
PersistentActionRegistry persistentActionRegistry, PersistentTasksExecutorRegistry persistentTasksExecutorRegistry,
PersistentActionService persistentActionService, PersistentTasksService persistentTasksService,
IndexNameExpressionResolver indexNameExpressionResolver) { IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, CreatePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, super(settings, CreatePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, Request::new); indexNameExpressionResolver, Request::new);
this.persistentTaskClusterService = persistentTaskClusterService; this.persistentTasksClusterService = persistentTasksClusterService;
PersistentActionExecutor executor = new PersistentActionExecutor(threadPool); NodePersistentTasksExecutor executor = new NodePersistentTasksExecutor(threadPool);
clusterService.addListener(new PersistentActionCoordinator(settings, persistentActionService, persistentActionRegistry, clusterService.addListener(new PersistentTasksNodeService(settings, persistentTasksService, persistentTasksExecutorRegistry,
transportService.getTaskManager(), executor)); transportService.getTaskManager(), threadPool, executor));
} }
@Override @Override
@ -213,8 +213,8 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
} }
@Override @Override
protected PersistentActionResponse newResponse() { protected PersistentTaskResponse newResponse() {
return new PersistentActionResponse(); return new PersistentTaskResponse();
} }
@Override @Override
@ -225,12 +225,12 @@ public class CreatePersistentTaskAction extends Action<CreatePersistentTaskActio
@Override @Override
protected final void masterOperation(final Request request, ClusterState state, protected final void masterOperation(final Request request, ClusterState state,
final ActionListener<PersistentActionResponse> listener) { final ActionListener<PersistentTaskResponse> listener) {
persistentTaskClusterService.createPersistentTask(request.action, request.request, request.stopped, request.removeOnCompletion, persistentTasksClusterService.createPersistentTask(request.action, request.request, request.stopped, request.removeOnCompletion,
new ActionListener<Long>() { new ActionListener<Long>() {
@Override @Override
public void onResponse(Long newTaskId) { public void onResponse(Long newTaskId) {
listener.onResponse(new PersistentActionResponse(newTaskId)); listener.onResponse(new PersistentTaskResponse(newTaskId));
} }
@Override @Override

18
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionExecutor.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/NodePersistentTasksExecutor.java
View File

@ -11,20 +11,20 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportResponse.Empty;
/** /**
* This component is responsible for execution of persistent actions. * This component is responsible for execution of persistent tasks.
*/ */
public class PersistentActionExecutor { public class NodePersistentTasksExecutor {
private final ThreadPool threadPool; private final ThreadPool threadPool;
public PersistentActionExecutor(ThreadPool threadPool) { public NodePersistentTasksExecutor(ThreadPool threadPool) {
this.threadPool = threadPool; this.threadPool = threadPool;
} }
public <Request extends PersistentActionRequest> void executeAction(Request request, public <Request extends PersistentTaskRequest> void executeTask(Request request,
NodePersistentTask task, NodePersistentTask task,
PersistentActionRegistry.PersistentActionHolder<Request> holder, PersistentTasksExecutor<Request> action,
ActionListener<Empty> listener) { ActionListener<Empty> listener) {
threadPool.executor(holder.getExecutor()).execute(new AbstractRunnable() { threadPool.executor(action.getExecutor()).execute(new AbstractRunnable() {
@Override @Override
public void onFailure(Exception e) { public void onFailure(Exception e) {
listener.onFailure(e); listener.onFailure(e);
@ -34,7 +34,7 @@ public class PersistentActionExecutor {
@Override @Override
protected void doRun() throws Exception { protected void doRun() throws Exception {
try { try {
holder.getPersistentAction().nodeOperation(task, request, listener); action.nodeOperation(task, request, listener);
} catch (Exception ex) { } catch (Exception ex) {
listener.onFailure(ex); listener.onFailure(ex);
} }

91
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRegistry.java
View File

@ -1,91 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import java.util.Collections;
import java.util.Map;
/**
* Components that registers all persistent actions
*/
public class PersistentActionRegistry extends AbstractComponent {
private volatile Map<String, PersistentActionHolder<?>> actions = Collections.emptyMap();
private final Object actionHandlerMutex = new Object();
public PersistentActionRegistry(Settings settings) {
super(settings);
}
public <Request extends PersistentActionRequest> void registerPersistentAction(String action,
TransportPersistentAction<Request> persistentAction) {
registerPersistentAction(new PersistentActionHolder<>(action, persistentAction, persistentAction.getExecutor()));
}
private <Request extends PersistentActionRequest> void registerPersistentAction(
PersistentActionHolder<Request> reg) {
synchronized (actionHandlerMutex) {
PersistentActionHolder<?> replaced = actions.get(reg.getAction());
actions = MapBuilder.newMapBuilder(actions).put(reg.getAction(), reg).immutableMap();
if (replaced != null) {
logger.warn("registered two handlers for persistent action {}, handlers: {}, {}", reg.getAction(), reg, replaced);
}
}
}
public void removeHandler(String action) {
synchronized (actionHandlerMutex) {
actions = MapBuilder.newMapBuilder(actions).remove(action).immutableMap();
}
}
@SuppressWarnings("unchecked")
public <Request extends PersistentActionRequest> PersistentActionHolder<Request> getPersistentActionHolderSafe(String action) {
PersistentActionHolder<Request> holder = (PersistentActionHolder<Request>) actions.get(action);
if (holder == null) {
throw new IllegalStateException("Unknown persistent action [" + action + "]");
}
return holder;
}
public <Request extends PersistentActionRequest>
TransportPersistentAction<Request> getPersistentActionSafe(String action) {
PersistentActionHolder<Request> holder = getPersistentActionHolderSafe(action);
return holder.getPersistentAction();
}
public static final class PersistentActionHolder<Request extends PersistentActionRequest> {
private final String action;
private final TransportPersistentAction<Request> persistentAction;
private final String executor;
public PersistentActionHolder(String action, TransportPersistentAction<Request> persistentAction, String executor) {
this.action = action;
this.persistentAction = persistentAction;
this.executor = executor;
}
public String getAction() {
return action;
}
public TransportPersistentAction<Request> getPersistentAction() {
return persistentAction;
}
public String getExecutor() {
return executor;
}
}
}

90
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionService.java
View File

@ -1,90 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.persistent;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskId;
import org.elasticsearch.threadpool.ThreadPool;
/**
* Service responsible for executing restartable actions that can survive disappearance of a coordinating and executor nodes.
*/
public class PersistentActionService extends AbstractComponent {
private final Client client;
private final ThreadPool threadPool;
private final ClusterService clusterService;
public PersistentActionService(Settings settings, ThreadPool threadPool, ClusterService clusterService, Client client) {
super(settings);
this.client = client;
this.threadPool = threadPool;
this.clusterService = clusterService;
}
public <Request extends PersistentActionRequest> void sendRequest(String action, Request request,
ActionListener<PersistentActionResponse> listener) {
CreatePersistentTaskAction.Request startRequest = new CreatePersistentTaskAction.Request(action, request);
try {
client.execute(CreatePersistentTaskAction.INSTANCE, startRequest, listener);
} catch (Exception e) {
listener.onFailure(e);
}
}
public void sendCompletionNotification(long taskId, Exception failure,
ActionListener<CompletionPersistentTaskAction.Response> listener) {
CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure);
// Need to fork otherwise: java.lang.AssertionError: should not be called by a cluster state applier.
// reason [the applied cluster state is not yet available])
try {
threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() {
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
@Override
protected void doRun() throws Exception {
client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, listener);
}
});
} catch (Exception e) {
listener.onFailure(e);
}
}
public void sendCancellation(long taskId, ActionListener<CancelTasksResponse> listener) {
DiscoveryNode localNode = clusterService.localNode();
CancelTasksRequest cancelTasksRequest = new CancelTasksRequest();
cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId));
cancelTasksRequest.setReason("persistent action was removed");
try {
client.admin().cluster().cancelTasks(cancelTasksRequest, listener);
} catch (Exception e) {
listener.onFailure(e);
}
}
public void updateStatus(long taskId, Task.Status status, ActionListener<UpdatePersistentTaskStatusAction.Response> listener) {
UpdatePersistentTaskStatusAction.Request updateStatusRequest = new UpdatePersistentTaskStatusAction.Request(taskId, status);
try {
client.execute(UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, listener);
} catch (Exception e) {
listener.onFailure(e);
}
}
}

4
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRequest.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskRequest.java
View File

@ -12,9 +12,9 @@ import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskId;
/** /**
* Base class for a request for a persistent action * Base class for a request for a persistent task
*/ */
public abstract class PersistentActionRequest extends ActionRequest implements NamedWriteable, ToXContent { public abstract class PersistentTaskRequest extends ActionRequest implements NamedWriteable, ToXContent {
@Override @Override
public Task createTask(long id, String type, String action, TaskId parentTaskId) { public Task createTask(long id, String type, String action, TaskId parentTaskId) {
return new NodePersistentTask(id, type, action, getDescription(), parentTaskId); return new NodePersistentTask(id, type, action, getDescription(), parentTaskId);

10
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionResponse.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskResponse.java
View File

@ -13,16 +13,16 @@ import java.io.IOException;
import java.util.Objects; import java.util.Objects;
/** /**
* Response upon a successful start or an persistent action * Response upon a successful start or an persistent task
*/ */
public class PersistentActionResponse extends ActionResponse { public class PersistentTaskResponse extends ActionResponse {
private long taskId; private long taskId;
public PersistentActionResponse() { public PersistentTaskResponse() {
super(); super();
} }
public PersistentActionResponse(long taskId) { public PersistentTaskResponse(long taskId) {
this.taskId = taskId; this.taskId = taskId;
} }
@ -46,7 +46,7 @@ public class PersistentActionResponse extends ActionResponse {
public boolean equals(Object o) { public boolean equals(Object o) {
if (this == o) return true; if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false; if (o == null || getClass() != o.getClass()) return false;
PersistentActionResponse that = (PersistentActionResponse) o; PersistentTaskResponse that = (PersistentTaskResponse) o;
return taskId == that.taskId; return taskId == that.taskId;
} }

64
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterService.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksClusterService.java
View File

@ -19,20 +19,20 @@ import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.Task;
import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.util.Objects; import java.util.Objects;
/** /**
* Component that runs only on the master node and is responsible for assigning running tasks to nodes * Component that runs only on the master node and is responsible for assigning running tasks to nodes
*/ */
public class PersistentTaskClusterService extends AbstractComponent implements ClusterStateListener { public class PersistentTasksClusterService extends AbstractComponent implements ClusterStateListener {
private final ClusterService clusterService; private final ClusterService clusterService;
private final PersistentActionRegistry registry; private final PersistentTasksExecutorRegistry registry;
public PersistentTaskClusterService(Settings settings, PersistentActionRegistry registry, ClusterService clusterService) { public PersistentTasksClusterService(Settings settings, PersistentTasksExecutorRegistry registry, ClusterService clusterService) {
super(settings); super(settings);
this.clusterService = clusterService; this.clusterService = clusterService;
clusterService.addListener(this); clusterService.addListener(this);
@ -47,15 +47,17 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
* @param request request * @param request request
* @param listener the listener that will be called when task is started * @param listener the listener that will be called when task is started
*/ */
public <Request extends PersistentActionRequest> void createPersistentTask(String action, Request request, boolean stopped, public <Request extends PersistentTaskRequest> void createPersistentTask(String action, Request request, boolean stopped,
boolean removeOnCompletion, boolean removeOnCompletion,
ActionListener<Long> listener) { ActionListener<Long> listener) {
clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() {
@Override @Override
public ClusterState execute(ClusterState currentState) throws Exception { public ClusterState execute(ClusterState currentState) throws Exception {
validate(action, clusterService.state(), request);
final Assignment assignment; final Assignment assignment;
if (stopped) { if (stopped) {
assignment = PersistentTasks.FINISHED_TASK_ASSIGNMENT; // the task is stopped no need to assign it anywhere // the task is stopped no need to assign it anywhere
assignment = PersistentTasksCustomMetaData.FINISHED_TASK_ASSIGNMENT;
} else { } else {
assignment = getAssignement(action, currentState, request); assignment = getAssignement(action, currentState, request);
} }
@ -70,7 +72,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
@Override @Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
listener.onResponse( listener.onResponse(
((PersistentTasks) newState.getMetaData().custom(PersistentTasks.TYPE)).getCurrentId()); ((PersistentTasksCustomMetaData) newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)).getCurrentId());
} }
}); });
} }
@ -94,7 +96,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() { clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() {
@Override @Override
public ClusterState execute(ClusterState currentState) throws Exception { public ClusterState execute(ClusterState currentState) throws Exception {
PersistentTasks.Builder tasksInProgress = builder(currentState); PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState);
if (tasksInProgress.hasTask(id)) { if (tasksInProgress.hasTask(id)) {
if (failure != null) { if (failure != null) {
// If the task failed - we need to restart it on another node, otherwise we just remove it // If the task failed - we need to restart it on another node, otherwise we just remove it
@ -132,7 +134,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
clusterService.submitStateUpdateTask("start persistent task", new ClusterStateUpdateTask() { clusterService.submitStateUpdateTask("start persistent task", new ClusterStateUpdateTask() {
@Override @Override
public ClusterState execute(ClusterState currentState) throws Exception { public ClusterState execute(ClusterState currentState) throws Exception {
PersistentTasks.Builder tasksInProgress = builder(currentState); PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState);
if (tasksInProgress.hasTask(id)) { if (tasksInProgress.hasTask(id)) {
return update(currentState, tasksInProgress return update(currentState, tasksInProgress
.assignTask(id, (action, request) -> getAssignement(action, currentState, request))); .assignTask(id, (action, request) -> getAssignement(action, currentState, request)));
@ -163,7 +165,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
clusterService.submitStateUpdateTask("remove persistent task", new ClusterStateUpdateTask() { clusterService.submitStateUpdateTask("remove persistent task", new ClusterStateUpdateTask() {
@Override @Override
public ClusterState execute(ClusterState currentState) throws Exception { public ClusterState execute(ClusterState currentState) throws Exception {
PersistentTasks.Builder tasksInProgress = builder(currentState); PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState);
if (tasksInProgress.hasTask(id)) { if (tasksInProgress.hasTask(id)) {
return update(currentState, tasksInProgress.removeTask(id)); return update(currentState, tasksInProgress.removeTask(id));
} else { } else {
@ -194,7 +196,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() { clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() {
@Override @Override
public ClusterState execute(ClusterState currentState) throws Exception { public ClusterState execute(ClusterState currentState) throws Exception {
PersistentTasks.Builder tasksInProgress = builder(currentState); PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState);
if (tasksInProgress.hasTask(id)) { if (tasksInProgress.hasTask(id)) {
return update(currentState, tasksInProgress.updateTaskStatus(id, status)); return update(currentState, tasksInProgress.updateTaskStatus(id, status));
} else { } else {
@ -214,10 +216,14 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
}); });
} }
private <Request extends PersistentActionRequest> Assignment getAssignement(String action, ClusterState currentState, Request request) { private <Request extends PersistentTaskRequest> Assignment getAssignement(String taskName, ClusterState currentState, Request request) {
TransportPersistentAction<Request> persistentAction = registry.getPersistentActionSafe(action); PersistentTasksExecutor<Request> persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName);
persistentAction.validate(request, currentState); return persistentTasksExecutor.getAssignment(request, currentState);
return persistentAction.getAssignment(request, currentState); }
private <Request extends PersistentTaskRequest> void validate(String taskName, ClusterState currentState, Request request) {
PersistentTasksExecutor<Request> persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName);
persistentTasksExecutor.validate(request, currentState);
} }
@Override @Override
@ -234,12 +240,12 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
} }
interface ExecutorNodeDecider { interface ExecutorNodeDecider {
<Request extends PersistentActionRequest> Assignment getAssignment(String action, ClusterState currentState, Request request); <Request extends PersistentTaskRequest> Assignment getAssignment(String action, ClusterState currentState, Request request);
} }
static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecider decider) { static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecider decider) {
PersistentTasks tasks = event.state().getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
PersistentTasks prevTasks = event.previousState().getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData prevTasks = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
if (tasks != null && (Objects.equals(tasks, prevTasks) == false || if (tasks != null && (Objects.equals(tasks, prevTasks) == false ||
event.nodesChanged() || event.nodesChanged() ||
event.routingTableChanged() || event.routingTableChanged() ||
@ -250,7 +256,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
if (taskInProgress.needsReassignment(event.state().nodes())) { if (taskInProgress.needsReassignment(event.state().nodes())) {
// there is an unassigned task or task with a disappeared node - we need to try assigning it // there is an unassigned task or task with a disappeared node - we need to try assigning it
if (Objects.equals(taskInProgress.getAssignment(), if (Objects.equals(taskInProgress.getAssignment(),
decider.getAssignment(taskInProgress.getAction(), event.state(), taskInProgress.getRequest())) == false) { decider.getAssignment(taskInProgress.getTaskName(), event.state(), taskInProgress.getRequest())) == false) {
// it looks like a assignment for at least one task is possible - let's trigger reassignment // it looks like a assignment for at least one task is possible - let's trigger reassignment
reassignmentRequired = true; reassignmentRequired = true;
break; break;
@ -270,7 +276,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
clusterService.submitStateUpdateTask("reassign persistent tasks", new ClusterStateUpdateTask() { clusterService.submitStateUpdateTask("reassign persistent tasks", new ClusterStateUpdateTask() {
@Override @Override
public ClusterState execute(ClusterState currentState) throws Exception { public ClusterState execute(ClusterState currentState) throws Exception {
return reassignTasks(currentState, logger, PersistentTaskClusterService.this::getAssignement); return reassignTasks(currentState, logger, PersistentTasksClusterService.this::getAssignement);
} }
@Override @Override
@ -286,7 +292,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
} }
static ClusterState reassignTasks(ClusterState currentState, Logger logger, ExecutorNodeDecider decider) { static ClusterState reassignTasks(ClusterState currentState, Logger logger, ExecutorNodeDecider decider) {
PersistentTasks tasks = currentState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
ClusterState clusterState = currentState; ClusterState clusterState = currentState;
DiscoveryNodes nodes = currentState.nodes(); DiscoveryNodes nodes = currentState.nodes();
if (tasks != null) { if (tasks != null) {
@ -295,7 +301,7 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
for (PersistentTask<?> task : tasks.tasks()) { for (PersistentTask<?> task : tasks.tasks()) {
if (task.needsReassignment(nodes)) { if (task.needsReassignment(nodes)) {
// there is an unassigned task - we need to try assigning it // there is an unassigned task - we need to try assigning it
Assignment assignment = decider.getAssignment(task.getAction(), clusterState, task.getRequest()); Assignment assignment = decider.getAssignment(task.getTaskName(), clusterState, task.getRequest());
if (Objects.equals(assignment, task.getAssignment()) == false) { if (Objects.equals(assignment, task.getAssignment()) == false) {
logger.trace("reassigning task {} from node {} to node {}", task.getId(), logger.trace("reassigning task {} from node {} to node {}", task.getId(),
task.getAssignment().getExecutorNode(), assignment.getExecutorNode()); task.getAssignment().getExecutorNode(), assignment.getExecutorNode());
@ -315,14 +321,14 @@ public class PersistentTaskClusterService extends AbstractComponent implements C
return clusterState; return clusterState;
} }
private static PersistentTasks.Builder builder(ClusterState currentState) { private static PersistentTasksCustomMetaData.Builder builder(ClusterState currentState) {
return PersistentTasks.builder(currentState.getMetaData().custom(PersistentTasks.TYPE)); return PersistentTasksCustomMetaData.builder(currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE));
} }
private static ClusterState update(ClusterState currentState, PersistentTasks.Builder tasksInProgress) { private static ClusterState update(ClusterState currentState, PersistentTasksCustomMetaData.Builder tasksInProgress) {
if (tasksInProgress.isChanged()) { if (tasksInProgress.isChanged()) {
return ClusterState.builder(currentState).metaData( return ClusterState.builder(currentState).metaData(
MetaData.builder(currentState.metaData()).putCustom(PersistentTasks.TYPE, tasksInProgress.build()) MetaData.builder(currentState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasksInProgress.build())
).build(); ).build();
} else { } else {
return currentState; return currentState;

107
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasks.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksCustomMetaData.java
View File

@ -43,7 +43,7 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constru
/** /**
* A cluster state record that contains a list of all running persistent tasks * A cluster state record that contains a list of all running persistent tasks
*/ */
public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom> implements MetaData.Custom { public final class PersistentTasksCustomMetaData extends AbstractNamedDiffable<MetaData.Custom> implements MetaData.Custom {
public static final String TYPE = "persistent_tasks"; public static final String TYPE = "persistent_tasks";
private static final String API_CONTEXT = MetaData.XContentContext.API.toString(); private static final String API_CONTEXT = MetaData.XContentContext.API.toString();
@ -53,21 +53,21 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
private final long currentId; private final long currentId;
public PersistentTasks(long currentId, Map<Long, PersistentTask<?>> tasks) { public PersistentTasksCustomMetaData(long currentId, Map<Long, PersistentTask<?>> tasks) {
this.currentId = currentId; this.currentId = currentId;
this.tasks = tasks; this.tasks = tasks;
} }
private static final ObjectParser<Builder, Void> PERSISTENT_TASKS_PARSER = new ObjectParser<>(TYPE, Builder::new); private static final ObjectParser<Builder, Void> PERSISTENT_TASKS_PARSER = new ObjectParser<>(TYPE, Builder::new);
private static final ObjectParser<TaskBuilder<PersistentActionRequest>, Void> PERSISTENT_TASK_PARSER = private static final ObjectParser<TaskBuilder<PersistentTaskRequest>, Void> PERSISTENT_TASK_PARSER =
new ObjectParser<>("tasks", TaskBuilder::new); new ObjectParser<>("tasks", TaskBuilder::new);
public static final ConstructingObjectParser<Assignment, Void> ASSIGNMENT_PARSER = public static final ConstructingObjectParser<Assignment, Void> ASSIGNMENT_PARSER =
new ConstructingObjectParser<>("assignment", objects -> new Assignment((String) objects[0], (String) objects[1])); new ConstructingObjectParser<>("assignment", objects -> new Assignment((String) objects[0], (String) objects[1]));
private static final NamedObjectParser<PersistentActionRequest, Void> REQUEST_PARSER = private static final NamedObjectParser<PersistentTaskRequest, Void> REQUEST_PARSER =
(XContentParser p, Void c, String name) -> p.namedObject(PersistentActionRequest.class, name, null); (XContentParser p, Void c, String name) -> p.namedObject(PersistentTaskRequest.class, name, null);
private static final NamedObjectParser<Status, Void> STATUS_PARSER = private static final NamedObjectParser<Status, Void> STATUS_PARSER =
(XContentParser p, Void c, String name) -> p.namedObject(Status.class, name, null); (XContentParser p, Void c, String name) -> p.namedObject(Status.class, name, null);
@ -82,20 +82,20 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
// Task parser initialization // Task parser initialization
PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setId, new ParseField("id")); PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setId, new ParseField("id"));
PERSISTENT_TASK_PARSER.declareString(TaskBuilder::setAction, new ParseField("action")); PERSISTENT_TASK_PARSER.declareString(TaskBuilder::setTaskName, new ParseField("name"));
PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setAllocationId, new ParseField("allocation_id")); PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setAllocationId, new ParseField("allocation_id"));
PERSISTENT_TASK_PARSER.declareBoolean(TaskBuilder::setRemoveOnCompletion, new ParseField("remove_on_completion")); PERSISTENT_TASK_PARSER.declareBoolean(TaskBuilder::setRemoveOnCompletion, new ParseField("remove_on_completion"));
PERSISTENT_TASK_PARSER.declareBoolean(TaskBuilder::setStopped, new ParseField("stopped")); PERSISTENT_TASK_PARSER.declareBoolean(TaskBuilder::setStopped, new ParseField("stopped"));
PERSISTENT_TASK_PARSER.declareNamedObjects( PERSISTENT_TASK_PARSER.declareNamedObjects(
(TaskBuilder<PersistentActionRequest> taskBuilder, List<PersistentActionRequest> objects) -> { (TaskBuilder<PersistentTaskRequest> taskBuilder, List<PersistentTaskRequest> objects) -> {
if (objects.size() != 1) { if (objects.size() != 1) {
throw new IllegalArgumentException("only one action request per task is allowed"); throw new IllegalArgumentException("only one request per task is allowed");
} }
taskBuilder.setRequest(objects.get(0)); taskBuilder.setRequest(objects.get(0));
}, REQUEST_PARSER, new ParseField("request")); }, REQUEST_PARSER, new ParseField("request"));
PERSISTENT_TASK_PARSER.declareNamedObjects( PERSISTENT_TASK_PARSER.declareNamedObjects(
(TaskBuilder<PersistentActionRequest> taskBuilder, List<Status> objects) -> { (TaskBuilder<PersistentTaskRequest> taskBuilder, List<Status> objects) -> {
if (objects.size() != 1) { if (objects.size() != 1) {
throw new IllegalArgumentException("only one status per task is allowed"); throw new IllegalArgumentException("only one status per task is allowed");
} }
@ -120,16 +120,16 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return this.tasks.get(id); return this.tasks.get(id);
} }
public Collection<PersistentTask<?>> findTasks(String actionName, Predicate<PersistentTask<?>> predicate) { public Collection<PersistentTask<?>> findTasks(String taskName, Predicate<PersistentTask<?>> predicate) {
return this.tasks().stream() return this.tasks().stream()
.filter(p -> actionName.equals(p.getAction())) .filter(p -> taskName.equals(p.getTaskName()))
.filter(predicate) .filter(predicate)
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
public boolean tasksExist(String actionName, Predicate<PersistentTask<?>> predicate) { public boolean tasksExist(String taskName, Predicate<PersistentTask<?>> predicate) {
return this.tasks().stream() return this.tasks().stream()
.filter(p -> actionName.equals(p.getAction())) .filter(p -> taskName.equals(p.getTaskName()))
.anyMatch(predicate); .anyMatch(predicate);
} }
@ -137,7 +137,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
public boolean equals(Object o) { public boolean equals(Object o) {
if (this == o) return true; if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false; if (o == null || getClass() != o.getClass()) return false;
PersistentTasks that = (PersistentTasks) o; PersistentTasksCustomMetaData that = (PersistentTasksCustomMetaData) o;
return currentId == that.currentId && return currentId == that.currentId &&
Objects.equals(tasks, that.tasks); Objects.equals(tasks, that.tasks);
} }
@ -152,8 +152,9 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return Strings.toString(this); return Strings.toString(this);
} }
public long getNumberOfTasksOnNode(String nodeId, String action) { public long getNumberOfTasksOnNode(String nodeId, String taskName) {
return tasks.values().stream().filter(task -> action.equals(task.action) && nodeId.equals(task.assignment.executorNode)).count(); return tasks.values().stream().filter(
task -> taskName.equals(task.taskName) && nodeId.equals(task.assignment.executorNode)).count();
} }
@Override @Override
@ -166,7 +167,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return ALL_CONTEXTS; return ALL_CONTEXTS;
} }
public static PersistentTasks fromXContent(XContentParser parser) throws IOException { public static PersistentTasksCustomMetaData fromXContent(XContentParser parser) throws IOException {
return PERSISTENT_TASKS_PARSER.parse(parser, null).build(); return PERSISTENT_TASKS_PARSER.parse(parser, null).build();
} }
@ -221,10 +222,10 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
/** /**
* A record that represents a single running persistent task * A record that represents a single running persistent task
*/ */
public static class PersistentTask<Request extends PersistentActionRequest> implements Writeable, ToXContent { public static class PersistentTask<Request extends PersistentTaskRequest> implements Writeable, ToXContent {
private final long id; private final long id;
private final long allocationId; private final long allocationId;
private final String action; private final String taskName;
private final Request request; private final Request request;
private final boolean stopped; private final boolean stopped;
private final boolean removeOnCompletion; private final boolean removeOnCompletion;
@ -235,25 +236,25 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
private final Long allocationIdOnLastStatusUpdate; private final Long allocationIdOnLastStatusUpdate;
public PersistentTask(long id, String action, Request request, boolean stopped, boolean removeOnCompletion, Assignment assignment) { public PersistentTask(long id, String taskName, Request request, boolean stopped, boolean removeOnCompletion, Assignment assignment) {
this(id, 0L, action, request, stopped, removeOnCompletion, null, assignment, null); this(id, 0L, taskName, request, stopped, removeOnCompletion, null, assignment, null);
} }
public PersistentTask(PersistentTask<Request> task, boolean stopped, Assignment assignment) { public PersistentTask(PersistentTask<Request> task, boolean stopped, Assignment assignment) {
this(task.id, task.allocationId + 1L, task.action, task.request, stopped, task.removeOnCompletion, task.status, this(task.id, task.allocationId + 1L, task.taskName, task.request, stopped, task.removeOnCompletion, task.status,
assignment, task.allocationId); assignment, task.allocationId);
} }
public PersistentTask(PersistentTask<Request> task, Status status) { public PersistentTask(PersistentTask<Request> task, Status status) {
this(task.id, task.allocationId, task.action, task.request, task.stopped, task.removeOnCompletion, status, this(task.id, task.allocationId, task.taskName, task.request, task.stopped, task.removeOnCompletion, status,
task.assignment, task.allocationId); task.assignment, task.allocationId);
} }
private PersistentTask(long id, long allocationId, String action, Request request, boolean stopped, boolean removeOnCompletion, private PersistentTask(long id, long allocationId, String taskName, Request request, boolean stopped, boolean removeOnCompletion,
Status status, Assignment assignment, Long allocationIdOnLastStatusUpdate) { Status status, Assignment assignment, Long allocationIdOnLastStatusUpdate) {
this.id = id; this.id = id;
this.allocationId = allocationId; this.allocationId = allocationId;
this.action = action; this.taskName = taskName;
this.request = request; this.request = request;
this.status = status; this.status = status;
this.stopped = stopped; this.stopped = stopped;
@ -268,8 +269,8 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
private PersistentTask(StreamInput in) throws IOException { private PersistentTask(StreamInput in) throws IOException {
id = in.readLong(); id = in.readLong();
allocationId = in.readLong(); allocationId = in.readLong();
action = in.readString(); taskName = in.readString();
request = (Request) in.readNamedWriteable(PersistentActionRequest.class); request = (Request) in.readNamedWriteable(PersistentTaskRequest.class);
stopped = in.readBoolean(); stopped = in.readBoolean();
removeOnCompletion = in.readBoolean(); removeOnCompletion = in.readBoolean();
status = in.readOptionalNamedWriteable(Task.Status.class); status = in.readOptionalNamedWriteable(Task.Status.class);
@ -281,7 +282,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
public void writeTo(StreamOutput out) throws IOException { public void writeTo(StreamOutput out) throws IOException {
out.writeLong(id); out.writeLong(id);
out.writeLong(allocationId); out.writeLong(allocationId);
out.writeString(action); out.writeString(taskName);
out.writeNamedWriteable(request); out.writeNamedWriteable(request);
out.writeBoolean(stopped); out.writeBoolean(stopped);
out.writeBoolean(removeOnCompletion); out.writeBoolean(removeOnCompletion);
@ -298,7 +299,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
PersistentTask<?> that = (PersistentTask<?>) o; PersistentTask<?> that = (PersistentTask<?>) o;
return id == that.id && return id == that.id &&
allocationId == that.allocationId && allocationId == that.allocationId &&
Objects.equals(action, that.action) && Objects.equals(taskName, that.taskName) &&
Objects.equals(request, that.request) && Objects.equals(request, that.request) &&
stopped == that.stopped && stopped == that.stopped &&
removeOnCompletion == that.removeOnCompletion && removeOnCompletion == that.removeOnCompletion &&
@ -309,7 +310,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(id, allocationId, action, request, stopped, removeOnCompletion, status, assignment, return Objects.hash(id, allocationId, taskName, request, stopped, removeOnCompletion, status, assignment,
allocationIdOnLastStatusUpdate); allocationIdOnLastStatusUpdate);
} }
@ -326,8 +327,8 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return allocationId; return allocationId;
} }
public String getAction() { public String getTaskName() {
return action; return taskName;
} }
public Request getRequest() { public Request getRequest() {
@ -380,7 +381,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
builder.startObject(); builder.startObject();
{ {
builder.field("id", id); builder.field("id", id);
builder.field("action", action); builder.field("name", taskName);
builder.startObject("request"); builder.startObject("request");
{ {
builder.field(request.getWriteableName(), request, params); builder.field(request.getWriteableName(), request, params);
@ -420,10 +421,10 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
} }
} }
private static class TaskBuilder<Request extends PersistentActionRequest> { private static class TaskBuilder<Request extends PersistentTaskRequest> {
private long id; private long id;
private long allocationId; private long allocationId;
private String action; private String taskName;
private Request request; private Request request;
private boolean stopped = true; private boolean stopped = true;
private boolean removeOnCompletion; private boolean removeOnCompletion;
@ -441,8 +442,8 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return this; return this;
} }
public TaskBuilder<Request> setAction(String action) { public TaskBuilder<Request> setTaskName(String taskName) {
this.action = action; this.taskName = taskName;
return this; return this;
} }
@ -478,7 +479,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
} }
public PersistentTask<Request> build() { public PersistentTask<Request> build() {
return new PersistentTask<>(id, allocationId, action, request, stopped, removeOnCompletion, status, return new PersistentTask<>(id, allocationId, taskName, request, stopped, removeOnCompletion, status,
assignment, allocationIdOnLastStatusUpdate); assignment, allocationIdOnLastStatusUpdate);
} }
} }
@ -488,7 +489,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return TYPE; return TYPE;
} }
public PersistentTasks(StreamInput in) throws IOException { public PersistentTasksCustomMetaData(StreamInput in) throws IOException {
currentId = in.readLong(); currentId = in.readLong();
tasks = in.readMap(StreamInput::readLong, PersistentTask::new); tasks = in.readMap(StreamInput::readLong, PersistentTask::new);
} }
@ -525,7 +526,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return new Builder(); return new Builder();
} }
public static Builder builder(PersistentTasks tasks) { public static Builder builder(PersistentTasksCustomMetaData tasks) {
return new Builder(tasks); return new Builder(tasks);
} }
@ -537,7 +538,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
public Builder() { public Builder() {
} }
public Builder(PersistentTasks tasksInProgress) { public Builder(PersistentTasksCustomMetaData tasksInProgress) {
if (tasksInProgress != null) { if (tasksInProgress != null) {
tasks.putAll(tasksInProgress.tasks); tasks.putAll(tasksInProgress.tasks);
currentId = tasksInProgress.currentId; currentId = tasksInProgress.currentId;
@ -551,7 +552,7 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return this; return this;
} }
private <Request extends PersistentActionRequest> Builder setTasks(List<TaskBuilder<Request>> tasks) { private <Request extends PersistentTaskRequest> Builder setTasks(List<TaskBuilder<Request>> tasks) {
for (TaskBuilder builder : tasks) { for (TaskBuilder builder : tasks) {
PersistentTask<?> task = builder.build(); PersistentTask<?> task = builder.build();
this.tasks.put(task.getId(), task); this.tasks.put(task.getId(), task);
@ -564,11 +565,11 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
* <p> * <p>
* After the task is added its id can be found by calling {{@link #getCurrentId()}} method. * After the task is added its id can be found by calling {{@link #getCurrentId()}} method.
*/ */
public <Request extends PersistentActionRequest> Builder addTask(String action, Request request, boolean stopped, public <Request extends PersistentTaskRequest> Builder addTask(String taskName, Request request, boolean stopped,
boolean removeOnCompletion, Assignment assignment) { boolean removeOnCompletion, Assignment assignment) {
changed = true; changed = true;
currentId++; currentId++;
tasks.put(currentId, new PersistentTask<>(currentId, action, request, stopped, removeOnCompletion, assignment)); tasks.put(currentId, new PersistentTask<>(currentId, taskName, request, stopped, removeOnCompletion, assignment));
return this; return this;
} }
@ -591,11 +592,11 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
* {@link #reassignTask(long, BiFunction)} instead * {@link #reassignTask(long, BiFunction)} instead
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public <Request extends PersistentActionRequest> Builder assignTask(long taskId, public <Request extends PersistentTaskRequest> Builder assignTask(long taskId,
BiFunction<String, Request, Assignment> executorNodeFunc) { BiFunction<String, Request, Assignment> executorNodeFunc) {
PersistentTask<Request> taskInProgress = (PersistentTask<Request>) tasks.get(taskId); PersistentTask<Request> taskInProgress = (PersistentTask<Request>) tasks.get(taskId);
if (taskInProgress != null && taskInProgress.assignment.isAssigned() == false) { // only assign unassigned tasks if (taskInProgress != null && taskInProgress.assignment.isAssigned() == false) { // only assign unassigned tasks
Assignment assignment = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); Assignment assignment = executorNodeFunc.apply(taskInProgress.taskName, taskInProgress.request);
if (assignment.isAssigned() || taskInProgress.isStopped()) { if (assignment.isAssigned() || taskInProgress.isStopped()) {
changed = true; changed = true;
tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment)); tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment));
@ -608,12 +609,12 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
* Reassigns the task to another node if the task exist * Reassigns the task to another node if the task exist
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public <Request extends PersistentActionRequest> Builder reassignTask(long taskId, public <Request extends PersistentTaskRequest> Builder reassignTask(long taskId,
BiFunction<String, Request, Assignment> executorNodeFunc) { BiFunction<String, Request, Assignment> executorNodeFunc) {
PersistentTask<Request> taskInProgress = (PersistentTask<Request>) tasks.get(taskId); PersistentTask<Request> taskInProgress = (PersistentTask<Request>) tasks.get(taskId);
if (taskInProgress != null) { if (taskInProgress != null) {
changed = true; changed = true;
Assignment assignment = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); Assignment assignment = executorNodeFunc.apply(taskInProgress.taskName, taskInProgress.request);
tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment)); tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment));
} }
return this; return this;
@ -680,8 +681,8 @@ public final class PersistentTasks extends AbstractNamedDiffable<MetaData.Custom
return changed; return changed;
} }
public PersistentTasks build() { public PersistentTasksCustomMetaData build() {
return new PersistentTasks(currentId, Collections.unmodifiableMap(tasks)); return new PersistentTasksCustomMetaData(currentId, Collections.unmodifiableMap(tasks));
} }
} }
} }

54
plugin/src/main/java/org/elasticsearch/xpack/persistent/TransportPersistentAction.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutor.java
View File

@ -6,41 +6,37 @@
package org.elasticsearch.xpack.persistent; package org.elasticsearch.xpack.persistent;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.Task;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import java.util.function.Predicate; import java.util.function.Predicate;
import java.util.function.Supplier;
/** /**
* An action that can survive restart of requesting or executing node. * An executor of tasks that can survive restart of requesting or executing node.
* These actions are using cluster state rather than only transport service to send requests and responses. * These tasks are using cluster state rather than only transport service to send requests and responses.
*/ */
public abstract class TransportPersistentAction<Request extends PersistentActionRequest> public abstract class PersistentTasksExecutor<Request extends PersistentTaskRequest> extends AbstractComponent {
extends HandledTransportAction<Request, PersistentActionResponse> {
private final String executor; private final String executor;
private final PersistentActionService persistentActionService; private final String taskName;
private final PersistentTasksService persistentTasksService;
protected TransportPersistentAction(Settings settings, String actionName, boolean canTripCircuitBreaker, ThreadPool threadPool, protected PersistentTasksExecutor(Settings settings, String taskName, PersistentTasksService persistentTasksService,
TransportService transportService, PersistentActionService persistentActionService, String executor) {
PersistentActionRegistry persistentActionRegistry, super(settings);
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, this.taskName = taskName;
Supplier<Request> requestSupplier, String executor) {
super(settings, actionName, canTripCircuitBreaker, threadPool, transportService, actionFilters, indexNameExpressionResolver,
requestSupplier);
this.executor = executor; this.executor = executor;
this.persistentActionService = persistentActionService; this.persistentTasksService = persistentTasksService;
persistentActionRegistry.registerPersistentAction(actionName, this); }
public String getTaskName() {
return taskName;
} }
public static final Assignment NO_NODE_FOUND = new Assignment(null, "no appropriate nodes found for the assignment"); public static final Assignment NO_NODE_FOUND = new Assignment(null, "no appropriate nodes found for the assignment");
@ -59,21 +55,20 @@ public abstract class TransportPersistentAction<Request extends PersistentAction
} }
} }
/** /**
* Finds the least loaded node that satisfies the selector criteria * Finds the least loaded node that satisfies the selector criteria
*/ */
protected DiscoveryNode selectLeastLoadedNode(ClusterState clusterState, Predicate<DiscoveryNode> selector) { protected DiscoveryNode selectLeastLoadedNode(ClusterState clusterState, Predicate<DiscoveryNode> selector) {
long minLoad = Long.MAX_VALUE; long minLoad = Long.MAX_VALUE;
DiscoveryNode minLoadedNode = null; DiscoveryNode minLoadedNode = null;
PersistentTasks persistentTasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData persistentTasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
for (DiscoveryNode node : clusterState.getNodes()) { for (DiscoveryNode node : clusterState.getNodes()) {
if (selector.test(node)) { if (selector.test(node)) {
if (persistentTasks == null) { if (persistentTasks == null) {
// We don't have any task running yet, pick the first available node // We don't have any task running yet, pick the first available node
return node; return node;
} }
long numberOfTasks = persistentTasks.getNumberOfTasksOnNode(node.getId(), actionName); long numberOfTasks = persistentTasks.getNumberOfTasksOnNode(node.getId(), taskName);
if (minLoad > numberOfTasks) { if (minLoad > numberOfTasks) {
minLoad = numberOfTasks; minLoad = numberOfTasks;
minLoadedNode = node; minLoadedNode = node;
@ -92,11 +87,6 @@ public abstract class TransportPersistentAction<Request extends PersistentAction
} }
@Override
protected void doExecute(Request request, ActionListener<PersistentActionResponse> listener) {
persistentActionService.sendRequest(actionName, request, listener);
}
/** /**
* Updates the persistent task status in the cluster state. * Updates the persistent task status in the cluster state.
* <p> * <p>
@ -104,10 +94,10 @@ public abstract class TransportPersistentAction<Request extends PersistentAction
* task allocator about the state of the currently running tasks. * task allocator about the state of the currently running tasks.
*/ */
protected void updatePersistentTaskStatus(NodePersistentTask task, Task.Status status, ActionListener<Empty> listener) { protected void updatePersistentTaskStatus(NodePersistentTask task, Task.Status status, ActionListener<Empty> listener) {
persistentActionService.updateStatus(task.getPersistentTaskId(), status, persistentTasksService.updateStatus(task.getPersistentTaskId(), status,
new ActionListener<UpdatePersistentTaskStatusAction.Response>() { new PersistentTaskOperationListener() {
@Override @Override
public void onResponse(UpdatePersistentTaskStatusAction.Response response) { public void onResponse(long taskId) {
listener.onResponse(Empty.INSTANCE); listener.onResponse(Empty.INSTANCE);
} }

41
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorRegistry.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
/**
* Components that registers all persistent task executors
*/
public class PersistentTasksExecutorRegistry extends AbstractComponent {
private final Map<String, PersistentTasksExecutor<?>> taskExecutors;
@SuppressWarnings("unchecked")
public PersistentTasksExecutorRegistry(Settings settings, Collection<PersistentTasksExecutor<?>> taskExecutors) {
super(settings);
Map<String, PersistentTasksExecutor<?>> map = new HashMap<>();
for (PersistentTasksExecutor<?> executor : taskExecutors) {
map.put(executor.getTaskName(), executor);
}
this.taskExecutors = Collections.unmodifiableMap(map);
}
@SuppressWarnings("unchecked")
public <Request extends PersistentTaskRequest> PersistentTasksExecutor<Request> getPersistentTaskExecutorSafe(String taskName) {
PersistentTasksExecutor<Request> executor = (PersistentTasksExecutor<Request>) taskExecutors.get(taskName);
if (executor == null) {
throw new IllegalStateException("Unknown persistent executor [" + taskName + "]");
}
return executor;
}
}

78
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinator.java → plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeService.java
View File

@ -9,7 +9,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier; import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
@ -19,12 +18,15 @@ import org.elasticsearch.common.inject.Provider;
import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskCancelledException;
import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.tasks.TaskManager;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
@ -37,33 +39,35 @@ import java.util.concurrent.atomic.AtomicReference;
import static java.util.Objects.requireNonNull; import static java.util.Objects.requireNonNull;
/** /**
* This component is responsible for coordination of execution of persistent actions on individual nodes. It runs on all * This component is responsible for coordination of execution of persistent tasks on individual nodes. It runs on all
* non-transport client nodes in the cluster and monitors cluster state changes to detect started commands. * non-transport client nodes in the cluster and monitors cluster state changes to detect started commands.
*/ */
public class PersistentActionCoordinator extends AbstractComponent implements ClusterStateListener { public class PersistentTasksNodeService extends AbstractComponent implements ClusterStateListener {
private final Map<PersistentTaskId, RunningPersistentTask> runningTasks = new HashMap<>(); private final Map<PersistentTaskId, RunningPersistentTask> runningTasks = new HashMap<>();
private final PersistentActionService persistentActionService; private final PersistentTasksService persistentTasksService;
private final PersistentActionRegistry persistentActionRegistry; private final PersistentTasksExecutorRegistry persistentTasksExecutorRegistry;
private final TaskManager taskManager; private final TaskManager taskManager;
private final PersistentActionExecutor persistentActionExecutor; private final ThreadPool threadPool;
private final NodePersistentTasksExecutor nodePersistentTasksExecutor;
public PersistentActionCoordinator(Settings settings, public PersistentTasksNodeService(Settings settings,
PersistentActionService persistentActionService, PersistentTasksService persistentTasksService,
PersistentActionRegistry persistentActionRegistry, PersistentTasksExecutorRegistry persistentTasksExecutorRegistry,
TaskManager taskManager, TaskManager taskManager, ThreadPool threadPool,
PersistentActionExecutor persistentActionExecutor) { NodePersistentTasksExecutor nodePersistentTasksExecutor) {
super(settings); super(settings);
this.persistentActionService = persistentActionService; this.persistentTasksService = persistentTasksService;
this.persistentActionRegistry = persistentActionRegistry; this.persistentTasksExecutorRegistry = persistentTasksExecutorRegistry;
this.taskManager = taskManager; this.taskManager = taskManager;
this.persistentActionExecutor = persistentActionExecutor; this.threadPool = threadPool;
this.nodePersistentTasksExecutor = nodePersistentTasksExecutor;
} }
@Override @Override
public void clusterChanged(ClusterChangedEvent event) { public void clusterChanged(ClusterChangedEvent event) {
PersistentTasks tasks = event.state().getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
PersistentTasks previousTasks = event.previousState().getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData previousTasks = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) { if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) {
// We have some changes let's check if they are related to our node // We have some changes let's check if they are related to our node
@ -111,10 +115,9 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
} }
private <Request extends PersistentActionRequest> void startTask(PersistentTask<Request> taskInProgress) { private <Request extends PersistentTaskRequest> void startTask(PersistentTask<Request> taskInProgress) {
PersistentActionRegistry.PersistentActionHolder<Request> holder = PersistentTasksExecutor<Request> action = persistentTasksExecutorRegistry.getPersistentTaskExecutorSafe(taskInProgress.getTaskName());
persistentActionRegistry.getPersistentActionHolderSafe(taskInProgress.getAction()); NodePersistentTask task = (NodePersistentTask) taskManager.register("persistent", taskInProgress.getTaskName() + "[c]",
NodePersistentTask task = (NodePersistentTask) taskManager.register("persistent", taskInProgress.getAction() + "[c]",
taskInProgress.getRequest()); taskInProgress.getRequest());
boolean processed = false; boolean processed = false;
try { try {
@ -124,7 +127,7 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
PersistentTaskListener listener = new PersistentTaskListener(runningPersistentTask); PersistentTaskListener listener = new PersistentTaskListener(runningPersistentTask);
try { try {
runningTasks.put(new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()), runningPersistentTask); runningTasks.put(new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()), runningPersistentTask);
persistentActionExecutor.executeAction(taskInProgress.getRequest(), task, holder, listener); nodePersistentTasksExecutor.executeTask(taskInProgress.getRequest(), task, action, listener);
} catch (Exception e) { } catch (Exception e) {
// Submit task failure // Submit task failure
listener.onFailure(e); listener.onFailure(e);
@ -149,9 +152,11 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
RunningPersistentTask task = runningTasks.remove(persistentTaskId); RunningPersistentTask task = runningTasks.remove(persistentTaskId);
if (task != null && task.getTask() != null) { if (task != null && task.getTask() != null) {
if (task.markAsCancelled()) { if (task.markAsCancelled()) {
persistentActionService.sendCancellation(task.getTask().getId(), new ActionListener<CancelTasksResponse>() { persistentTasksService.sendCancellation(task.getTask().getId(), new PersistentTaskOperationListener() {
@Override @Override
public void onResponse(CancelTasksResponse cancelTasksResponse) { public void onResponse(long taskId) {
logger.trace("Persistent task with id {} was cancelled", taskId);
} }
@Override @Override
@ -171,7 +176,24 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
taskManager.unregister(task.getTask()); taskManager.unregister(task.getTask());
} else { } else {
if (task.restartCompletionNotification()) { if (task.restartCompletionNotification()) {
persistentActionService.sendCompletionNotification(task.getId(), task.getFailure(), new PublishedResponseListener(task)); // Need to fork otherwise: java.lang.AssertionError: should not be called by a cluster state applier.
// reason [the applied cluster state is not yet available])
PublishedResponseListener listener = new PublishedResponseListener(task);
try {
threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() {
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
@Override
protected void doRun() throws Exception {
persistentTasksService.sendCompletionNotification(task.getId(), task.getFailure(), listener);
}
});
} catch (Exception e) {
listener.onFailure(e);
}
} else { } else {
logger.warn("attempt to resend notification for task {} in the {} state", task.getId(), task.getState()); logger.warn("attempt to resend notification for task {} in the {} state", task.getId(), task.getState());
} }
@ -184,7 +206,7 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
} else { } else {
logger.trace("sending notification for failed task {}", task.getId()); logger.trace("sending notification for failed task {}", task.getId());
if (task.startNotification(e)) { if (task.startNotification(e)) {
persistentActionService.sendCompletionNotification(task.getId(), e, new PublishedResponseListener(task)); persistentTasksService.sendCompletionNotification(task.getId(), e, new PublishedResponseListener(task));
} else { } else {
logger.warn("attempt to send notification for task {} in the {} state", task.getId(), task.getState()); logger.warn("attempt to send notification for task {} in the {} state", task.getId(), task.getState());
} }
@ -223,7 +245,7 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
} }
} }
private class PublishedResponseListener implements ActionListener<CompletionPersistentTaskAction.Response> { private class PublishedResponseListener implements PersistentTaskOperationListener {
private final RunningPersistentTask task; private final RunningPersistentTask task;
PublishedResponseListener(final RunningPersistentTask task) { PublishedResponseListener(final RunningPersistentTask task) {
@ -232,7 +254,7 @@ public class PersistentActionCoordinator extends AbstractComponent implements Cl
@Override @Override
public void onResponse(CompletionPersistentTaskAction.Response response) { public void onResponse(long taskId) {
logger.trace("notification for task {} was successful", task.getId()); logger.trace("notification for task {} was successful", task.getId());
if (task.markAsNotified() == false) { if (task.markAsNotified() == false) {
logger.warn("attempt to mark task {} in the {} state as NOTIFIED", task.getId(), task.getState()); logger.warn("attempt to mark task {} in the {} state as NOTIFIED", task.getId(), task.getState());

135
plugin/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksService.java Normal file
View File

@ -0,0 +1,135 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.persistent;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskId;
/**
* This service is used by persistent actions to propagate changes in the action state and notify about completion
*/
public class PersistentTasksService extends AbstractComponent {
private final Client client;
private final ClusterService clusterService;
public PersistentTasksService(Settings settings, ClusterService clusterService, Client client) {
super(settings);
this.client = client;
this.clusterService = clusterService;
}
/**
* Creates the specified persistent action and tries to start it immediately, upon completion the task is
* removed from the cluster state
*/
public <Request extends PersistentTaskRequest> void createPersistentActionTask(String action, Request request,
PersistentTaskOperationListener listener) {
createPersistentActionTask(action, request, false, true, listener);
}
/**
* Creates the specified persistent action. The action is started unless the stopped parameter is equal to true.
* If removeOnCompletion parameter is equal to true, the task is removed from the cluster state upon completion.
* Otherwise it will remain there in the stopped state.
*/
public <Request extends PersistentTaskRequest> void createPersistentActionTask(String action, Request request,
boolean stopped,
boolean removeOnCompletion,
PersistentTaskOperationListener listener) {
CreatePersistentTaskAction.Request createPersistentActionRequest = new CreatePersistentTaskAction.Request(action, request);
createPersistentActionRequest.setStopped(stopped);
createPersistentActionRequest.setRemoveOnCompletion(removeOnCompletion);
try {
client.execute(CreatePersistentTaskAction.INSTANCE, createPersistentActionRequest, ActionListener.wrap(
o -> listener.onResponse(o.getTaskId()), listener::onFailure));
} catch (Exception e) {
listener.onFailure(e);
}
}
/**
* Notifies the PersistentTasksClusterService about successful (failure == null) completion of a task or its failure
*
*/
public void sendCompletionNotification(long taskId, Exception failure, PersistentTaskOperationListener listener) {
CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure);
try {
client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, ActionListener.wrap(o -> listener.onResponse(taskId),
listener::onFailure));
} catch (Exception e) {
listener.onFailure(e);
}
}
/**
* Cancels the persistent task.
*/
public void sendCancellation(long taskId, PersistentTaskOperationListener listener) {
DiscoveryNode localNode = clusterService.localNode();
CancelTasksRequest cancelTasksRequest = new CancelTasksRequest();
cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId));
cancelTasksRequest.setReason("persistent action was removed");
try {
client.admin().cluster().cancelTasks(cancelTasksRequest, ActionListener.wrap(o -> listener.onResponse(taskId),
listener::onFailure));
} catch (Exception e) {
listener.onFailure(e);
}
}
/**
* Updates status of the persistent task
*/
public void updateStatus(long taskId, Task.Status status, PersistentTaskOperationListener listener) {
UpdatePersistentTaskStatusAction.Request updateStatusRequest = new UpdatePersistentTaskStatusAction.Request(taskId, status);
try {
client.execute(UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, ActionListener.wrap(
o -> listener.onResponse(taskId), listener::onFailure));
} catch (Exception e) {
listener.onFailure(e);
}
}
/**
* Removes a persistent task
*/
public void removeTask(long taskId, PersistentTaskOperationListener listener) {
RemovePersistentTaskAction.Request removeRequest = new RemovePersistentTaskAction.Request(taskId);
try {
client.execute(RemovePersistentTaskAction.INSTANCE, removeRequest, ActionListener.wrap(o -> listener.onResponse(taskId),
listener::onFailure));
} catch (Exception e) {
listener.onFailure(e);
}
}
/**
* Starts a persistent task
*/
public void startTask(long taskId, PersistentTaskOperationListener listener) {
StartPersistentTaskAction.Request startRequest = new StartPersistentTaskAction.Request(taskId);
try {
client.execute(StartPersistentTaskAction.INSTANCE, startRequest, ActionListener.wrap(o -> listener.onResponse(taskId),
listener::onFailure));
} catch (Exception e) {
listener.onFailure(e);
}
}
public interface PersistentTaskOperationListener {
void onResponse(long taskId);
void onFailure(Exception e);
}
}

8
plugin/src/main/java/org/elasticsearch/xpack/persistent/RemovePersistentTaskAction.java
View File

@ -148,16 +148,16 @@ public class RemovePersistentTaskAction extends Action<RemovePersistentTaskActio
public static class TransportAction extends TransportMasterNodeAction<Request, Response> { public static class TransportAction extends TransportMasterNodeAction<Request, Response> {
private final PersistentTaskClusterService persistentTaskClusterService; private final PersistentTasksClusterService persistentTasksClusterService;
@Inject @Inject
public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters, ThreadPool threadPool, ActionFilters actionFilters,
PersistentTaskClusterService persistentTaskClusterService, PersistentTasksClusterService persistentTasksClusterService,
IndexNameExpressionResolver indexNameExpressionResolver) { IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, RemovePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, super(settings, RemovePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, Request::new); indexNameExpressionResolver, Request::new);
this.persistentTaskClusterService = persistentTaskClusterService; this.persistentTasksClusterService = persistentTasksClusterService;
} }
@Override @Override
@ -178,7 +178,7 @@ public class RemovePersistentTaskAction extends Action<RemovePersistentTaskActio
@Override @Override
protected final void masterOperation(final Request request, ClusterState state, final ActionListener<Response> listener) { protected final void masterOperation(final Request request, ClusterState state, final ActionListener<Response> listener) {
persistentTaskClusterService.removePersistentTask(request.taskId, new ActionListener<Empty>() { persistentTasksClusterService.removePersistentTask(request.taskId, new ActionListener<Empty>() {
@Override @Override
public void onResponse(Empty empty) { public void onResponse(Empty empty) {
listener.onResponse(new Response(true)); listener.onResponse(new Response(true));

10
plugin/src/main/java/org/elasticsearch/xpack/persistent/StartPersistentTaskAction.java
View File

@ -31,7 +31,7 @@ import java.io.IOException;
import java.util.Objects; import java.util.Objects;
/** /**
* This action can be used to start persistent action previously created using {@link CreatePersistentTaskAction} * This action can be used to start a persistent task previously created using {@link CreatePersistentTaskAction}
*/ */
public class StartPersistentTaskAction extends Action<StartPersistentTaskAction.Request, public class StartPersistentTaskAction extends Action<StartPersistentTaskAction.Request,
StartPersistentTaskAction.Response, StartPersistentTaskAction.Response,
@ -151,16 +151,16 @@ public class StartPersistentTaskAction extends Action<StartPersistentTaskAction.
public static class TransportAction extends TransportMasterNodeAction<Request, Response> { public static class TransportAction extends TransportMasterNodeAction<Request, Response> {
private final PersistentTaskClusterService persistentTaskClusterService; private final PersistentTasksClusterService persistentTasksClusterService;
@Inject @Inject
public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters, ThreadPool threadPool, ActionFilters actionFilters,
PersistentTaskClusterService persistentTaskClusterService, PersistentTasksClusterService persistentTasksClusterService,
IndexNameExpressionResolver indexNameExpressionResolver) { IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, StartPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, super(settings, StartPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, Request::new); indexNameExpressionResolver, Request::new);
this.persistentTaskClusterService = persistentTaskClusterService; this.persistentTasksClusterService = persistentTasksClusterService;
} }
@Override @Override
@ -181,7 +181,7 @@ public class StartPersistentTaskAction extends Action<StartPersistentTaskAction.
@Override @Override
protected final void masterOperation(final Request request, ClusterState state, final ActionListener<Response> listener) { protected final void masterOperation(final Request request, ClusterState state, final ActionListener<Response> listener) {
persistentTaskClusterService.startPersistentTask(request.taskId, new ActionListener<Empty>() { persistentTasksClusterService.startPersistentTask(request.taskId, new ActionListener<Empty>() {
@Override @Override
public void onResponse(Empty empty) { public void onResponse(Empty empty) {
listener.onResponse(new Response(true)); listener.onResponse(new Response(true));

8
plugin/src/main/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskStatusAction.java
View File

@ -164,16 +164,16 @@ public class UpdatePersistentTaskStatusAction extends Action<UpdatePersistentTas
public static class TransportAction extends TransportMasterNodeAction<Request, Response> { public static class TransportAction extends TransportMasterNodeAction<Request, Response> {
private final PersistentTaskClusterService persistentTaskClusterService; private final PersistentTasksClusterService persistentTasksClusterService;
@Inject @Inject
public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters, ThreadPool threadPool, ActionFilters actionFilters,
PersistentTaskClusterService persistentTaskClusterService, PersistentTasksClusterService persistentTasksClusterService,
IndexNameExpressionResolver indexNameExpressionResolver) { IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, UpdatePersistentTaskStatusAction.NAME, transportService, clusterService, threadPool, actionFilters, super(settings, UpdatePersistentTaskStatusAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, Request::new); indexNameExpressionResolver, Request::new);
this.persistentTaskClusterService = persistentTaskClusterService; this.persistentTasksClusterService = persistentTasksClusterService;
} }
@Override @Override
@ -194,7 +194,7 @@ public class UpdatePersistentTaskStatusAction extends Action<UpdatePersistentTas
@Override @Override
protected final void masterOperation(final Request request, ClusterState state, final ActionListener<Response> listener) { protected final void masterOperation(final Request request, ClusterState state, final ActionListener<Response> listener) {
persistentTaskClusterService.updatePersistentTaskStatus(request.taskId, request.status, new ActionListener<Empty>() { persistentTasksClusterService.updatePersistentTaskStatus(request.taskId, request.status, new ActionListener<Empty>() {
@Override @Override
public void onResponse(Empty empty) { public void onResponse(Empty empty) {
listener.onResponse(new Response(true)); listener.onResponse(new Response(true));

31
plugin/src/main/java/org/elasticsearch/xpack/persistent/package-info.java
View File

@ -5,31 +5,30 @@
*/ */
/** /**
* The Persistent Actions are actions responsible for executing restartable actions that can survive disappearance of a * The Persistent Tasks Executors are responsible for executing restartable tasks that can survive disappearance of a
* coordinating and executor nodes. * coordinating and executor nodes.
* <p> * <p>
* In order to be resilient to node restarts, the persistent actions are using the cluster state instead of a transport service to send * In order to be resilient to node restarts, the persistent tasks are using the cluster state instead of a transport service to send
* requests and responses. The execution is done in six phases: * requests and responses. The execution is done in six phases:
* <p> * <p>
* 1. The coordinating node sends an ordinary transport request to the master node to start a new persistent action. This action is handled * 1. The coordinating node sends an ordinary transport request to the master node to start a new persistent task. This task is handled
* by the {@link org.elasticsearch.xpack.persistent.PersistentActionService}, which is using * by the {@link org.elasticsearch.xpack.persistent.PersistentTasksService}, which is using
* {@link org.elasticsearch.xpack.persistent.PersistentTaskClusterService} to update cluster state with the record about running persistent * {@link org.elasticsearch.xpack.persistent.PersistentTasksClusterService} to update cluster state with the record about running persistent
* task. * task.
* <p> * <p>
* 2. The master node updates the {@link org.elasticsearch.xpack.persistent.PersistentTasks} in the cluster state to indicate that * 2. The master node updates the {@link org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData} in the cluster state to indicate
* there is a new persistent action * that there is a new persistent task is running in the system.
* running in the system.
* <p> * <p>
* 3. The {@link org.elasticsearch.xpack.persistent.PersistentActionCoordinator} running on every node in the cluster monitors changes in * 3. The {@link org.elasticsearch.xpack.persistent.PersistentTasksNodeService} running on every node in the cluster monitors changes in
* the cluster state and starts execution of all new actions assigned to the node it is running on. * the cluster state and starts execution of all new tasks assigned to the node it is running on.
* <p> * <p>
* 4. If the action fails to start on the node, the {@link org.elasticsearch.xpack.persistent.PersistentActionCoordinator} uses the * 4. If the task fails to start on the node, the {@link org.elasticsearch.xpack.persistent.PersistentTasksNodeService} uses the
* {@link org.elasticsearch.xpack.persistent.PersistentTasks} to notify the * {@link org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData} to notify the
* {@link org.elasticsearch.xpack.persistent.PersistentActionService}, which reassigns the action to another node in the cluster. * {@link org.elasticsearch.xpack.persistent.PersistentTasksService}, which reassigns the action to another node in the cluster.
* <p> * <p>
* 5. If action finishes successfully on the node and calls listener.onResponse(), the corresponding persistent action is removed from the * 5. If a task finishes successfully on the node and calls listener.onResponse(), the corresponding persistent action is removed from the
* cluster state unless . * cluster state unless removeOnCompletion flag for this task is set to false.
* <p> * <p>
* 6. The {@link org.elasticsearch.xpack.persistent.RemovePersistentTaskAction} action can be also used to remove the persistent action. * 6. The {@link org.elasticsearch.xpack.persistent.RemovePersistentTaskAction} action can be also used to remove the persistent task.
*/ */
package org.elasticsearch.xpack.persistent; package org.elasticsearch.xpack.persistent;

55
plugin/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
View File

@ -28,8 +28,7 @@ import org.elasticsearch.xpack.ml.client.MachineLearningClient;
import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentActionResponse; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.junit.Before; import org.junit.Before;
import java.util.Collections; import java.util.Collections;
@ -99,7 +98,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
// test that license restricted apis do not work // test that license restricted apis do not work
try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) { try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) {
client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress()); client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress());
PlainListenableActionFuture<PersistentActionResponse> listener = new PlainListenableActionFuture<>(client.threadPool()); PlainListenableActionFuture<OpenJobAction.Response> listener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), listener); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), listener);
listener.actionGet(); listener.actionGet();
fail("open job action should not be enabled!"); fail("open job action should not be enabled!");
@ -123,9 +122,9 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
// test that license restricted apis do now work // test that license restricted apis do now work
try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) { try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) {
client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress()); client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress());
PlainListenableActionFuture<PersistentActionResponse> listener = new PlainListenableActionFuture<>(client.threadPool()); PlainListenableActionFuture<OpenJobAction.Response> listener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), listener); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), listener);
PersistentActionResponse response = listener.actionGet(); OpenJobAction.Response response = listener.actionGet();
assertNotNull(response); assertNotNull(response);
} }
} }
@ -192,12 +191,12 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet(); PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet();
assertNotNull(putDatafeedResponse); assertNotNull(putDatafeedResponse);
// open job // open job
PlainListenableActionFuture<PersistentActionResponse> openJobListener = new PlainListenableActionFuture<>(client.threadPool()); PlainListenableActionFuture<OpenJobAction.Response> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener);
PersistentActionResponse openJobResponse = openJobListener.actionGet(); OpenJobAction.Response openJobResponse = openJobListener.actionGet();
assertNotNull(openJobResponse); assertNotNull(openJobResponse);
// start datafeed // start datafeed
PlainListenableActionFuture<PersistentActionResponse> listener = new PlainListenableActionFuture<>(client.threadPool()); PlainListenableActionFuture<StartDatafeedAction.Response> listener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener); new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener);
listener.actionGet(); listener.actionGet();
} }
@ -218,7 +217,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
assertEquals(DatafeedState.STOPPED, datafeedState); assertEquals(DatafeedState.STOPPED, datafeedState);
ClusterState state = client().admin().cluster().prepareState().get().getState(); ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(0, tasks.taskMap().size()); assertEquals(0, tasks.taskMap().size());
}); });
@ -228,12 +227,12 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) { try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) {
client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress()); client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress());
// open job // open job
PlainListenableActionFuture<PersistentActionResponse> openJobListener = new PlainListenableActionFuture<>(client.threadPool()); PlainListenableActionFuture<OpenJobAction.Response> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener);
PersistentActionResponse openJobResponse = openJobListener.actionGet(); OpenJobAction.Response openJobResponse = openJobListener.actionGet();
assertNotNull(openJobResponse); assertNotNull(openJobResponse);
// start datafeed // start datafeed
PlainListenableActionFuture<PersistentActionResponse> listener = new PlainListenableActionFuture<>(client.threadPool()); PlainListenableActionFuture<StartDatafeedAction.Response> listener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener); new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener);
listener.actionGet(); listener.actionGet();
} }
@ -246,7 +245,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
assertEquals(DatafeedState.STARTED, datafeedState); assertEquals(DatafeedState.STARTED, datafeedState);
ClusterState state = client().admin().cluster().prepareState().get().getState(); ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(2, tasks.taskMap().size()); assertEquals(2, tasks.taskMap().size());
}); });
@ -266,7 +265,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
assertEquals(DatafeedState.STOPPED, datafeedState); assertEquals(DatafeedState.STOPPED, datafeedState);
ClusterState state = client().admin().cluster().prepareState().get().getState(); ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(0, tasks.taskMap().size()); assertEquals(0, tasks.taskMap().size());
}); });
} }
@ -287,9 +286,9 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
new PutDatafeedAction.Request(createDatafeed("foobar", "foo", Collections.singletonList("foo"))), putDatafeedListener); new PutDatafeedAction.Request(createDatafeed("foobar", "foo", Collections.singletonList("foo"))), putDatafeedListener);
PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet(); PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet();
assertNotNull(putDatafeedResponse); assertNotNull(putDatafeedResponse);
PlainListenableActionFuture<PersistentActionResponse> openJobListener = new PlainListenableActionFuture<>(client.threadPool()); PlainListenableActionFuture<OpenJobAction.Response> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener);
PersistentActionResponse openJobResponse = openJobListener.actionGet(); OpenJobAction.Response openJobResponse = openJobListener.actionGet();
assertNotNull(openJobResponse); assertNotNull(openJobResponse);
} }
@ -303,14 +302,14 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
JobState jobState = getJobStats("foo").getState(); JobState jobState = getJobStats("foo").getState();
assertEquals(JobState.CLOSED, jobState); assertEquals(JobState.CLOSED, jobState);
ClusterState state = client().admin().cluster().prepareState().get().getState(); ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(0, tasks.taskMap().size()); assertEquals(0, tasks.taskMap().size());
}); });
// test that license restricted apis do not work // test that license restricted apis do not work
try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) { try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) {
client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress()); client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress());
PlainListenableActionFuture<PersistentActionResponse> listener = new PlainListenableActionFuture<>(client.threadPool()); PlainListenableActionFuture<StartDatafeedAction.Response> listener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener); new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener);
listener.actionGet(); listener.actionGet();
fail("start datafeed action should not be enabled!"); fail("start datafeed action should not be enabled!");
@ -328,14 +327,14 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) { try (TransportClient client = new TestXPackTransportClient(internalCluster().transportClient().settings())) {
client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress()); client.addTransportAddress(internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress());
// re-open job now that the license is valid again // re-open job now that the license is valid again
PlainListenableActionFuture<PersistentActionResponse> openJobListener = new PlainListenableActionFuture<>(client.threadPool()); PlainListenableActionFuture<OpenJobAction.Response> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener);
PersistentActionResponse openJobResponse = openJobListener.actionGet(); OpenJobAction.Response openJobResponse = openJobListener.actionGet();
assertNotNull(openJobResponse); assertNotNull(openJobResponse);
PlainListenableActionFuture<PersistentActionResponse> listener = new PlainListenableActionFuture<>(client.threadPool()); PlainListenableActionFuture<StartDatafeedAction.Response> listener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener); new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), listener);
PersistentActionResponse response = listener.actionGet(); StartDatafeedAction.Response response = listener.actionGet();
assertNotNull(response); assertNotNull(response);
} }
} }
@ -356,14 +355,14 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
new PutDatafeedAction.Request(createDatafeed("foobar", "foo", Collections.singletonList("foo"))), putDatafeedListener); new PutDatafeedAction.Request(createDatafeed("foobar", "foo", Collections.singletonList("foo"))), putDatafeedListener);
PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet(); PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet();
assertNotNull(putDatafeedResponse); assertNotNull(putDatafeedResponse);
PlainListenableActionFuture<PersistentActionResponse> openJobListener = new PlainListenableActionFuture<>(client.threadPool()); PlainListenableActionFuture<OpenJobAction.Response> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener);
PersistentActionResponse openJobResponse = openJobListener.actionGet(); OpenJobAction.Response openJobResponse = openJobListener.actionGet();
assertNotNull(openJobResponse); assertNotNull(openJobResponse);
PlainListenableActionFuture<PersistentActionResponse> startDatafeedListener = new PlainListenableActionFuture<>( PlainListenableActionFuture<StartDatafeedAction.Response> startDatafeedListener = new PlainListenableActionFuture<>(
client.threadPool()); client.threadPool());
new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), startDatafeedListener); new MachineLearningClient(client).startDatafeed(new StartDatafeedAction.Request("foobar", 0L), startDatafeedListener);
PersistentActionResponse startDatafeedResponse = startDatafeedListener.actionGet(); StartDatafeedAction.Response startDatafeedResponse = startDatafeedListener.actionGet();
assertNotNull(startDatafeedResponse); assertNotNull(startDatafeedResponse);
} }
@ -400,9 +399,9 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
new MachineLearningClient(client).putJob(new PutJobAction.Request(createJob("foo").build()), putJobListener); new MachineLearningClient(client).putJob(new PutJobAction.Request(createJob("foo").build()), putJobListener);
PutJobAction.Response putJobResponse = putJobListener.actionGet(); PutJobAction.Response putJobResponse = putJobListener.actionGet();
assertNotNull(putJobResponse); assertNotNull(putJobResponse);
PlainListenableActionFuture<PersistentActionResponse> openJobListener = new PlainListenableActionFuture<>(client.threadPool()); PlainListenableActionFuture<OpenJobAction.Response> openJobListener = new PlainListenableActionFuture<>(client.threadPool());
new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener); new MachineLearningClient(client).openJob(new OpenJobAction.Request("foo"), openJobListener);
PersistentActionResponse openJobResponse = openJobListener.actionGet(); OpenJobAction.Response openJobResponse = openJobListener.actionGet();
assertNotNull(openJobResponse); assertNotNull(openJobResponse);
} }

24
plugin/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
View File

@ -26,8 +26,8 @@ import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.config.JobTests; import org.elasticsearch.xpack.ml.job.config.JobTests;
import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
@ -36,7 +36,7 @@ import static org.elasticsearch.xpack.ml.action.OpenJobActionTests.createJobTask
import static org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests.createDatafeedConfig; import static org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests.createDatafeedConfig;
import static org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests.createDatafeedJob; import static org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests.createDatafeedJob;
import static org.elasticsearch.xpack.ml.job.config.JobTests.buildJobBuilder; import static org.elasticsearch.xpack.ml.job.config.JobTests.buildJobBuilder;
import static org.elasticsearch.xpack.persistent.PersistentTasks.INITIAL_ASSIGNMENT; import static org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.sameInstance; import static org.hamcrest.Matchers.sameInstance;
@ -133,7 +133,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
assertThat(result.getJobs().get("1"), sameInstance(job1)); assertThat(result.getJobs().get("1"), sameInstance(job1));
assertThat(result.getDatafeeds().get("1"), nullValue()); assertThat(result.getDatafeeds().get("1"), nullValue());
builder.deleteJob("1", new PersistentTasks(0L, Collections.emptyMap())); builder.deleteJob("1", new PersistentTasksCustomMetaData(0L, Collections.emptyMap()));
result = builder.build(); result = builder.build();
assertThat(result.getJobs().get("1"), nullValue()); assertThat(result.getJobs().get("1"), nullValue());
assertThat(result.getDatafeeds().get("1"), nullValue()); assertThat(result.getDatafeeds().get("1"), nullValue());
@ -151,7 +151,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
PersistentTask<OpenJobAction.Request> task = createJobTask(0L, "1", null, JobState.CLOSED); PersistentTask<OpenJobAction.Request> task = createJobTask(0L, "1", null, JobState.CLOSED);
MlMetadata.Builder builder2 = new MlMetadata.Builder(result); MlMetadata.Builder builder2 = new MlMetadata.Builder(result);
ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
() -> builder2.deleteJob("1", new PersistentTasks(0L, Collections.singletonMap(0L, task)))); () -> builder2.deleteJob("1", new PersistentTasksCustomMetaData(0L, Collections.singletonMap(0L, task))));
assertThat(e.status(), equalTo(RestStatus.CONFLICT)); assertThat(e.status(), equalTo(RestStatus.CONFLICT));
} }
@ -163,7 +163,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
builder.putDatafeed(datafeedConfig1); builder.putDatafeed(datafeedConfig1);
ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
() -> builder.deleteJob(job1.getId(), new PersistentTasks(0L, Collections.emptyMap()))); () -> builder.deleteJob(job1.getId(), new PersistentTasksCustomMetaData(0L, Collections.emptyMap())));
assertThat(e.status(), equalTo(RestStatus.CONFLICT)); assertThat(e.status(), equalTo(RestStatus.CONFLICT));
String expectedMsg = "Cannot delete job [" + job1.getId() + "] while datafeed [" + datafeedConfig1.getId() + "] refers to it"; String expectedMsg = "Cannot delete job [" + job1.getId() + "] while datafeed [" + datafeedConfig1.getId() + "] refers to it";
assertThat(e.getMessage(), equalTo(expectedMsg)); assertThat(e.getMessage(), equalTo(expectedMsg));
@ -172,7 +172,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
public void testRemoveJob_failBecauseJobDoesNotExist() { public void testRemoveJob_failBecauseJobDoesNotExist() {
MlMetadata.Builder builder1 = new MlMetadata.Builder(); MlMetadata.Builder builder1 = new MlMetadata.Builder();
expectThrows(ResourceNotFoundException.class, expectThrows(ResourceNotFoundException.class,
() -> builder1.deleteJob("1", new PersistentTasks(0L, Collections.emptyMap()))); () -> builder1.deleteJob("1", new PersistentTasksCustomMetaData(0L, Collections.emptyMap())));
} }
public void testCrudDatafeed() { public void testCrudDatafeed() {
@ -187,7 +187,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
assertThat(result.getDatafeeds().get("datafeed1"), sameInstance(datafeedConfig1)); assertThat(result.getDatafeeds().get("datafeed1"), sameInstance(datafeedConfig1));
builder = new MlMetadata.Builder(result); builder = new MlMetadata.Builder(result);
builder.removeDatafeed("datafeed1", new PersistentTasks(0, Collections.emptyMap())); builder.removeDatafeed("datafeed1", new PersistentTasksCustomMetaData(0, Collections.emptyMap()));
result = builder.build(); result = builder.build();
assertThat(result.getJobs().get("job_id"), sameInstance(job1)); assertThat(result.getJobs().get("job_id"), sameInstance(job1));
assertThat(result.getDatafeeds().get("datafeed1"), nullValue()); assertThat(result.getDatafeeds().get("datafeed1"), nullValue());
@ -271,8 +271,8 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
StartDatafeedAction.Request request = new StartDatafeedAction.Request(datafeedConfig1.getId(), 0L); StartDatafeedAction.Request request = new StartDatafeedAction.Request(datafeedConfig1.getId(), 0L);
PersistentTask<StartDatafeedAction.Request> taskInProgress = PersistentTask<StartDatafeedAction.Request> taskInProgress =
new PersistentTask<>(0, StartDatafeedAction.NAME, request, false, true, INITIAL_ASSIGNMENT); new PersistentTask<>(0, StartDatafeedAction.NAME, request, false, true, INITIAL_ASSIGNMENT);
PersistentTasks tasksInProgress = PersistentTasksCustomMetaData tasksInProgress =
new PersistentTasks(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress)); new PersistentTasksCustomMetaData(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress));
DatafeedUpdate.Builder update = new DatafeedUpdate.Builder(datafeedConfig1.getId()); DatafeedUpdate.Builder update = new DatafeedUpdate.Builder(datafeedConfig1.getId());
update.setScrollSize(5000); update.setScrollSize(5000);
@ -333,8 +333,8 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
StartDatafeedAction.Request request = new StartDatafeedAction.Request("datafeed1", 0L); StartDatafeedAction.Request request = new StartDatafeedAction.Request("datafeed1", 0L);
PersistentTask<StartDatafeedAction.Request> taskInProgress = PersistentTask<StartDatafeedAction.Request> taskInProgress =
new PersistentTask<>(0, StartDatafeedAction.NAME, request, false, true, INITIAL_ASSIGNMENT); new PersistentTask<>(0, StartDatafeedAction.NAME, request, false, true, INITIAL_ASSIGNMENT);
PersistentTasks tasksInProgress = PersistentTasksCustomMetaData tasksInProgress =
new PersistentTasks(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress)); new PersistentTasksCustomMetaData(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress));
MlMetadata.Builder builder2 = new MlMetadata.Builder(result); MlMetadata.Builder builder2 = new MlMetadata.Builder(result);
ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,

22
plugin/src/test/java/org/elasticsearch/xpack/ml/action/CloseJobActionTests.java
View File

@ -13,11 +13,11 @@ import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
@ -35,10 +35,11 @@ public class CloseJobActionTests extends ESTestCase {
createJobTask(1L, "job_id", null, randomFrom(JobState.OPENED, JobState.FAILED)); createJobTask(1L, "job_id", null, randomFrom(JobState.OPENED, JobState.FAILED));
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")) ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
.putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, Collections.singletonMap(1L, task)))); .putCustom(PersistentTasksCustomMetaData.TYPE,
new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task))));
ClusterState result = CloseJobAction.moveJobToClosingState("job_id", csBuilder.build()); ClusterState result = CloseJobAction.moveJobToClosingState("job_id", csBuilder.build());
PersistentTasks actualTasks = result.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData actualTasks = result.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(JobState.CLOSING, actualTasks.getTask(1L).getStatus()); assertEquals(JobState.CLOSING, actualTasks.getTask(1L).getStatus());
MlMetadata actualMetadata = result.metaData().custom(MlMetadata.TYPE); MlMetadata actualMetadata = result.metaData().custom(MlMetadata.TYPE);
@ -49,7 +50,7 @@ public class CloseJobActionTests extends ESTestCase {
MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); MlMetadata.Builder mlBuilder = new MlMetadata.Builder();
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")) ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
.putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, Collections.emptyMap()))); .putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(1L, Collections.emptyMap())));
expectThrows(ResourceNotFoundException.class, () -> CloseJobAction.moveJobToClosingState("job_id", csBuilder.build())); expectThrows(ResourceNotFoundException.class, () -> CloseJobAction.moveJobToClosingState("job_id", csBuilder.build()));
} }
@ -59,14 +60,15 @@ public class CloseJobActionTests extends ESTestCase {
PersistentTask<OpenJobAction.Request> task = createJobTask(1L, "job_id", null, JobState.OPENING); PersistentTask<OpenJobAction.Request> task = createJobTask(1L, "job_id", null, JobState.OPENING);
ClusterState.Builder csBuilder1 = ClusterState.builder(new ClusterName("_name")) ClusterState.Builder csBuilder1 = ClusterState.builder(new ClusterName("_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
.putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, Collections.singletonMap(1L, task)))); .putCustom(PersistentTasksCustomMetaData.TYPE,
new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task))));
ElasticsearchStatusException result = ElasticsearchStatusException result =
expectThrows(ElasticsearchStatusException.class, () -> CloseJobAction.moveJobToClosingState("job_id", csBuilder1.build())); expectThrows(ElasticsearchStatusException.class, () -> CloseJobAction.moveJobToClosingState("job_id", csBuilder1.build()));
assertEquals("cannot close job [job_id], expected job state [opened], but got [opening]", result.getMessage()); assertEquals("cannot close job [job_id], expected job state [opened], but got [opening]", result.getMessage());
ClusterState.Builder csBuilder2 = ClusterState.builder(new ClusterName("_name")) ClusterState.Builder csBuilder2 = ClusterState.builder(new ClusterName("_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
.putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, Collections.emptyMap()))); .putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(1L, Collections.emptyMap())));
result = expectThrows(ElasticsearchStatusException.class, () -> CloseJobAction.moveJobToClosingState("job_id", csBuilder2.build())); result = expectThrows(ElasticsearchStatusException.class, () -> CloseJobAction.moveJobToClosingState("job_id", csBuilder2.build()));
assertEquals("cannot close job [job_id], expected job state [opened], but got [closed]", result.getMessage()); assertEquals("cannot close job [job_id], expected job state [opened], but got [closed]", result.getMessage());
} }
@ -81,7 +83,7 @@ public class CloseJobActionTests extends ESTestCase {
tasks.put(2L, createDatafeedTask(2L, "datafeed_id", 0L, null, DatafeedState.STARTED)); tasks.put(2L, createDatafeedTask(2L, "datafeed_id", 0L, null, DatafeedState.STARTED));
ClusterState cs1 = ClusterState.builder(new ClusterName("_name")) ClusterState cs1 = ClusterState.builder(new ClusterName("_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
.putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, tasks))).build(); .putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(1L, tasks))).build();
ElasticsearchStatusException e = ElasticsearchStatusException e =
expectThrows(ElasticsearchStatusException.class, () -> CloseJobAction.validateAndFindTask("job_id", cs1)); expectThrows(ElasticsearchStatusException.class, () -> CloseJobAction.validateAndFindTask("job_id", cs1));
@ -95,7 +97,7 @@ public class CloseJobActionTests extends ESTestCase {
} }
ClusterState cs2 = ClusterState.builder(new ClusterName("_name")) ClusterState cs2 = ClusterState.builder(new ClusterName("_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
.putCustom(PersistentTasks.TYPE, new PersistentTasks(1L, tasks))).build(); .putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(1L, tasks))).build();
assertEquals(jobTask, CloseJobAction.validateAndFindTask("job_id", cs2)); assertEquals(jobTask, CloseJobAction.validateAndFindTask("job_id", cs2));
} }
@ -103,7 +105,7 @@ public class CloseJobActionTests extends ESTestCase {
String nodeId, DatafeedState datafeedState) { String nodeId, DatafeedState datafeedState) {
PersistentTask<StartDatafeedAction.Request> task = PersistentTask<StartDatafeedAction.Request> task =
new PersistentTask<>(id, StartDatafeedAction.NAME, new StartDatafeedAction.Request(datafeedId, startTime), false, true, new PersistentTask<>(id, StartDatafeedAction.NAME, new StartDatafeedAction.Request(datafeedId, startTime), false, true,
new PersistentTasks.Assignment(nodeId, "test assignment")); new PersistentTasksCustomMetaData.Assignment(nodeId, "test assignment"));
task = new PersistentTask<>(task, datafeedState); task = new PersistentTask<>(task, datafeedState);
return task; return task;
} }

21
plugin/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java
View File

@ -24,10 +24,9 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.persistent.PersistentActionCoordinator; import org.elasticsearch.xpack.persistent.PersistentTasksNodeService;
import org.elasticsearch.xpack.persistent.PersistentActionRequest; import org.elasticsearch.xpack.persistent.PersistentTaskRequest;
import org.elasticsearch.xpack.persistent.PersistentActionResponse; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.Security;
import org.junit.After; import org.junit.After;
@ -70,12 +69,13 @@ public class DatafeedJobsIT extends SecurityIntegTestCase {
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedWriteables()); List<NamedWriteableRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedWriteables());
entries.addAll(new SearchModule(Settings.EMPTY, true, Collections.emptyList()).getNamedWriteables()); entries.addAll(new SearchModule(Settings.EMPTY, true, Collections.emptyList()).getNamedWriteables());
entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new)); entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new));
entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new)); entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE,
entries.add(new NamedWriteableRegistry.Entry(PersistentActionRequest.class, StartDatafeedAction.NAME, PersistentTasksCustomMetaData::new));
entries.add(new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, StartDatafeedAction.NAME,
StartDatafeedAction.Request::new)); StartDatafeedAction.Request::new));
entries.add(new NamedWriteableRegistry.Entry(PersistentActionRequest.class, OpenJobAction.NAME, OpenJobAction.Request::new)); entries.add(new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, OpenJobAction.NAME, OpenJobAction.Request::new));
entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, PersistentActionCoordinator.Status.NAME, entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME,
PersistentActionCoordinator.Status::new)); PersistentTasksNodeService.Status::new));
entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, JobState.NAME, JobState::fromStream)); entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, JobState.NAME, JobState::fromStream));
entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream)); entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream));
final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries); final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
@ -187,8 +187,7 @@ public class DatafeedJobsIT extends SecurityIntegTestCase {
assertTrue(putDatafeedResponse.isAcknowledged()); assertTrue(putDatafeedResponse.isAcknowledged());
StartDatafeedAction.Request startDatafeedRequest = new StartDatafeedAction.Request(datafeedConfig.getId(), 0L); StartDatafeedAction.Request startDatafeedRequest = new StartDatafeedAction.Request(datafeedConfig.getId(), 0L);
PersistentActionResponse startDatafeedResponse = StartDatafeedAction.Response startDatafeedResponse = client().execute(StartDatafeedAction.INSTANCE, startDatafeedRequest).get();
client().execute(StartDatafeedAction.INSTANCE, startDatafeedRequest).get();
assertBusy(() -> { assertBusy(() -> {
DataCounts dataCounts = getDataCounts(job.getId()); DataCounts dataCounts = getDataCounts(job.getId());
assertThat(dataCounts.getProcessedRecordCount(), equalTo(numDocs1)); assertThat(dataCounts.getProcessedRecordCount(), equalTo(numDocs1));

45
plugin/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionTests.java
View File

@ -29,12 +29,11 @@ import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.net.InetAddress; import java.net.InetAddress;
import java.util.ArrayList; import java.util.ArrayList;
@ -58,14 +57,14 @@ public class OpenJobActionTests extends ESTestCase {
PersistentTask<OpenJobAction.Request> task = PersistentTask<OpenJobAction.Request> task =
createJobTask(1L, "job_id", "_node_id", randomFrom(JobState.CLOSED, JobState.FAILED)); createJobTask(1L, "job_id", "_node_id", randomFrom(JobState.CLOSED, JobState.FAILED));
PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task)); PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
OpenJobAction.validate("job_id", mlBuilder.build(), tasks, nodes); OpenJobAction.validate("job_id", mlBuilder.build(), tasks, nodes);
OpenJobAction.validate("job_id", mlBuilder.build(), new PersistentTasks(1L, Collections.emptyMap()), nodes); OpenJobAction.validate("job_id", mlBuilder.build(), new PersistentTasksCustomMetaData(1L, Collections.emptyMap()), nodes);
OpenJobAction.validate("job_id", mlBuilder.build(), null, nodes); OpenJobAction.validate("job_id", mlBuilder.build(), null, nodes);
task = createJobTask(1L, "job_id", "_other_node_id", JobState.OPENED); task = createJobTask(1L, "job_id", "_other_node_id", JobState.OPENED);
tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task)); tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
OpenJobAction.validate("job_id", mlBuilder.build(), tasks, nodes); OpenJobAction.validate("job_id", mlBuilder.build(), tasks, nodes);
} }
@ -95,7 +94,7 @@ public class OpenJobActionTests extends ESTestCase {
JobState jobState = randomFrom(JobState.OPENING, JobState.OPENED, JobState.CLOSING); JobState jobState = randomFrom(JobState.OPENING, JobState.OPENED, JobState.CLOSING);
PersistentTask<OpenJobAction.Request> task = createJobTask(1L, "job_id", "_node_id", jobState); PersistentTask<OpenJobAction.Request> task = createJobTask(1L, "job_id", "_node_id", jobState);
PersistentTasks tasks1 = new PersistentTasks(1L, Collections.singletonMap(1L, task)); PersistentTasksCustomMetaData tasks1 = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
Exception e = expectThrows(ElasticsearchStatusException.class, Exception e = expectThrows(ElasticsearchStatusException.class,
() -> OpenJobAction.validate("job_id", mlBuilder.build(), tasks1, nodes)); () -> OpenJobAction.validate("job_id", mlBuilder.build(), tasks1, nodes));
@ -103,7 +102,7 @@ public class OpenJobActionTests extends ESTestCase {
jobState = randomFrom(JobState.OPENING, JobState.CLOSING); jobState = randomFrom(JobState.OPENING, JobState.CLOSING);
task = createJobTask(1L, "job_id", "_other_node_id", jobState); task = createJobTask(1L, "job_id", "_other_node_id", jobState);
PersistentTasks tasks2 = new PersistentTasks(1L, Collections.singletonMap(1L, task)); PersistentTasksCustomMetaData tasks2 = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
e = expectThrows(ElasticsearchStatusException.class, e = expectThrows(ElasticsearchStatusException.class,
() -> OpenJobAction.validate("job_id", mlBuilder.build(), tasks2, nodes)); () -> OpenJobAction.validate("job_id", mlBuilder.build(), tasks2, nodes));
@ -129,14 +128,14 @@ public class OpenJobActionTests extends ESTestCase {
new Assignment("_node_id1", "test assignment"))); new Assignment("_node_id1", "test assignment")));
taskMap.put(2L, new PersistentTask<>(2L, OpenJobAction.NAME, new OpenJobAction.Request("job_id3"), false, true, taskMap.put(2L, new PersistentTask<>(2L, OpenJobAction.NAME, new OpenJobAction.Request("job_id3"), false, true,
new Assignment("_node_id2", "test assignment"))); new Assignment("_node_id2", "test assignment")));
PersistentTasks tasks = new PersistentTasks(3L, taskMap); PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(3L, taskMap);
ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
MetaData.Builder metaData = MetaData.builder(); MetaData.Builder metaData = MetaData.builder();
RoutingTable.Builder routingTable = RoutingTable.builder(); RoutingTable.Builder routingTable = RoutingTable.builder();
addJobAndIndices(metaData, routingTable, "job_id1", "job_id2", "job_id3", "job_id4"); addJobAndIndices(metaData, routingTable, "job_id1", "job_id2", "job_id3", "job_id4");
cs.nodes(nodes); cs.nodes(nodes);
metaData.putCustom(PersistentTasks.TYPE, tasks); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
cs.metaData(metaData); cs.metaData(metaData);
cs.routingTable(routingTable.build()); cs.routingTable(routingTable.build());
Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id4", cs.build(), 2, logger); Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id4", cs.build(), 2, logger);
@ -160,14 +159,14 @@ public class OpenJobActionTests extends ESTestCase {
taskMap.put(id, createJobTask(id, "job_id" + id, nodeId, JobState.OPENED)); taskMap.put(id, createJobTask(id, "job_id" + id, nodeId, JobState.OPENED));
} }
} }
PersistentTasks tasks = new PersistentTasks(numNodes * maxRunningJobsPerNode, taskMap); PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(numNodes * maxRunningJobsPerNode, taskMap);
ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
MetaData.Builder metaData = MetaData.builder(); MetaData.Builder metaData = MetaData.builder();
RoutingTable.Builder routingTable = RoutingTable.builder(); RoutingTable.Builder routingTable = RoutingTable.builder();
addJobAndIndices(metaData, routingTable, "job_id1", "job_id2"); addJobAndIndices(metaData, routingTable, "job_id1", "job_id2");
cs.nodes(nodes); cs.nodes(nodes);
metaData.putCustom(PersistentTasks.TYPE, tasks); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
cs.metaData(metaData); cs.metaData(metaData);
cs.routingTable(routingTable.build()); cs.routingTable(routingTable.build());
Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id2", cs.build(), 2, logger); Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id2", cs.build(), 2, logger);
@ -187,14 +186,14 @@ public class OpenJobActionTests extends ESTestCase {
PersistentTask<OpenJobAction.Request> task = PersistentTask<OpenJobAction.Request> task =
new PersistentTask<>(1L, OpenJobAction.NAME, new OpenJobAction.Request("job_id1"), false, true, new PersistentTask<>(1L, OpenJobAction.NAME, new OpenJobAction.Request("job_id1"), false, true,
new Assignment("_node_id1", "test assignment")); new Assignment("_node_id1", "test assignment"));
PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task)); PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
MetaData.Builder metaData = MetaData.builder(); MetaData.Builder metaData = MetaData.builder();
RoutingTable.Builder routingTable = RoutingTable.builder(); RoutingTable.Builder routingTable = RoutingTable.builder();
addJobAndIndices(metaData, routingTable, "job_id1", "job_id2"); addJobAndIndices(metaData, routingTable, "job_id1", "job_id2");
cs.nodes(nodes); cs.nodes(nodes);
metaData.putCustom(PersistentTasks.TYPE, tasks); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
cs.metaData(metaData); cs.metaData(metaData);
cs.routingTable(routingTable.build()); cs.routingTable(routingTable.build());
Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id2", cs.build(), 2, logger); Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id2", cs.build(), 2, logger);
@ -220,7 +219,7 @@ public class OpenJobActionTests extends ESTestCase {
taskMap.put(2L, createJobTask(2L, "job_id3", "_node_id2", JobState.OPENING)); taskMap.put(2L, createJobTask(2L, "job_id3", "_node_id2", JobState.OPENING));
taskMap.put(3L, createJobTask(3L, "job_id4", "_node_id2", JobState.OPENING)); taskMap.put(3L, createJobTask(3L, "job_id4", "_node_id2", JobState.OPENING));
taskMap.put(4L, createJobTask(4L, "job_id5", "_node_id3", JobState.OPENING)); taskMap.put(4L, createJobTask(4L, "job_id5", "_node_id3", JobState.OPENING));
PersistentTasks tasks = new PersistentTasks(5L, taskMap); PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(5L, taskMap);
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
csBuilder.nodes(nodes); csBuilder.nodes(nodes);
@ -228,7 +227,7 @@ public class OpenJobActionTests extends ESTestCase {
RoutingTable.Builder routingTable = RoutingTable.builder(); RoutingTable.Builder routingTable = RoutingTable.builder();
addJobAndIndices(metaData, routingTable, "job_id1", "job_id2", "job_id3", "job_id4", "job_id5", "job_id6", "job_id7"); addJobAndIndices(metaData, routingTable, "job_id1", "job_id2", "job_id3", "job_id4", "job_id5", "job_id6", "job_id7");
csBuilder.routingTable(routingTable.build()); csBuilder.routingTable(routingTable.build());
metaData.putCustom(PersistentTasks.TYPE, tasks); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
csBuilder.metaData(metaData); csBuilder.metaData(metaData);
ClusterState cs = csBuilder.build(); ClusterState cs = csBuilder.build();
@ -237,30 +236,30 @@ public class OpenJobActionTests extends ESTestCase {
PersistentTask<OpenJobAction.Request> lastTask = createJobTask(5L, "job_id6", "_node_id3", JobState.OPENING); PersistentTask<OpenJobAction.Request> lastTask = createJobTask(5L, "job_id6", "_node_id3", JobState.OPENING);
taskMap.put(5L, lastTask); taskMap.put(5L, lastTask);
tasks = new PersistentTasks(6L, taskMap); tasks = new PersistentTasksCustomMetaData(6L, taskMap);
csBuilder = ClusterState.builder(cs); csBuilder = ClusterState.builder(cs);
csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasks.TYPE, tasks)); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
cs = csBuilder.build(); cs = csBuilder.build();
result = OpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, logger); result = OpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, logger);
assertNull("no node selected, because OPENING state", result.getExecutorNode()); assertNull("no node selected, because OPENING state", result.getExecutorNode());
assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
taskMap.put(5L, new PersistentTask<>(lastTask, false, new Assignment("_node_id3", "test assignment"))); taskMap.put(5L, new PersistentTask<>(lastTask, false, new Assignment("_node_id3", "test assignment")));
tasks = new PersistentTasks(6L, taskMap); tasks = new PersistentTasksCustomMetaData(6L, taskMap);
csBuilder = ClusterState.builder(cs); csBuilder = ClusterState.builder(cs);
csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasks.TYPE, tasks)); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
cs = csBuilder.build(); cs = csBuilder.build();
result = OpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, logger); result = OpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, logger);
assertNull("no node selected, because stale task", result.getExecutorNode()); assertNull("no node selected, because stale task", result.getExecutorNode());
assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
taskMap.put(5L, new PersistentTask<>(lastTask, null)); taskMap.put(5L, new PersistentTask<>(lastTask, null));
tasks = new PersistentTasks(6L, taskMap); tasks = new PersistentTasksCustomMetaData(6L, taskMap);
csBuilder = ClusterState.builder(cs); csBuilder = ClusterState.builder(cs);
csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasks.TYPE, tasks)); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
cs = csBuilder.build(); cs = csBuilder.build();
result = OpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, logger); result = OpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, logger);
assertNull("no node selected, because null state", result.getExecutorNode()); assertNull("no node selected, because null state", result.getExecutorNode());

30
plugin/src/test/java/org/elasticsearch/xpack/ml/action/StartDatafeedActionTests.java
View File

@ -23,9 +23,9 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests;
import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.net.InetAddress; import java.net.InetAddress;
import java.util.Collections; import java.util.Collections;
@ -35,7 +35,7 @@ import java.util.Map;
import static org.elasticsearch.xpack.ml.action.OpenJobActionTests.createJobTask; import static org.elasticsearch.xpack.ml.action.OpenJobActionTests.createJobTask;
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createDatafeed; import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createDatafeed;
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createScheduledJob; import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createScheduledJob;
import static org.elasticsearch.xpack.persistent.PersistentTasks.INITIAL_ASSIGNMENT; import static org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
public class StartDatafeedActionTests extends ESTestCase { public class StartDatafeedActionTests extends ESTestCase {
@ -48,7 +48,7 @@ public class StartDatafeedActionTests extends ESTestCase {
JobState jobState = randomFrom(JobState.FAILED, JobState.CLOSED, JobState.CLOSING, JobState.OPENING); JobState jobState = randomFrom(JobState.FAILED, JobState.CLOSED, JobState.CLOSING, JobState.OPENING);
PersistentTask<OpenJobAction.Request> task = createJobTask(0L, job.getId(), "node_id", jobState); PersistentTask<OpenJobAction.Request> task = createJobTask(0L, job.getId(), "node_id", jobState);
PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task)); PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task));
DiscoveryNodes nodes = DiscoveryNodes.builder() DiscoveryNodes nodes = DiscoveryNodes.builder()
.add(new DiscoveryNode("node_name", "node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), .add(new DiscoveryNode("node_name", "node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
@ -57,7 +57,7 @@ public class StartDatafeedActionTests extends ESTestCase {
ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name")) ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build()) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build())
.putCustom(PersistentTasks.TYPE, tasks)) .putCustom(PersistentTasksCustomMetaData.TYPE, tasks))
.nodes(nodes); .nodes(nodes);
Assignment result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build()); Assignment result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build());
@ -66,10 +66,10 @@ public class StartDatafeedActionTests extends ESTestCase {
"] while state [opened] is required", result.getExplanation()); "] while state [opened] is required", result.getExplanation());
task = createJobTask(0L, job.getId(), "node_id", JobState.OPENED); task = createJobTask(0L, job.getId(), "node_id", JobState.OPENED);
tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task)); tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task));
cs = ClusterState.builder(new ClusterName("cluster_name")) cs = ClusterState.builder(new ClusterName("cluster_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build()) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build())
.putCustom(PersistentTasks.TYPE, tasks)) .putCustom(PersistentTasksCustomMetaData.TYPE, tasks))
.nodes(nodes); .nodes(nodes);
result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build()); result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build());
assertEquals("node_id", result.getExecutorNode()); assertEquals("node_id", result.getExecutorNode());
@ -83,7 +83,7 @@ public class StartDatafeedActionTests extends ESTestCase {
String nodeId = randomBoolean() ? "node_id2" : null; String nodeId = randomBoolean() ? "node_id2" : null;
PersistentTask<OpenJobAction.Request> task = createJobTask(0L, job.getId(), nodeId, JobState.OPENED); PersistentTask<OpenJobAction.Request> task = createJobTask(0L, job.getId(), nodeId, JobState.OPENED);
PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task)); PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task));
DiscoveryNodes nodes = DiscoveryNodes.builder() DiscoveryNodes nodes = DiscoveryNodes.builder()
.add(new DiscoveryNode("node_name", "node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), .add(new DiscoveryNode("node_name", "node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
@ -92,7 +92,7 @@ public class StartDatafeedActionTests extends ESTestCase {
ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name")) ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build()) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build())
.putCustom(PersistentTasks.TYPE, tasks)) .putCustom(PersistentTasksCustomMetaData.TYPE, tasks))
.nodes(nodes); .nodes(nodes);
Assignment result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build()); Assignment result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build());
@ -101,10 +101,10 @@ public class StartDatafeedActionTests extends ESTestCase {
result.getExplanation()); result.getExplanation());
task = createJobTask(0L, job.getId(), "node_id1", JobState.OPENED); task = createJobTask(0L, job.getId(), "node_id1", JobState.OPENED);
tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task)); tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task));
cs = ClusterState.builder(new ClusterName("cluster_name")) cs = ClusterState.builder(new ClusterName("cluster_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build()) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build())
.putCustom(PersistentTasks.TYPE, tasks)) .putCustom(PersistentTasksCustomMetaData.TYPE, tasks))
.nodes(nodes); .nodes(nodes);
result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build()); result = StartDatafeedAction.selectNode(logger, "datafeed_id", cs.build());
assertEquals("node_id1", result.getExecutorNode()); assertEquals("node_id1", result.getExecutorNode());
@ -128,7 +128,7 @@ public class StartDatafeedActionTests extends ESTestCase {
PersistentTask<OpenJobAction.Request> task = PersistentTask<OpenJobAction.Request> task =
new PersistentTask<>(0L, OpenJobAction.NAME, new OpenJobAction.Request("job_id"), false, true, new PersistentTask<>(0L, OpenJobAction.NAME, new OpenJobAction.Request("job_id"), false, true,
INITIAL_ASSIGNMENT); INITIAL_ASSIGNMENT);
PersistentTasks tasks = new PersistentTasks(0L, Collections.singletonMap(0L, task)); PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(0L, Collections.singletonMap(0L, task));
DatafeedConfig datafeedConfig1 = DatafeedJobRunnerTests.createDatafeedConfig("foo-datafeed", "job_id").build(); DatafeedConfig datafeedConfig1 = DatafeedJobRunnerTests.createDatafeedConfig("foo-datafeed", "job_id").build();
MlMetadata mlMetadata2 = new MlMetadata.Builder(mlMetadata1) MlMetadata mlMetadata2 = new MlMetadata.Builder(mlMetadata1)
.putDatafeed(datafeedConfig1) .putDatafeed(datafeedConfig1)
@ -158,7 +158,7 @@ public class StartDatafeedActionTests extends ESTestCase {
Map<Long, PersistentTask<?>> taskMap = new HashMap<>(); Map<Long, PersistentTask<?>> taskMap = new HashMap<>();
taskMap.put(0L, jobTask); taskMap.put(0L, jobTask);
taskMap.put(1L, datafeedTask); taskMap.put(1L, datafeedTask);
PersistentTasks tasks = new PersistentTasks(2L, taskMap); PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(2L, taskMap);
Exception e = expectThrows(ElasticsearchStatusException.class, Exception e = expectThrows(ElasticsearchStatusException.class,
() -> StartDatafeedAction.validate("datafeed_id", mlMetadata1, tasks, nodes)); () -> StartDatafeedAction.validate("datafeed_id", mlMetadata1, tasks, nodes));
@ -186,7 +186,7 @@ public class StartDatafeedActionTests extends ESTestCase {
Map<Long, PersistentTask<?>> taskMap = new HashMap<>(); Map<Long, PersistentTask<?>> taskMap = new HashMap<>();
taskMap.put(0L, jobTask); taskMap.put(0L, jobTask);
taskMap.put(1L, datafeedTask); taskMap.put(1L, datafeedTask);
PersistentTasks tasks = new PersistentTasks(2L, taskMap); PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(2L, taskMap);
StartDatafeedAction.validate("datafeed_id", mlMetadata1, tasks, nodes); StartDatafeedAction.validate("datafeed_id", mlMetadata1, tasks, nodes);
datafeedTask = new PersistentTask<>(0L, StartDatafeedAction.NAME, new StartDatafeedAction.Request("datafeed_id", 0L), datafeedTask = new PersistentTask<>(0L, StartDatafeedAction.NAME, new StartDatafeedAction.Request("datafeed_id", 0L),

22
plugin/src/test/java/org/elasticsearch/xpack/ml/action/StopDatafeedActionRequestTests.java
View File

@ -15,9 +15,9 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase;
import org.elasticsearch.xpack.persistent.PersistentActionRequest; import org.elasticsearch.xpack.persistent.PersistentTaskRequest;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.util.Collections; import java.util.Collections;
@ -47,10 +47,10 @@ public class StopDatafeedActionRequestTests extends AbstractStreamableXContentTe
} }
public void testValidate() { public void testValidate() {
PersistentTask<?> task = new PersistentTask<PersistentActionRequest>(1L, StartDatafeedAction.NAME, PersistentTask<?> task = new PersistentTask<PersistentTaskRequest>(1L, StartDatafeedAction.NAME,
new StartDatafeedAction.Request("foo", 0L), false, false, new PersistentTasks.Assignment("node_id", "")); new StartDatafeedAction.Request("foo", 0L), false, false, new PersistentTasksCustomMetaData.Assignment("node_id", ""));
task = new PersistentTask<>(task, DatafeedState.STARTED); task = new PersistentTask<>(task, DatafeedState.STARTED);
PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task)); PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
Job job = createDatafeedJob().build(); Job job = createDatafeedJob().build();
MlMetadata mlMetadata1 = new MlMetadata.Builder().putJob(job, false).build(); MlMetadata mlMetadata1 = new MlMetadata.Builder().putJob(job, false).build();
@ -66,14 +66,14 @@ public class StopDatafeedActionRequestTests extends AbstractStreamableXContentTe
} }
public void testValidate_alreadyStopped() { public void testValidate_alreadyStopped() {
PersistentTasks tasks; PersistentTasksCustomMetaData tasks;
if (randomBoolean()) { if (randomBoolean()) {
PersistentTask<?> task = new PersistentTask<PersistentActionRequest>(1L, StartDatafeedAction.NAME, PersistentTask<?> task = new PersistentTask<PersistentTaskRequest>(1L, StartDatafeedAction.NAME,
new StartDatafeedAction.Request("foo", 0L), false, false, new PersistentTasks.Assignment("node_id", "")); new StartDatafeedAction.Request("foo", 0L), false, false, new PersistentTasksCustomMetaData.Assignment("node_id", ""));
task = new PersistentTask<>(task, DatafeedState.STOPPED); task = new PersistentTask<>(task, DatafeedState.STOPPED);
tasks = new PersistentTasks(1L, Collections.singletonMap(1L, task)); tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(1L, task));
} else { } else {
tasks = randomBoolean() ? null : new PersistentTasks(0L, Collections.emptyMap()); tasks = randomBoolean() ? null : new PersistentTasksCustomMetaData(0L, Collections.emptyMap());
} }
Job job = createDatafeedJob().build(); Job job = createDatafeedJob().build();

25
plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunnerTests.java
View File

@ -8,7 +8,6 @@ package org.elasticsearch.xpack.ml.datafeed;
import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.Version; import org.elasticsearch.Version;
import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.client.Client; import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState;
@ -41,10 +40,10 @@ import org.elasticsearch.xpack.ml.job.persistence.MockClientBuilder;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.notifications.AuditMessage; import org.elasticsearch.xpack.ml.notifications.AuditMessage;
import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction.Response; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.junit.Before; import org.junit.Before;
import org.mockito.ArgumentCaptor; import org.mockito.ArgumentCaptor;
@ -84,6 +83,7 @@ public class DatafeedJobRunnerTests extends ESTestCase {
private DatafeedJobRunner datafeedJobRunner; private DatafeedJobRunner datafeedJobRunner;
private long currentTime = 120000; private long currentTime = 120000;
private Auditor auditor; private Auditor auditor;
private PersistentTasksService persistentTasksService;
@Before @Before
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@ -93,14 +93,14 @@ public class DatafeedJobRunnerTests extends ESTestCase {
mlMetadata.putJob(job, false); mlMetadata.putJob(job, false);
mlMetadata.putDatafeed(createDatafeedConfig("datafeed_id", job.getId()).build()); mlMetadata.putDatafeed(createDatafeedConfig("datafeed_id", job.getId()).build());
PersistentTask<OpenJobAction.Request> task = createJobTask(0L, job.getId(), "node_id", JobState.OPENED); PersistentTask<OpenJobAction.Request> task = createJobTask(0L, job.getId(), "node_id", JobState.OPENED);
PersistentTasks tasks = new PersistentTasks(1L, Collections.singletonMap(0L, task)); PersistentTasksCustomMetaData tasks = new PersistentTasksCustomMetaData(1L, Collections.singletonMap(0L, task));
DiscoveryNodes nodes = DiscoveryNodes.builder() DiscoveryNodes nodes = DiscoveryNodes.builder()
.add(new DiscoveryNode("node_name", "node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), .add(new DiscoveryNode("node_name", "node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
Collections.emptyMap(), Collections.emptySet(), Version.CURRENT)) Collections.emptyMap(), Collections.emptySet(), Version.CURRENT))
.build(); .build();
ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name")) ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name"))
.metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build()) .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build())
.putCustom(PersistentTasks.TYPE, tasks)) .putCustom(PersistentTasksCustomMetaData.TYPE, tasks))
.nodes(nodes); .nodes(nodes);
clusterService = mock(ClusterService.class); clusterService = mock(ClusterService.class);
@ -140,7 +140,9 @@ public class DatafeedJobRunnerTests extends ESTestCase {
when(client.execute(same(PostDataAction.INSTANCE), any())).thenReturn(jobDataFuture); when(client.execute(same(PostDataAction.INSTANCE), any())).thenReturn(jobDataFuture);
when(client.execute(same(FlushJobAction.INSTANCE), any())).thenReturn(flushJobFuture); when(client.execute(same(FlushJobAction.INSTANCE), any())).thenReturn(flushJobFuture);
datafeedJobRunner = new DatafeedJobRunner(threadPool, client, clusterService, jobProvider, () -> currentTime, auditor) { persistentTasksService = mock(PersistentTasksService.class);
datafeedJobRunner = new DatafeedJobRunner(threadPool, client, clusterService, jobProvider, () -> currentTime,
persistentTasksService, auditor) {
@Override @Override
DataExtractorFactory createDataExtractorFactory(DatafeedConfig datafeedConfig, Job job) { DataExtractorFactory createDataExtractorFactory(DatafeedConfig datafeedConfig, Job job) {
return dataExtractorFactory; return dataExtractorFactory;
@ -153,13 +155,12 @@ public class DatafeedJobRunnerTests extends ESTestCase {
consumer.accept(new ResourceNotFoundException("dummy")); consumer.accept(new ResourceNotFoundException("dummy"));
return null; return null;
}).when(jobProvider).bucketsViaInternalClient(any(), any(), any(), any()); }).when(jobProvider).bucketsViaInternalClient(any(), any(), any(), any());
doAnswer(invocationOnMock -> { doAnswer(invocationOnMock -> {
@SuppressWarnings("rawtypes") @SuppressWarnings("rawtypes")
ActionListener<Response> listener = (ActionListener<Response>) invocationOnMock.getArguments()[2]; PersistentTaskOperationListener listener = (PersistentTaskOperationListener) invocationOnMock.getArguments()[2];
listener.onResponse(new Response(true)); listener.onResponse(0L);
return null; return null;
}).when(client).execute(same(UpdatePersistentTaskStatusAction.INSTANCE), any(), any()); }).when(persistentTasksService).updateStatus(anyLong(), any(), any());
} }
public void testLookbackOnly_WarnsWhenNoDataIsRetrieved() throws Exception { public void testLookbackOnly_WarnsWhenNoDataIsRetrieved() throws Exception {

18
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
View File

@ -26,8 +26,8 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.io.IOException; import java.io.IOException;
import java.util.Collection; import java.util.Collection;
@ -165,7 +165,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet(); client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet();
assertBusy(() -> { assertBusy(() -> {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
PersistentTask task = tasks.taskMap().values().iterator().next(); PersistentTask task = tasks.taskMap().values().iterator().next();
DiscoveryNode node = clusterState.nodes().resolveNode(task.getExecutorNode()); DiscoveryNode node = clusterState.nodes().resolveNode(task.getExecutorNode());
@ -216,7 +216,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
// Sample each cs update and keep track each time a node holds more than `maxConcurrentJobAllocations` opening jobs. // Sample each cs update and keep track each time a node holds more than `maxConcurrentJobAllocations` opening jobs.
List<String> violations = new CopyOnWriteArrayList<>(); List<String> violations = new CopyOnWriteArrayList<>();
internalCluster().clusterService(nonMlNode).addListener(event -> { internalCluster().clusterService(nonMlNode).addListener(event -> {
PersistentTasks tasks = event.state().metaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = event.state().metaData().custom(PersistentTasksCustomMetaData.TYPE);
if (tasks == null) { if (tasks == null) {
return; return;
} }
@ -247,7 +247,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
assertBusy(() -> { assertBusy(() -> {
ClusterState state = client().admin().cluster().prepareState().get().getState(); ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(numJobs, tasks.taskMap().size()); assertEquals(numJobs, tasks.taskMap().size());
for (PersistentTask<?> task : tasks.taskMap().values()) { for (PersistentTask<?> task : tasks.taskMap().values()) {
assertNotNull(task.getExecutorNode()); assertNotNull(task.getExecutorNode());
@ -271,7 +271,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
ensureStableCluster(1, nonMlNode); ensureStableCluster(1, nonMlNode);
assertBusy(() -> { assertBusy(() -> {
ClusterState state = client(nonMlNode).admin().cluster().prepareState().get().getState(); ClusterState state = client(nonMlNode).admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(numJobs, tasks.taskMap().size()); assertEquals(numJobs, tasks.taskMap().size());
for (PersistentTask<?> task : tasks.taskMap().values()) { for (PersistentTask<?> task : tasks.taskMap().values()) {
assertNull(task.getExecutorNode()); assertNull(task.getExecutorNode());
@ -287,7 +287,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
ensureStableCluster(1 + numMlNodes); ensureStableCluster(1 + numMlNodes);
assertBusy(() -> { assertBusy(() -> {
ClusterState state = client().admin().cluster().prepareState().get().getState(); ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.metaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(numJobs, tasks.taskMap().size()); assertEquals(numJobs, tasks.taskMap().size());
for (PersistentTask<?> task : tasks.taskMap().values()) { for (PersistentTask<?> task : tasks.taskMap().values()) {
assertNotNull(task.getExecutorNode()); assertNotNull(task.getExecutorNode());
@ -332,7 +332,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
client().execute(CloseJobAction.INSTANCE, closeJobRequest); client().execute(CloseJobAction.INSTANCE, closeJobRequest);
assertBusy(() -> { assertBusy(() -> {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(0, tasks.taskMap().size()); assertEquals(0, tasks.taskMap().size());
}); });
logger.info("Stop data node"); logger.info("Stop data node");
@ -357,7 +357,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
private void assertJobTask(String jobId, JobState expectedState, boolean hasExecutorNode) { private void assertJobTask(String jobId, JobState expectedState, boolean hasExecutorNode) {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(1, tasks.taskMap().size()); assertEquals(1, tasks.taskMap().size());
PersistentTask<?> task = tasks.findTasks(OpenJobAction.NAME, p -> { PersistentTask<?> task = tasks.findTasks(OpenJobAction.NAME, p -> {
return p.getRequest() instanceof OpenJobAction.Request && return p.getRequest() instanceof OpenJobAction.Request &&

6
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java
View File

@ -24,8 +24,8 @@ import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import java.util.Collections; import java.util.Collections;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -125,7 +125,7 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase {
disrupt.run(); disrupt.run();
assertBusy(() -> { assertBusy(() -> {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = clusterState.metaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE);
assertNotNull(tasks); assertNotNull(tasks);
assertEquals(2, tasks.taskMap().size()); assertEquals(2, tasks.taskMap().size());
for (PersistentTask<?> task : tasks.tasks()) { for (PersistentTask<?> task : tasks.tasks()) {

6
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java
View File

@ -17,7 +17,7 @@ import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
public class TooManyJobsIT extends BaseMlIntegTestCase { public class TooManyJobsIT extends BaseMlIntegTestCase {
@ -49,10 +49,10 @@ public class TooManyJobsIT extends BaseMlIntegTestCase {
client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request("2")).actionGet(); client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request("2")).actionGet();
assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.CLOSED); assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.CLOSED);
ClusterState state = client().admin().cluster().prepareState().get().getState(); ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasks tasks = state.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(1, tasks.taskMap().size()); assertEquals(1, tasks.taskMap().size());
// now just double check that the first job is still opened: // now just double check that the first job is still opened:
PersistentTasks.PersistentTask task = tasks.taskMap().values().iterator().next(); PersistentTasksCustomMetaData.PersistentTask task = tasks.taskMap().values().iterator().next();
assertEquals(JobState.OPENED, task.getStatus()); assertEquals(JobState.OPENED, task.getStatus());
OpenJobAction.Request openJobRequest = (OpenJobAction.Request) task.getRequest(); OpenJobAction.Request openJobRequest = (OpenJobAction.Request) task.getRequest();
assertEquals("1", openJobRequest.getJobId()); assertEquals("1", openJobRequest.getJobId());

24
plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
View File

@ -34,7 +34,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles;
import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory; import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction; import org.elasticsearch.xpack.persistent.PersistentTasksService;
import org.junit.Before; import org.junit.Before;
import org.mockito.Mockito; import org.mockito.Mockito;
@ -122,16 +122,15 @@ public class AutodetectProcessManagerTests extends ESTestCase {
public void testOpenJob() { public void testOpenJob() {
Client client = mock(Client.class); Client client = mock(Client.class);
PersistentTasksService persistentTasksService = mock(PersistentTasksService.class);
AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); AutodetectCommunicator communicator = mock(AutodetectCommunicator.class);
when(jobManager.getJobOrThrowIfUnknown("foo")).thenReturn(createJobDetails("foo")); when(jobManager.getJobOrThrowIfUnknown("foo")).thenReturn(createJobDetails("foo"));
AutodetectProcessManager manager = createManager(communicator, client); AutodetectProcessManager manager = createManager(communicator, client, persistentTasksService);
manager.openJob("foo", 1L, false, e -> {}); manager.openJob("foo", 1L, false, e -> {});
assertEquals(1, manager.numberOfOpenJobs()); assertEquals(1, manager.numberOfOpenJobs());
assertTrue(manager.jobHasActiveAutodetectProcess("foo")); assertTrue(manager.jobHasActiveAutodetectProcess("foo"));
UpdatePersistentTaskStatusAction.Request expectedRequest = verify(persistentTasksService).updateStatus(eq(1L), eq(JobState.OPENED), any());
new UpdatePersistentTaskStatusAction.Request(1L, JobState.OPENED);
verify(client).execute(eq(UpdatePersistentTaskStatusAction.INSTANCE), eq(expectedRequest), any());
} }
public void testOpenJob_exceedMaxNumJobs() { public void testOpenJob_exceedMaxNumJobs() {
@ -149,6 +148,7 @@ public class AutodetectProcessManagerTests extends ESTestCase {
when(jobManager.getJobOrThrowIfUnknown("foobar")).thenReturn(createJobDetails("foobar")); when(jobManager.getJobOrThrowIfUnknown("foobar")).thenReturn(createJobDetails("foobar"));
Client client = mock(Client.class); Client client = mock(Client.class);
PersistentTasksService persistentTasksService = mock(PersistentTasksService.class);
ThreadPool threadPool = mock(ThreadPool.class); ThreadPool threadPool = mock(ThreadPool.class);
ThreadPool.Cancellable cancellable = mock(ThreadPool.Cancellable.class); ThreadPool.Cancellable cancellable = mock(ThreadPool.Cancellable.class);
when(threadPool.scheduleWithFixedDelay(any(), any(), any())).thenReturn(cancellable); when(threadPool.scheduleWithFixedDelay(any(), any(), any())).thenReturn(cancellable);
@ -161,7 +161,7 @@ public class AutodetectProcessManagerTests extends ESTestCase {
Settings.Builder settings = Settings.builder(); Settings.Builder settings = Settings.builder();
settings.put(AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE.getKey(), 3); settings.put(AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE.getKey(), 3);
AutodetectProcessManager manager = spy(new AutodetectProcessManager(settings.build(), client, threadPool, jobManager, jobProvider, AutodetectProcessManager manager = spy(new AutodetectProcessManager(settings.build(), client, threadPool, jobManager, jobProvider,
jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory)); jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory, persistentTasksService));
DataCounts dataCounts = new DataCounts("foo"); DataCounts dataCounts = new DataCounts("foo");
ModelSnapshot modelSnapshot = new ModelSnapshot.Builder("foo").build(); ModelSnapshot modelSnapshot = new ModelSnapshot.Builder("foo").build();
@ -319,11 +319,11 @@ public class AutodetectProcessManagerTests extends ESTestCase {
handler.accept(new DataCounts(jobId)); handler.accept(new DataCounts(jobId));
return null; return null;
}).when(jobProvider).dataCounts(eq("my_id"), any(), any()); }).when(jobProvider).dataCounts(eq("my_id"), any(), any());
PersistentTasksService persistentTasksService = mock(PersistentTasksService.class);
AutodetectProcess autodetectProcess = mock(AutodetectProcess.class); AutodetectProcess autodetectProcess = mock(AutodetectProcess.class);
AutodetectProcessFactory autodetectProcessFactory = (j, modelSnapshot, quantiles, filters, i, e) -> autodetectProcess; AutodetectProcessFactory autodetectProcessFactory = (j, modelSnapshot, quantiles, filters, i, e) -> autodetectProcess;
AutodetectProcessManager manager = new AutodetectProcessManager(Settings.EMPTY, client, threadPool, jobManager, jobProvider, AutodetectProcessManager manager = new AutodetectProcessManager(Settings.EMPTY, client, threadPool, jobManager, jobProvider,
jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory); jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory, persistentTasksService);
expectThrows(EsRejectedExecutionException.class, expectThrows(EsRejectedExecutionException.class,
() -> manager.create("my_id", 1L, dataCounts, modelSnapshot, quantiles, filters, false, e -> {})); () -> manager.create("my_id", 1L, dataCounts, modelSnapshot, quantiles, filters, false, e -> {}));
@ -332,14 +332,16 @@ public class AutodetectProcessManagerTests extends ESTestCase {
private AutodetectProcessManager createManager(AutodetectCommunicator communicator) { private AutodetectProcessManager createManager(AutodetectCommunicator communicator) {
Client client = mock(Client.class); Client client = mock(Client.class);
return createManager(communicator, client); PersistentTasksService persistentTasksService = mock(PersistentTasksService.class);
return createManager(communicator, client, persistentTasksService);
} }
private AutodetectProcessManager createManager(AutodetectCommunicator communicator, Client client) { private AutodetectProcessManager createManager(AutodetectCommunicator communicator, Client client,
PersistentTasksService persistentTasksService) {
ThreadPool threadPool = mock(ThreadPool.class); ThreadPool threadPool = mock(ThreadPool.class);
AutodetectProcessFactory autodetectProcessFactory = mock(AutodetectProcessFactory.class); AutodetectProcessFactory autodetectProcessFactory = mock(AutodetectProcessFactory.class);
AutodetectProcessManager manager = new AutodetectProcessManager(Settings.EMPTY, client, threadPool, jobManager, jobProvider, AutodetectProcessManager manager = new AutodetectProcessManager(Settings.EMPTY, client, threadPool, jobManager, jobProvider,
jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory); jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory, persistentTasksService);
manager = spy(manager); manager = spy(manager);
doReturn(communicator).when(manager) doReturn(communicator).when(manager)
.create(any(), anyLong(), eq(dataCounts), eq(modelSnapshot), eq(quantiles), eq(filters), anyBoolean(), any()); .create(any(), anyLong(), eq(dataCounts), eq(modelSnapshot), eq(quantiles), eq(filters), anyBoolean(), any());

50
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionRegistryTests.java
View File

@ -1,50 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.ThreadPool;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class PersistentActionRegistryTests extends ESTestCase {
public void testActionLookup() {
PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY);
TransportPersistentAction<?> action1 = mock(TransportPersistentAction.class);
when(action1.getExecutor()).thenReturn(ThreadPool.Names.MANAGEMENT);
TransportPersistentAction<?> action2 = mock(TransportPersistentAction.class);
when(action2.getExecutor()).thenReturn(ThreadPool.Names.GENERIC);
registry.registerPersistentAction("test1", action1);
registry.registerPersistentAction("test2", action2);
assertEquals(registry.getPersistentActionHolderSafe("test1").getAction(), "test1");
assertEquals(registry.getPersistentActionHolderSafe("test1").getExecutor(), ThreadPool.Names.MANAGEMENT);
assertEquals(registry.getPersistentActionHolderSafe("test1").getPersistentAction(), action1);
assertEquals(registry.getPersistentActionSafe("test1"), action1);
assertEquals(registry.getPersistentActionHolderSafe("test2").getAction(), "test2");
assertEquals(registry.getPersistentActionHolderSafe("test2").getExecutor(), ThreadPool.Names.GENERIC);
assertEquals(registry.getPersistentActionHolderSafe("test2").getPersistentAction(), action2);
assertEquals(registry.getPersistentActionSafe("test2"), action2);
try {
registry.getPersistentActionHolderSafe("test3");
fail("Should have failed");
} catch (IllegalStateException ex) {
assertEquals(ex.getMessage(), "Unknown persistent action [test3]");
}
try {
registry.getPersistentActionSafe("test3");
fail("Should have failed");
} catch (IllegalStateException ex) {
assertEquals(ex.getMessage(), "Unknown persistent action [test3]");
}
}
}

89
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterServiceTests.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksClusterServiceTests.java
View File

@ -18,9 +18,9 @@ import org.elasticsearch.cluster.routing.RoutingTable;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest; import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -29,13 +29,13 @@ import java.util.HashSet;
import java.util.List; import java.util.List;
import static java.util.Collections.emptyMap; import static java.util.Collections.emptyMap;
import static org.elasticsearch.xpack.persistent.TransportPersistentAction.NO_NODE_FOUND; import static org.elasticsearch.xpack.persistent.PersistentTasksExecutor.NO_NODE_FOUND;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.nullValue;
public class PersistentTaskClusterServiceTests extends ESTestCase { public class PersistentTasksClusterServiceTests extends ESTestCase {
public void testReassignmentRequired() { public void testReassignmentRequired() {
int numberOfIterations = randomIntBetween(1, 30); int numberOfIterations = randomIntBetween(1, 30);
@ -50,10 +50,10 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
clusterState = insignificantChange(clusterState); clusterState = insignificantChange(clusterState);
} }
ClusterChangedEvent event = new ClusterChangedEvent("test", clusterState, previousState); ClusterChangedEvent event = new ClusterChangedEvent("test", clusterState, previousState);
assertThat(dumpEvent(event), PersistentTaskClusterService.reassignmentRequired(event, assertThat(dumpEvent(event), PersistentTasksClusterService.reassignmentRequired(event,
new PersistentTaskClusterService.ExecutorNodeDecider() { new PersistentTasksClusterService.ExecutorNodeDecider() {
@Override @Override
public <Request extends PersistentActionRequest> Assignment getAssignment( public <Request extends PersistentTaskRequest> Assignment getAssignment(
String action, ClusterState currentState, Request request) { String action, ClusterState currentState, Request request) {
if ("never_assign".equals(((TestRequest) request).getTestParam())) { if ("never_assign".equals(((TestRequest) request).getTestParam())) {
return NO_NODE_FOUND; return NO_NODE_FOUND;
@ -66,14 +66,14 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
public void testReassignTasksWithNoTasks() { public void testReassignTasksWithNoTasks() {
ClusterState clusterState = initialState(); ClusterState clusterState = initialState();
assertThat(reassign(clusterState).metaData().custom(PersistentTasks.TYPE), nullValue()); assertThat(reassign(clusterState).metaData().custom(PersistentTasksCustomMetaData.TYPE), nullValue());
} }
public void testReassignConsidersClusterStateUpdates() { public void testReassignConsidersClusterStateUpdates() {
ClusterState clusterState = initialState(); ClusterState clusterState = initialState();
ClusterState.Builder builder = ClusterState.builder(clusterState); ClusterState.Builder builder = ClusterState.builder(clusterState);
PersistentTasks.Builder tasks = PersistentTasks.builder( PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(
clusterState.metaData().custom(PersistentTasks.TYPE)); clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE));
DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes());
addTestNodes(nodes, randomIntBetween(1, 10)); addTestNodes(nodes, randomIntBetween(1, 10));
int numberOfTasks = randomIntBetween(2, 40); int numberOfTasks = randomIntBetween(2, 40);
@ -81,11 +81,11 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
addTask(tasks, "should_assign", "assign_one", randomBoolean() ? null : "no_longer_exits", false); addTask(tasks, "should_assign", "assign_one", randomBoolean() ? null : "no_longer_exits", false);
} }
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, tasks.build()); MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build());
clusterState = builder.metaData(metaData).nodes(nodes).build(); clusterState = builder.metaData(metaData).nodes(nodes).build();
ClusterState newClusterState = reassign(clusterState); ClusterState newClusterState = reassign(clusterState);
PersistentTasks tasksInProgress = newClusterState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasksInProgress, notNullValue()); assertThat(tasksInProgress, notNullValue());
} }
@ -93,8 +93,8 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
public void testReassignTasks() { public void testReassignTasks() {
ClusterState clusterState = initialState(); ClusterState clusterState = initialState();
ClusterState.Builder builder = ClusterState.builder(clusterState); ClusterState.Builder builder = ClusterState.builder(clusterState);
PersistentTasks.Builder tasks = PersistentTasks.builder( PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(
clusterState.metaData().custom(PersistentTasks.TYPE)); clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE));
DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes());
addTestNodes(nodes, randomIntBetween(1, 10)); addTestNodes(nodes, randomIntBetween(1, 10));
int numberOfTasks = randomIntBetween(0, 40); int numberOfTasks = randomIntBetween(0, 40);
@ -118,11 +118,11 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
} }
} }
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, tasks.build()); MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build());
clusterState = builder.metaData(metaData).nodes(nodes).build(); clusterState = builder.metaData(metaData).nodes(nodes).build();
ClusterState newClusterState = reassign(clusterState); ClusterState newClusterState = reassign(clusterState);
PersistentTasks tasksInProgress = newClusterState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasksInProgress, notNullValue()); assertThat(tasksInProgress, notNullValue());
assertThat("number of tasks shouldn't change as a result or reassignment", assertThat("number of tasks shouldn't change as a result or reassignment",
@ -133,15 +133,15 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
for (PersistentTask<?> task : tasksInProgress.tasks()) { for (PersistentTask<?> task : tasksInProgress.tasks()) {
if (task.isStopped()) { if (task.isStopped()) {
assertThat("stopped tasks should be never assigned", task.getExecutorNode(), nullValue()); assertThat("stopped tasks should be never assigned", task.getExecutorNode(), nullValue());
assertThat(task.getAssignment().getExplanation(), equalTo("explanation: " + task.getAction())); assertThat(task.getAssignment().getExplanation(), equalTo("explanation: " + task.getTaskName()));
} else { } else {
// explanation should correspond to the action name // explanation should correspond to the action name
switch (task.getAction()) { switch (task.getTaskName()) {
case "should_assign": case "should_assign":
assertThat(task.getExecutorNode(), notNullValue()); assertThat(task.getExecutorNode(), notNullValue());
assertThat(task.isAssigned(), equalTo(true)); assertThat(task.isAssigned(), equalTo(true));
if (clusterState.nodes().nodeExists(task.getExecutorNode()) == false) { if (clusterState.nodes().nodeExists(task.getExecutorNode()) == false) {
logger.info(clusterState.metaData().custom(PersistentTasks.TYPE).toString()); logger.info(clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE).toString());
} }
assertThat("task should be assigned to a node that is in the cluster, was assigned to " + task.getExecutorNode(), assertThat("task should be assigned to a node that is in the cluster, was assigned to " + task.getExecutorNode(),
clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true)); clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true));
@ -162,7 +162,7 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
} }
break; break;
default: default:
fail("Unknown action " + task.getAction()); fail("Unknown action " + task.getTaskName());
} }
} }
} }
@ -176,10 +176,10 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
} }
private ClusterState reassign(ClusterState clusterState) { private ClusterState reassign(ClusterState clusterState) {
return PersistentTaskClusterService.reassignTasks(clusterState, logger, return PersistentTasksClusterService.reassignTasks(clusterState, logger,
new PersistentTaskClusterService.ExecutorNodeDecider() { new PersistentTasksClusterService.ExecutorNodeDecider() {
@Override @Override
public <Request extends PersistentActionRequest> Assignment getAssignment( public <Request extends PersistentTaskRequest> Assignment getAssignment(
String action, ClusterState currentState, Request request) { String action, ClusterState currentState, Request request) {
TestRequest testRequest = (TestRequest) request; TestRequest testRequest = (TestRequest) request;
switch (testRequest.getTestParam()) { switch (testRequest.getTestParam()) {
@ -203,7 +203,7 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
private Assignment assignOnlyOneTaskAtATime(ClusterState clusterState) { private Assignment assignOnlyOneTaskAtATime(ClusterState clusterState) {
DiscoveryNodes nodes = clusterState.nodes(); DiscoveryNodes nodes = clusterState.nodes();
PersistentTasks tasksInProgress = clusterState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasksInProgress = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
if (tasksInProgress.findTasks("assign_one", if (tasksInProgress.findTasks("assign_one",
task -> task.isStopped() == false && nodes.nodeExists(task.getExecutorNode())).isEmpty()) { task -> task.isStopped() == false && nodes.nodeExists(task.getExecutorNode())).isEmpty()) {
return randomNodeAssignment(clusterState.nodes()); return randomNodeAssignment(clusterState.nodes());
@ -232,12 +232,12 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
return "nodes_changed: " + event.nodesChanged() + return "nodes_changed: " + event.nodesChanged() +
" nodes_removed:" + event.nodesRemoved() + " nodes_removed:" + event.nodesRemoved() +
" routing_table_changed:" + event.routingTableChanged() + " routing_table_changed:" + event.routingTableChanged() +
" tasks: " + event.state().metaData().custom(PersistentTasks.TYPE); " tasks: " + event.state().metaData().custom(PersistentTasksCustomMetaData.TYPE);
} }
private ClusterState significantChange(ClusterState clusterState) { private ClusterState significantChange(ClusterState clusterState) {
ClusterState.Builder builder = ClusterState.builder(clusterState); ClusterState.Builder builder = ClusterState.builder(clusterState);
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
if (tasks != null) { if (tasks != null) {
if (randomBoolean()) { if (randomBoolean()) {
for (PersistentTask<?> task : tasks.tasks()) { for (PersistentTask<?> task : tasks.tasks()) {
@ -255,11 +255,12 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
// we don't have any unassigned tasks - add some // we don't have any unassigned tasks - add some
if (randomBoolean()) { if (randomBoolean()) {
logger.info("added random task"); logger.info("added random task");
addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasks.builder(tasks), null, false); addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks), null,
false);
tasksOrNodesChanged = true; tasksOrNodesChanged = true;
} else { } else {
logger.info("added unassignable task with custom assignment message"); logger.info("added unassignable task with custom assignment message");
addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasks.builder(tasks), addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks),
new Assignment(null, "change me"), "never_assign", false); new Assignment(null, "change me"), "never_assign", false);
tasksOrNodesChanged = true; tasksOrNodesChanged = true;
} }
@ -282,10 +283,10 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
return builder.build(); return builder.build();
} }
private PersistentTasks removeTasksWithChangingAssignment(PersistentTasks tasks) { private PersistentTasksCustomMetaData removeTasksWithChangingAssignment(PersistentTasksCustomMetaData tasks) {
if (tasks != null) { if (tasks != null) {
boolean changed = false; boolean changed = false;
PersistentTasks.Builder tasksBuilder = PersistentTasks.builder(tasks); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
for (PersistentTask<?> task : tasks.tasks()) { for (PersistentTask<?> task : tasks.tasks()) {
// Remove all unassigned tasks that cause changing assignments they might trigger a significant change // Remove all unassigned tasks that cause changing assignments they might trigger a significant change
if ("never_assign".equals(((TestRequest) task.getRequest()).getTestParam()) && if ("never_assign".equals(((TestRequest) task.getRequest()).getTestParam()) &&
@ -304,9 +305,9 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
private ClusterState insignificantChange(ClusterState clusterState) { private ClusterState insignificantChange(ClusterState clusterState) {
ClusterState.Builder builder = ClusterState.builder(clusterState); ClusterState.Builder builder = ClusterState.builder(clusterState);
PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
tasks = removeTasksWithChangingAssignment(tasks); tasks = removeTasksWithChangingAssignment(tasks);
PersistentTasks.Builder tasksBuilder = PersistentTasks.builder(tasks); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
if (randomBoolean()) { if (randomBoolean()) {
if (hasAssignableTasks(tasks, clusterState.nodes()) == false) { if (hasAssignableTasks(tasks, clusterState.nodes()) == false) {
@ -328,7 +329,7 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
} else { } else {
logger.info("changed routing table"); logger.info("changed routing table");
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()); MetaData.Builder metaData = MetaData.builder(clusterState.metaData());
metaData.putCustom(PersistentTasks.TYPE, tasksBuilder.build()); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build());
RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable()); RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable());
changeRoutingTable(metaData, routingTable); changeRoutingTable(metaData, routingTable);
builder.metaData(metaData).routingTable(routingTable.build()); builder.metaData(metaData).routingTable(routingTable.build());
@ -350,12 +351,12 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
// clear the task // clear the task
if (randomBoolean()) { if (randomBoolean()) {
logger.info("removed all tasks"); logger.info("removed all tasks");
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE,
PersistentTasks.builder().build()); PersistentTasksCustomMetaData.builder().build());
return builder.metaData(metaData).build(); return builder.metaData(metaData).build();
} else { } else {
logger.info("set task custom to null"); logger.info("set task custom to null");
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).removeCustom(PersistentTasks.TYPE); MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).removeCustom(PersistentTasksCustomMetaData.TYPE);
return builder.metaData(metaData).build(); return builder.metaData(metaData).build();
} }
} }
@ -374,11 +375,11 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
.numberOfReplicas(1) .numberOfReplicas(1)
.build(); .build();
MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).put(indexMetaData, false) MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).put(indexMetaData, false)
.putCustom(PersistentTasks.TYPE, tasksBuilder.build()); .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build());
return builder.metaData(metaData).build(); return builder.metaData(metaData).build();
} }
private boolean hasAssignableTasks(PersistentTasks tasks, DiscoveryNodes discoveryNodes) { private boolean hasAssignableTasks(PersistentTasksCustomMetaData tasks, DiscoveryNodes discoveryNodes) {
if (tasks == null || tasks.tasks().isEmpty()) { if (tasks == null || tasks.tasks().isEmpty()) {
return false; return false;
} }
@ -393,26 +394,26 @@ public class PersistentTaskClusterServiceTests extends ESTestCase {
}); });
} }
private boolean hasTasksAssignedTo(PersistentTasks tasks, String nodeId) { private boolean hasTasksAssignedTo(PersistentTasksCustomMetaData tasks, String nodeId) {
return tasks != null && tasks.tasks().stream().anyMatch( return tasks != null && tasks.tasks().stream().anyMatch(
task -> nodeId.equals(task.getExecutorNode())) == false; task -> nodeId.equals(task.getExecutorNode())) == false;
} }
private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder,
MetaData.Builder metaData, PersistentTasks.Builder tasks, MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks,
String node, boolean stopped) { String node, boolean stopped) {
return addRandomTask(clusterStateBuilder, metaData, tasks, new Assignment(node, randomAsciiOfLength(10)), return addRandomTask(clusterStateBuilder, metaData, tasks, new Assignment(node, randomAsciiOfLength(10)),
randomAsciiOfLength(10), stopped); randomAsciiOfLength(10), stopped);
} }
private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder,
MetaData.Builder metaData, PersistentTasks.Builder tasks, MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks,
Assignment assignment, String param, boolean stopped) { Assignment assignment, String param, boolean stopped) {
return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasks.TYPE, return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksCustomMetaData.TYPE,
tasks.addTask(randomAsciiOfLength(10), new TestRequest(param), stopped, randomBoolean(), assignment).build())); tasks.addTask(randomAsciiOfLength(10), new TestRequest(param), stopped, randomBoolean(), assignment).build()));
} }
private void addTask(PersistentTasks.Builder tasks, String action, String param, String node, boolean stopped) { private void addTask(PersistentTasksCustomMetaData.Builder tasks, String action, String param, String node, boolean stopped) {
tasks.addTask(action, new TestRequest(param), stopped, randomBoolean(), new Assignment(node, "explanation: " + action)); tasks.addTask(action, new TestRequest(param), stopped, randomBoolean(), new Assignment(node, "explanation: " + action));
} }

54
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksTests.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksCustomMetaDataTests.java
View File

@ -21,12 +21,12 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.Task;
import org.elasticsearch.test.AbstractDiffableSerializationTestCase; import org.elasticsearch.test.AbstractDiffableSerializationTestCase;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.PersistentTasks.Builder; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Builder;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.Status; import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.Status;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestPersistentAction; import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest; import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@ -35,17 +35,17 @@ import java.util.Collections;
import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_GATEWAY; import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_GATEWAY;
import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_SNAPSHOT; import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_SNAPSHOT;
import static org.elasticsearch.xpack.persistent.TransportPersistentAction.NO_NODE_FOUND; import static org.elasticsearch.xpack.persistent.PersistentTasksExecutor.NO_NODE_FOUND;
public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<Custom> { public class PersistentTasksCustomMetaDataTests extends AbstractDiffableSerializationTestCase<Custom> {
@Override @Override
protected PersistentTasks createTestInstance() { protected PersistentTasksCustomMetaData createTestInstance() {
int numberOfTasks = randomInt(10); int numberOfTasks = randomInt(10);
PersistentTasks.Builder tasks = PersistentTasks.builder(); PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder();
for (int i = 0; i < numberOfTasks; i++) { for (int i = 0; i < numberOfTasks; i++) {
boolean stopped = randomBoolean(); boolean stopped = randomBoolean();
tasks.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), tasks.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)),
stopped, randomBoolean(), stopped ? new Assignment(null, "stopped") : randomAssignment()); stopped, randomBoolean(), stopped ? new Assignment(null, "stopped") : randomAssignment());
if (randomBoolean()) { if (randomBoolean()) {
// From time to time update status // From time to time update status
@ -57,22 +57,22 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
@Override @Override
protected Writeable.Reader<Custom> instanceReader() { protected Writeable.Reader<Custom> instanceReader() {
return PersistentTasks::new; return PersistentTasksCustomMetaData::new;
} }
@Override @Override
protected NamedWriteableRegistry getNamedWriteableRegistry() { protected NamedWriteableRegistry getNamedWriteableRegistry() {
return new NamedWriteableRegistry(Arrays.asList( return new NamedWriteableRegistry(Arrays.asList(
new Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new), new Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::new),
new Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom), new Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::readDiffFrom),
new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new), new Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new),
new Entry(Task.Status.class, Status.NAME, Status::new) new Entry(Task.Status.class, Status.NAME, Status::new)
)); ));
} }
@Override @Override
protected Custom makeTestChanges(Custom testInstance) { protected Custom makeTestChanges(Custom testInstance) {
PersistentTasks tasksInProgress = (PersistentTasks) testInstance; PersistentTasksCustomMetaData tasksInProgress = (PersistentTasksCustomMetaData) testInstance;
Builder builder = new Builder(); Builder builder = new Builder();
switch (randomInt(3)) { switch (randomInt(3)) {
case 0: case 0:
@ -105,12 +105,12 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
@Override @Override
protected Writeable.Reader<Diff<Custom>> diffReader() { protected Writeable.Reader<Diff<Custom>> diffReader() {
return PersistentTasks::readDiffFrom; return PersistentTasksCustomMetaData::readDiffFrom;
} }
@Override @Override
protected PersistentTasks doParseInstance(XContentParser parser) throws IOException { protected PersistentTasksCustomMetaData doParseInstance(XContentParser parser) throws IOException {
return PersistentTasks.fromXContent(parser); return PersistentTasksCustomMetaData.fromXContent(parser);
} }
@Override @Override
@ -137,19 +137,19 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
private Builder addRandomTask(Builder builder) { private Builder addRandomTask(Builder builder) {
boolean stopped = randomBoolean(); boolean stopped = randomBoolean();
builder.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(), builder.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(),
stopped ? new Assignment(null, "stopped") : randomAssignment()); stopped ? new Assignment(null, "stopped") : randomAssignment());
return builder; return builder;
} }
private long pickRandomTask(PersistentTasks testInstance) { private long pickRandomTask(PersistentTasksCustomMetaData testInstance) {
return randomFrom(new ArrayList<>(testInstance.tasks())).getId(); return randomFrom(new ArrayList<>(testInstance.tasks())).getId();
} }
@Override @Override
protected NamedXContentRegistry xContentRegistry() { protected NamedXContentRegistry xContentRegistry() {
return new NamedXContentRegistry(Arrays.asList( return new NamedXContentRegistry(Arrays.asList(
new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(TestPersistentAction.NAME), new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(TestPersistentTasksExecutor.NAME),
TestRequest::fromXContent), TestRequest::fromXContent),
new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent)
)); ));
@ -157,9 +157,9 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public void testSerializationContext() throws Exception { public void testSerializationContext() throws Exception {
PersistentTasks testInstance = createTestInstance(); PersistentTasksCustomMetaData testInstance = createTestInstance();
for (int i = 0; i < randomInt(10); i++) { for (int i = 0; i < randomInt(10); i++) {
testInstance = (PersistentTasks) makeTestChanges(testInstance); testInstance = (PersistentTasksCustomMetaData) makeTestChanges(testInstance);
} }
ToXContent.MapParams params = new ToXContent.MapParams( ToXContent.MapParams params = new ToXContent.MapParams(
@ -170,7 +170,7 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
XContentBuilder shuffled = shuffleXContent(builder); XContentBuilder shuffled = shuffleXContent(builder);
XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled.bytes()); XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled.bytes());
PersistentTasks newInstance = doParseInstance(parser); PersistentTasksCustomMetaData newInstance = doParseInstance(parser);
assertNotSame(newInstance, testInstance); assertNotSame(newInstance, testInstance);
assertEquals(testInstance.tasks().size(), newInstance.tasks().size()); assertEquals(testInstance.tasks().size(), newInstance.tasks().size());
@ -179,7 +179,7 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
assertNotNull(newTask); assertNotNull(newTask);
// Things that should be serialized // Things that should be serialized
assertEquals(testTask.getAction(), newTask.getAction()); assertEquals(testTask.getTaskName(), newTask.getTaskName());
assertEquals(testTask.getId(), newTask.getId()); assertEquals(testTask.getId(), newTask.getId());
assertEquals(testTask.getStatus(), newTask.getStatus()); assertEquals(testTask.getStatus(), newTask.getStatus());
assertEquals(testTask.getRequest(), newTask.getRequest()); assertEquals(testTask.getRequest(), newTask.getRequest());
@ -192,7 +192,7 @@ public class PersistentTasksTests extends AbstractDiffableSerializationTestCase<
} }
public void testBuilder() { public void testBuilder() {
PersistentTasks persistentTasks = null; PersistentTasksCustomMetaData persistentTasks = null;
long lastKnownTask = -1; long lastKnownTask = -1;
for (int i = 0; i < randomIntBetween(10, 100); i++) { for (int i = 0; i < randomIntBetween(10, 100); i++) {
final Builder builder; final Builder builder;

68
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionFullRestartIT.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorFullRestartIT.java
View File

@ -8,14 +8,16 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.junit.annotations.TestLogging;
import org.elasticsearch.xpack.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.xpack.persistent.PersistentTasksExecutorIT.PersistentTaskOperationFuture;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestPersistentAction; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest; import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest;
import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.List;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThan;
@ -23,10 +25,10 @@ import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.nullValue;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, minNumDataNodes = 1) @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, minNumDataNodes = 1)
public class PersistentActionFullRestartIT extends ESIntegTestCase { public class PersistentTasksExecutorFullRestartIT extends ESIntegTestCase {
@Override @Override
protected Collection<Class<? extends Plugin>> nodePlugins() { protected Collection<Class<? extends Plugin>> nodePlugins() {
return Collections.singletonList(TestPersistentActionPlugin.class); return Collections.singletonList(TestPersistentTasksPlugin.class);
} }
@Override @Override
@ -40,35 +42,38 @@ public class PersistentActionFullRestartIT extends ESIntegTestCase {
@TestLogging("org.elasticsearch.xpack.persistent:TRACE,org.elasticsearch.cluster.service:DEBUG") @TestLogging("org.elasticsearch.xpack.persistent:TRACE,org.elasticsearch.cluster.service:DEBUG")
public void testFullClusterRestart() throws Exception { public void testFullClusterRestart() throws Exception {
PersistentTasksService service = internalCluster().getInstance(PersistentTasksService.class);
int numberOfTasks = randomIntBetween(1, 10); int numberOfTasks = randomIntBetween(1, 10);
long[] taskIds = new long[numberOfTasks]; long[] taskIds = new long[numberOfTasks];
List<PersistentTaskOperationFuture> futures = new ArrayList<>(numberOfTasks);
boolean[] stopped = new boolean[numberOfTasks]; boolean[] stopped = new boolean[numberOfTasks];
int runningTasks = 0; int runningTasks = 0;
for (int i = 0; i < numberOfTasks; i++) { for (int i = 0; i < numberOfTasks; i++) {
if (randomBoolean()) { stopped[i] = randomBoolean();
if (stopped[i] == false) {
runningTasks++; runningTasks++;
taskIds[i] = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId();
stopped[i] = false;
} else {
taskIds[i] = CreatePersistentTaskAction.INSTANCE.newRequestBuilder(client())
.setAction(TestPersistentAction.NAME)
.setRequest(new TestRequest("Blah"))
.setStopped(true)
.get().getTaskId();
stopped[i] = true;
} }
PersistentTaskOperationFuture future = new PersistentTaskOperationFuture();
futures.add(future);
service.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), stopped[i], true, future);
} }
for (int i = 0; i < numberOfTasks; i++) {
taskIds[i] = futures.get(i).get();
}
final int numberOfRunningTasks = runningTasks; final int numberOfRunningTasks = runningTasks;
PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData() PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE); .custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks));
if (numberOfRunningTasks > 0) { if (numberOfRunningTasks > 0) {
// Make sure that at least one of the tasks is running // Make sure that at least one of the tasks is running
assertBusy(() -> { assertBusy(() -> {
// Wait for the task to start // Wait for the task to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.getTasks().size(), greaterThan(0)); .getTasks().size(), greaterThan(0));
}); });
} }
@ -76,7 +81,7 @@ public class PersistentActionFullRestartIT extends ESIntegTestCase {
internalCluster().fullRestart(); internalCluster().fullRestart();
ensureYellow(); ensureYellow();
tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasks.TYPE); tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks));
// Check that cluster state is correct // Check that cluster state is correct
for (int i = 0; i < numberOfTasks; i++) { for (int i = 0; i < numberOfTasks; i++) {
@ -88,12 +93,13 @@ public class PersistentActionFullRestartIT extends ESIntegTestCase {
logger.info("Waiting for {} original tasks to start", numberOfRunningTasks); logger.info("Waiting for {} original tasks to start", numberOfRunningTasks);
assertBusy(() -> { assertBusy(() -> {
// Wait for the running task to start automatically // Wait for the running task to start automatically
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
equalTo(numberOfRunningTasks)); .getTasks().size(), equalTo(numberOfRunningTasks));
}); });
// Start all other tasks // Start all other tasks
tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasks.TYPE); tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
service = internalCluster().getInstance(PersistentTasksService.class);
for (int i = 0; i < numberOfTasks; i++) { for (int i = 0; i < numberOfTasks; i++) {
PersistentTask<?> task = tasksInProgress.getTask(taskIds[i]); PersistentTask<?> task = tasksInProgress.getTask(taskIds[i]);
assertNotNull(task); assertNotNull(task);
@ -101,26 +107,28 @@ public class PersistentActionFullRestartIT extends ESIntegTestCase {
assertThat(task.isStopped(), equalTo(stopped[i])); assertThat(task.isStopped(), equalTo(stopped[i]));
assertThat(task.getExecutorNode(), stopped[i] ? nullValue() : notNullValue()); assertThat(task.getExecutorNode(), stopped[i] ? nullValue() : notNullValue());
if (stopped[i]) { if (stopped[i]) {
assertAcked(StartPersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(task.getId()).get()); PersistentTaskOperationFuture startFuture = new PersistentTaskOperationFuture();
service.startTask(task.getId(), startFuture);
assertEquals(startFuture.get(), (Long) task.getId());
} }
} }
logger.info("Waiting for {} tasks to start", numberOfTasks); logger.info("Waiting for {} tasks to start", numberOfTasks);
assertBusy(() -> { assertBusy(() -> {
// Wait for all tasks to start // Wait for all tasks to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
equalTo(numberOfTasks)); .getTasks().size(), equalTo(numberOfTasks));
}); });
logger.info("Complete all tasks"); logger.info("Complete all tasks");
// Complete the running task and make sure it finishes properly // Complete the running task and make sure it finishes properly
assertThat(new TestPersistentActionPlugin.TestTasksRequestBuilder(client()).setOperation("finish").get().getTasks().size(), assertThat(new TestPersistentTasksPlugin.TestTasksRequestBuilder(client()).setOperation("finish").get().getTasks().size(),
equalTo(numberOfTasks)); equalTo(numberOfTasks));
assertBusy(() -> { assertBusy(() -> {
// Make sure the task is removed from the cluster state // Make sure the task is removed from the cluster state
assertThat(((PersistentTasks) internalCluster().clusterService().state().getMetaData() assertThat(((PersistentTasksCustomMetaData) internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE)).tasks(), empty()); .custom(PersistentTasksCustomMetaData.TYPE)).tasks(), empty());
}); });
} }

141
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionIT.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorIT.java
View File

@ -6,12 +6,15 @@
package org.elasticsearch.xpack.persistent; package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.BaseFuture;
import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskId;
import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.tasks.TaskInfo;
import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestPersistentAction; import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestTasksRequestBuilder; import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest;
import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestTasksRequestBuilder;
import org.junit.After; import org.junit.After;
import java.util.Collection; import java.util.Collection;
@ -26,11 +29,11 @@ import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.nullValue;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, minNumDataNodes = 2) @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, minNumDataNodes = 2)
public class PersistentActionIT extends ESIntegTestCase { public class PersistentTasksExecutorIT extends ESIntegTestCase {
@Override @Override
protected Collection<Class<? extends Plugin>> nodePlugins() { protected Collection<Class<? extends Plugin>> nodePlugins() {
return Collections.singletonList(TestPersistentActionPlugin.class); return Collections.singletonList(TestPersistentTasksPlugin.class);
} }
@Override @Override
@ -47,14 +50,30 @@ public class PersistentActionIT extends ESIntegTestCase {
assertNoRunningTasks(); assertNoRunningTasks();
} }
public static class PersistentTaskOperationFuture extends BaseFuture<Long> implements PersistentTaskOperationListener {
@Override
public void onResponse(long taskId) {
set(taskId);
}
@Override
public void onFailure(Exception e) {
setException(e);
}
}
public void testPersistentActionRestart() throws Exception { public void testPersistentActionRestart() throws Exception {
long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId(); PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class);
PersistentTaskOperationFuture future = new PersistentTaskOperationFuture();
persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future);
long taskId = future.get();
assertBusy(() -> { assertBusy(() -> {
// Wait for the task to start // Wait for the task to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
equalTo(1)); .getTasks().size(), equalTo(1));
}); });
TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]")
.get().getTasks().get(0); .get().getTasks().get(0);
logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId()); logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId());
// Verifying parent // Verifying parent
@ -68,7 +87,7 @@ public class PersistentActionIT extends ESIntegTestCase {
assertBusy(() -> { assertBusy(() -> {
// Wait for the task to restart // Wait for the task to restart
List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.getTasks(); .getTasks();
logger.info("Found {} tasks", tasks.size()); logger.info("Found {} tasks", tasks.size());
assertThat(tasks.size(), equalTo(1)); assertThat(tasks.size(), equalTo(1));
@ -78,24 +97,29 @@ public class PersistentActionIT extends ESIntegTestCase {
logger.info("Removing persistent task with id {}", firstRunningTask.getId()); logger.info("Removing persistent task with id {}", firstRunningTask.getId());
// Remove the persistent task // Remove the persistent task
assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture();
persistentTasksService.removeTask(taskId, removeFuture);
assertEquals(removeFuture.get(), (Long) taskId);
logger.info("Waiting for persistent task with id {} to disappear", firstRunningTask.getId()); logger.info("Waiting for persistent task with id {} to disappear", firstRunningTask.getId());
assertBusy(() -> { assertBusy(() -> {
// Wait for the task to disappear completely // Wait for the task to disappear completely
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(), assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(),
empty()); empty());
}); });
} }
public void testPersistentActionCompletion() throws Exception { public void testPersistentActionCompletion() throws Exception {
long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId(); PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class);
PersistentTaskOperationFuture future = new PersistentTaskOperationFuture();
persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future);
long taskId = future.get();
assertBusy(() -> { assertBusy(() -> {
// Wait for the task to start // Wait for the task to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
equalTo(1)); .getTasks().size(), equalTo(1));
}); });
TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]")
.get().getTasks().get(0); .get().getTasks().get(0);
logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId()); logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId());
// Verifying parent // Verifying parent
@ -106,14 +130,14 @@ public class PersistentActionIT extends ESIntegTestCase {
public void testPersistentActionCompletionWithoutRemoval() throws Exception { public void testPersistentActionCompletionWithoutRemoval() throws Exception {
boolean stopped = randomBoolean(); boolean stopped = randomBoolean();
long taskId = CreatePersistentTaskAction.INSTANCE.newRequestBuilder(client()) PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class);
.setAction(TestPersistentAction.NAME) PersistentTaskOperationFuture future = new PersistentTaskOperationFuture();
.setRequest(new TestPersistentActionPlugin.TestRequest("Blah")) persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), stopped, false,
.setRemoveOnCompletion(false) future);
.setStopped(stopped).get().getTaskId(); long taskId = future.get();
PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData() PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE); .custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasksInProgress.tasks().size(), equalTo(1)); assertThat(tasksInProgress.tasks().size(), equalTo(1));
assertThat(tasksInProgress.getTask(taskId).isStopped(), equalTo(stopped)); assertThat(tasksInProgress.getTask(taskId).isStopped(), equalTo(stopped));
assertThat(tasksInProgress.getTask(taskId).getExecutorNode(), stopped ? nullValue() : notNullValue()); assertThat(tasksInProgress.getTask(taskId).getExecutorNode(), stopped ? nullValue() : notNullValue());
@ -124,24 +148,26 @@ public class PersistentActionIT extends ESIntegTestCase {
for (int i = 0; i < numberOfIters; i++) { for (int i = 0; i < numberOfIters; i++) {
logger.info("iteration {}", i); logger.info("iteration {}", i);
if (stopped) { if (stopped) {
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(), assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
empty()); .getTasks(), empty());
assertAcked(StartPersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); PersistentTaskOperationFuture startFuture = new PersistentTaskOperationFuture();
persistentTasksService.startTask(taskId, startFuture);
assertEquals(startFuture.get(), (Long) taskId);
} }
assertBusy(() -> { assertBusy(() -> {
// Wait for the task to start // Wait for the task to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks() assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.size(), equalTo(1)); .getTasks().size(), equalTo(1));
}); });
TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]")
.get().getTasks().get(0); .get().getTasks().get(0);
stopOrCancelTask(firstRunningTask.getTaskId()); stopOrCancelTask(firstRunningTask.getTaskId());
assertBusy(() -> { assertBusy(() -> {
// Wait for the task to finish // Wait for the task to finish
List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]")
.getTasks(); .get().getTasks();
logger.info("Found {} tasks", tasks.size()); logger.info("Found {} tasks", tasks.size());
assertThat(tasks.size(), equalTo(0)); assertThat(tasks.size(), equalTo(0));
}); });
@ -150,30 +176,36 @@ public class PersistentActionIT extends ESIntegTestCase {
assertBusy(() -> { assertBusy(() -> {
// Wait for the task to be marked as stopped // Wait for the task to be marked as stopped
PersistentTasks tasks = internalCluster().clusterService().state().getMetaData() PersistentTasksCustomMetaData tasks = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE); .custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasks.tasks().size(), equalTo(1)); assertThat(tasks.tasks().size(), equalTo(1));
assertThat(tasks.getTask(taskId).isStopped(), equalTo(true)); assertThat(tasks.getTask(taskId).isStopped(), equalTo(true));
assertThat(tasks.getTask(taskId).shouldRemoveOnCompletion(), equalTo(false)); assertThat(tasks.getTask(taskId).shouldRemoveOnCompletion(), equalTo(false));
}); });
logger.info("Removing action record from cluster state"); logger.info("Removing action record from cluster state");
assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture();
persistentTasksService.removeTask(taskId, removeFuture);
assertEquals(removeFuture.get(), (Long) taskId);
} }
public void testPersistentActionWithNoAvailableNode() throws Exception { public void testPersistentActionWithNoAvailableNode() throws Exception {
long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah") PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class);
.executorNodeAttr("test").get().getTaskId(); PersistentTaskOperationFuture future = new PersistentTaskOperationFuture();
TestRequest testRequest = new TestRequest("Blah");
testRequest.setExecutorNodeAttr("test");
persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, testRequest, future);
long taskId = future.get();
Settings nodeSettings = Settings.builder().put(nodeSettings(0)).put("node.attr.test_attr", "test").build(); Settings nodeSettings = Settings.builder().put(nodeSettings(0)).put("node.attr.test_attr", "test").build();
String newNode = internalCluster().startNode(nodeSettings); String newNode = internalCluster().startNode(nodeSettings);
String newNodeId = internalCluster().clusterService(newNode).localNode().getId(); String newNodeId = internalCluster().clusterService(newNode).localNode().getId();
assertBusy(() -> { assertBusy(() -> {
// Wait for the task to start // Wait for the task to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks()
equalTo(1)); .size(), equalTo(1));
}); });
TaskInfo taskInfo = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") TaskInfo taskInfo = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]")
.get().getTasks().get(0); .get().getTasks().get(0);
// Verifying the the task runs on the new node // Verifying the the task runs on the new node
@ -183,27 +215,32 @@ public class PersistentActionIT extends ESIntegTestCase {
assertBusy(() -> { assertBusy(() -> {
// Wait for the task to disappear completely // Wait for the task to disappear completely
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(), assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(),
empty()); empty());
}); });
// Remove the persistent task // Remove the persistent task
assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture();
persistentTasksService.removeTask(taskId, removeFuture);
assertEquals(removeFuture.get(), (Long) taskId);
} }
public void testPersistentActionStatusUpdate() throws Exception { public void testPersistentActionStatusUpdate() throws Exception {
TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get(); PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class);
PersistentTaskOperationFuture future = new PersistentTaskOperationFuture();
persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future);
future.get();
assertBusy(() -> { assertBusy(() -> {
// Wait for the task to start // Wait for the task to start
assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks()
equalTo(1)); .size(), equalTo(1));
}); });
TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]")
.get().getTasks().get(0); .get().getTasks().get(0);
PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData() PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE); .custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasksInProgress.tasks().size(), equalTo(1)); assertThat(tasksInProgress.tasks().size(), equalTo(1));
assertThat(tasksInProgress.tasks().iterator().next().getStatus(), nullValue()); assertThat(tasksInProgress.tasks().iterator().next().getStatus(), nullValue());
@ -216,8 +253,8 @@ public class PersistentActionIT extends ESIntegTestCase {
int finalI = i; int finalI = i;
assertBusy(() -> { assertBusy(() -> {
PersistentTasks tasks = internalCluster().clusterService().state().getMetaData() PersistentTasksCustomMetaData tasks = internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE); .custom(PersistentTasksCustomMetaData.TYPE);
assertThat(tasks.tasks().size(), equalTo(1)); assertThat(tasks.tasks().size(), equalTo(1));
assertThat(tasks.tasks().iterator().next().getStatus(), notNullValue()); assertThat(tasks.tasks().iterator().next().getStatus(), notNullValue());
assertThat(tasks.tasks().iterator().next().getStatus().toString(), equalTo("{\"phase\":\"phase " + (finalI + 1) + "\"}")); assertThat(tasks.tasks().iterator().next().getStatus().toString(), equalTo("{\"phase\":\"phase " + (finalI + 1) + "\"}"));
@ -252,14 +289,14 @@ public class PersistentActionIT extends ESIntegTestCase {
private void assertNoRunningTasks() throws Exception { private void assertNoRunningTasks() throws Exception {
assertBusy(() -> { assertBusy(() -> {
// Wait for the task to finish // Wait for the task to finish
List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get()
.getTasks(); .getTasks();
logger.info("Found {} tasks", tasks.size()); logger.info("Found {} tasks", tasks.size());
assertThat(tasks.size(), equalTo(0)); assertThat(tasks.size(), equalTo(0));
// Make sure the task is removed from the cluster state // Make sure the task is removed from the cluster state
assertThat(((PersistentTasks) internalCluster().clusterService().state().getMetaData() assertThat(((PersistentTasksCustomMetaData) internalCluster().clusterService().state().getMetaData()
.custom(PersistentTasks.TYPE)).tasks(), empty()); .custom(PersistentTasksCustomMetaData.TYPE)).tasks(), empty());
}); });
} }

10
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionResponseTests.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksExecutorResponseTests.java
View File

@ -7,15 +7,15 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.test.AbstractStreamableTestCase; import org.elasticsearch.test.AbstractStreamableTestCase;
public class PersistentActionResponseTests extends AbstractStreamableTestCase<PersistentActionResponse> { public class PersistentTasksExecutorResponseTests extends AbstractStreamableTestCase<PersistentTaskResponse> {
@Override @Override
protected PersistentActionResponse createTestInstance() { protected PersistentTaskResponse createTestInstance() {
return new PersistentActionResponse(randomLong()); return new PersistentTaskResponse(randomLong());
} }
@Override @Override
protected PersistentActionResponse createBlankInstance() { protected PersistentTaskResponse createBlankInstance() {
return new PersistentActionResponse(); return new PersistentTaskResponse();
} }
} }

6
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinatorStatusTests.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeServiceStatusTests.java
View File

@ -7,12 +7,12 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.test.AbstractWireSerializingTestCase;
import org.elasticsearch.xpack.persistent.PersistentActionCoordinator.State; import org.elasticsearch.xpack.persistent.PersistentTasksNodeService.State;
import org.elasticsearch.xpack.persistent.PersistentActionCoordinator.Status; import org.elasticsearch.xpack.persistent.PersistentTasksNodeService.Status;
import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsString;
public class PersistentActionCoordinatorStatusTests extends AbstractWireSerializingTestCase<Status> { public class PersistentTasksNodeServiceStatusTests extends AbstractWireSerializingTestCase<Status> {
@Override @Override
protected Status createTestInstance() { protected Status createTestInstance() {

204
plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinatorTests.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/PersistentTasksNodeServiceTests.java
View File

@ -7,7 +7,6 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.Version; import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaData;
@ -22,12 +21,13 @@ import org.elasticsearch.tasks.TaskManager;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.xpack.persistent.CompletionPersistentTaskAction.Response; import org.elasticsearch.xpack.persistent.PersistentTasksService.PersistentTaskOperationListener;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest; import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
@ -37,7 +37,7 @@ import static org.hamcrest.core.IsEqual.equalTo;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
public class PersistentActionCoordinatorTests extends ESTestCase { public class PersistentTasksNodeServiceTests extends ESTestCase {
private ClusterService createClusterService() { private ClusterService createClusterService() {
ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
@ -58,21 +58,21 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
public void testStartTask() throws Exception { public void testStartTask() throws Exception {
ClusterService clusterService = createClusterService(); ClusterService clusterService = createClusterService();
PersistentActionService persistentActionService = mock(PersistentActionService.class); PersistentTasksService persistentTasksService = mock(PersistentTasksService.class);
PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); @SuppressWarnings("unchecked") PersistentTasksExecutor<TestRequest> action = mock(PersistentTasksExecutor.class);
@SuppressWarnings("unchecked") TransportPersistentAction<TestRequest> action = mock(TransportPersistentAction.class);
when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME);
registry.registerPersistentAction("test", action); when(action.getTaskName()).thenReturn("test");
PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action));
int nonLocalNodesCount = randomInt(10); int nonLocalNodesCount = randomInt(10);
MockExecutor executor = new MockExecutor(); MockExecutor executor = new MockExecutor();
PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService, PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService,
registry, new TaskManager(Settings.EMPTY), executor); registry, new TaskManager(Settings.EMPTY), mock(ThreadPool.class), executor);
ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY))
.build(); .build();
PersistentTasks.Builder tasks = PersistentTasks.builder(); PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder();
boolean added = false; boolean added = false;
if (nonLocalNodesCount > 0) { if (nonLocalNodesCount > 0) {
for (int i = 0; i < randomInt(5); i++) { for (int i = 0; i < randomInt(5); i++) {
@ -91,7 +91,7 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
} }
MetaData.Builder metaData = MetaData.builder(state.metaData()); MetaData.Builder metaData = MetaData.builder(state.metaData());
metaData.putCustom(PersistentTasks.TYPE, tasks.build()); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build());
ClusterState newClusterState = ClusterState.builder(state).metaData(metaData).build(); ClusterState newClusterState = ClusterState.builder(state).metaData(metaData).build();
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
@ -149,24 +149,24 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
public void testTaskCancellation() { public void testTaskCancellation() {
ClusterService clusterService = createClusterService(); ClusterService clusterService = createClusterService();
AtomicLong capturedTaskId = new AtomicLong(); AtomicLong capturedTaskId = new AtomicLong();
AtomicReference<ActionListener<CancelTasksResponse>> capturedListener = new AtomicReference<>(); AtomicReference<PersistentTaskOperationListener> capturedListener = new AtomicReference<>();
PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, null, null, null) { PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, null, null) {
@Override @Override
public void sendCancellation(long taskId, ActionListener<CancelTasksResponse> listener) { public void sendCancellation(long taskId, PersistentTaskOperationListener listener) {
capturedTaskId.set(taskId); capturedTaskId.set(taskId);
capturedListener.set(listener); capturedListener.set(listener);
} }
}; };
PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); @SuppressWarnings("unchecked") PersistentTasksExecutor<TestRequest> action = mock(PersistentTasksExecutor.class);
@SuppressWarnings("unchecked") TransportPersistentAction<TestRequest> action = mock(TransportPersistentAction.class);
when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME);
registry.registerPersistentAction("test", action); when(action.getTaskName()).thenReturn("test");
PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action));
int nonLocalNodesCount = randomInt(10); int nonLocalNodesCount = randomInt(10);
MockExecutor executor = new MockExecutor(); MockExecutor executor = new MockExecutor();
TaskManager taskManager = new TaskManager(Settings.EMPTY); TaskManager taskManager = new TaskManager(Settings.EMPTY);
PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService, PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService,
registry, taskManager, executor); registry, taskManager, mock(ThreadPool.class), executor);
ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY))
.build(); .build();
@ -203,7 +203,7 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
// That should trigger cancellation request // That should trigger cancellation request
assertThat(capturedTaskId.get(), equalTo(localId)); assertThat(capturedTaskId.get(), equalTo(localId));
// Notify successful cancellation // Notify successful cancellation
capturedListener.get().onResponse(new CancelTasksResponse()); capturedListener.get().onResponse(localId);
// finish or fail task // finish or fail task
if (randomBoolean()) { if (randomBoolean()) {
@ -217,106 +217,118 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
} }
public void testNotificationFailure() { public void testNotificationFailure() throws Exception {
ClusterService clusterService = createClusterService(); Settings settings = Settings.builder()
AtomicLong capturedTaskId = new AtomicLong(-1L); .put("node.name", PersistentTasksNodeServiceTests.class.getSimpleName())
AtomicReference<Exception> capturedException = new AtomicReference<>();
AtomicReference<ActionListener<Response>> capturedListener = new AtomicReference<>();
PersistentActionService persistentActionService =
new PersistentActionService(Settings.EMPTY, mock(ThreadPool.class), clusterService, null) {
@Override
public void sendCompletionNotification(long taskId, Exception failure, ActionListener<Response> listener) {
capturedTaskId.set(taskId);
capturedException.set(failure);
capturedListener.set(listener);
}
};
PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY);
@SuppressWarnings("unchecked") TransportPersistentAction<TestRequest> action = mock(TransportPersistentAction.class);
when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME);
registry.registerPersistentAction("test", action);
int nonLocalNodesCount = randomInt(10);
MockExecutor executor = new MockExecutor();
TaskManager taskManager = new TaskManager(Settings.EMPTY);
PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService,
registry, taskManager, executor);
ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY))
.build(); .build();
ThreadPool threadPool = new ThreadPool(settings);
try {
ClusterService clusterService = createClusterService();
AtomicLong capturedTaskId = new AtomicLong(-1L);
AtomicReference<Exception> capturedException = new AtomicReference<>();
AtomicReference<PersistentTaskOperationListener> capturedListener = new AtomicReference<>();
PersistentTasksService persistentTasksService =
new PersistentTasksService(Settings.EMPTY, clusterService, null) {
@Override
public void sendCompletionNotification(long taskId, Exception failure, PersistentTaskOperationListener listener) {
capturedTaskId.set(taskId);
capturedException.set(failure);
capturedListener.set(listener);
}
};
@SuppressWarnings("unchecked") PersistentTasksExecutor<TestRequest> action = mock(PersistentTasksExecutor.class);
when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME);
when(action.getTaskName()).thenReturn("test");
PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY,
Collections.singletonList(action));
ClusterState newClusterState = state; int nonLocalNodesCount = randomInt(10);
// Allocate first task MockExecutor executor = new MockExecutor();
state = newClusterState; TaskManager taskManager = new TaskManager(Settings.EMPTY);
newClusterState = addTask(state, "test", new TestRequest(), "this_node"); PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService,
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); registry, taskManager, threadPool, executor);
// Fail the task ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY))
executor.get(0).listener.onFailure(new RuntimeException("test failure")); .build();
// Check that notification was sent ClusterState newClusterState = state;
assertThat(capturedException.get().getMessage(), equalTo("test failure")); // Allocate first task
capturedException.set(null); state = newClusterState;
newClusterState = addTask(state, "test", new TestRequest(), "this_node");
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
// Simulate failure to notify // Fail the task
capturedListener.get().onFailure(new IOException("simulated notification failure")); executor.get(0).listener.onFailure(new RuntimeException("test failure"));
// Allocate another task // Check that notification was sent
state = newClusterState; assertThat(capturedException.get().getMessage(), equalTo("test failure"));
newClusterState = addTask(state, "test", new TestRequest(), "other_node"); capturedException.set(null);
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
// Check that notification was sent again // Simulate failure to notify
assertThat(capturedException.get().getMessage(), equalTo("test failure")); capturedListener.get().onFailure(new IOException("simulated notification failure"));
// Check the the task is still known by the task manager // Allocate another task
assertThat(taskManager.getTasks().size(), equalTo(1)); state = newClusterState;
long id = taskManager.getTasks().values().iterator().next().getParentTaskId().getId(); newClusterState = addTask(state, "test", new TestRequest(), "other_node");
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
// This time acknowledge notification // Check that notification was sent again
capturedListener.get().onResponse(new Response()); assertBusy(() -> {
assertNotNull(capturedException.get());
assertThat(capturedException.get().getMessage(), equalTo("test failure"));
});
// Reallocate failed task to another node // Check the the task is still known by the task manager
state = newClusterState; assertThat(taskManager.getTasks().size(), equalTo(1));
newClusterState = reallocateTask(state, id, "other_node"); long id = taskManager.getTasks().values().iterator().next().getParentTaskId().getId();
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
// Check the the task is now removed from task manager // This time acknowledge notification
assertThat(taskManager.getTasks().values(), empty()); capturedListener.get().onResponse(id);
// Reallocate failed task to another node
state = newClusterState;
newClusterState = reallocateTask(state, id, "other_node");
coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state));
// Check the the task is now removed from task manager
assertThat(taskManager.getTasks().values(), empty());
} finally {
assertTrue(ESTestCase.terminate(threadPool));
}
} }
private <Request extends PersistentActionRequest> ClusterState addTask(ClusterState state, String action, Request request, private <Request extends PersistentTaskRequest> ClusterState addTask(ClusterState state, String action, Request request,
String node) { String node) {
PersistentTasks.Builder builder = PersistentTasksCustomMetaData.Builder builder =
PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE)); PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE));
return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE, return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE,
builder.addTask(action, request, false, true, new Assignment(node, "test assignment")).build())).build(); builder.addTask(action, request, false, true, new Assignment(node, "test assignment")).build())).build();
} }
private ClusterState reallocateTask(ClusterState state, long taskId, String node) { private ClusterState reallocateTask(ClusterState state, long taskId, String node) {
PersistentTasks.Builder builder = PersistentTasksCustomMetaData.Builder builder =
PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE)); PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE));
assertTrue(builder.hasTask(taskId)); assertTrue(builder.hasTask(taskId));
return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE, return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE,
builder.reassignTask(taskId, new Assignment(node, "test assignment")).build())).build(); builder.reassignTask(taskId, new Assignment(node, "test assignment")).build())).build();
} }
private ClusterState removeTask(ClusterState state, long taskId) { private ClusterState removeTask(ClusterState state, long taskId) {
PersistentTasks.Builder builder = PersistentTasksCustomMetaData.Builder builder =
PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE)); PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE));
assertTrue(builder.hasTask(taskId)); assertTrue(builder.hasTask(taskId));
return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE, return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE,
builder.removeTask(taskId).build())).build(); builder.removeTask(taskId).build())).build();
} }
private class Execution { private class Execution {
private final PersistentActionRequest request; private final PersistentTaskRequest request;
private final NodePersistentTask task; private final NodePersistentTask task;
private final PersistentActionRegistry.PersistentActionHolder<?> holder; private final PersistentTasksExecutor<?> holder;
private final ActionListener<Empty> listener; private final ActionListener<Empty> listener;
Execution(PersistentActionRequest request, NodePersistentTask task, PersistentActionRegistry.PersistentActionHolder<?> holder, Execution(PersistentTaskRequest request, NodePersistentTask task, PersistentTasksExecutor<?> holder,
ActionListener<Empty> listener) { ActionListener<Empty> listener) {
this.request = request; this.request = request;
this.task = task; this.task = task;
@ -325,7 +337,7 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
} }
} }
private class MockExecutor extends PersistentActionExecutor { private class MockExecutor extends NodePersistentTasksExecutor {
private List<Execution> executions = new ArrayList<>(); private List<Execution> executions = new ArrayList<>();
MockExecutor() { MockExecutor() {
@ -333,10 +345,10 @@ public class PersistentActionCoordinatorTests extends ESTestCase {
} }
@Override @Override
public <Request extends PersistentActionRequest> void executeAction(Request request, NodePersistentTask task, public <Request extends PersistentTaskRequest> void executeTask(Request request, NodePersistentTask task,
PersistentActionRegistry.PersistentActionHolder<Request> holder, PersistentTasksExecutor<Request> action,
ActionListener<Empty> listener) { ActionListener<Empty> listener) {
executions.add(new Execution(request, task, holder, listener)); executions.add(new Execution(request, task, action, listener));
} }
public Execution get(int i) { public Execution get(int i) {

6
plugin/src/test/java/org/elasticsearch/xpack/persistent/StartPersistentActionRequestTests.java
View File

@ -8,8 +8,8 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
import org.elasticsearch.xpack.persistent.CreatePersistentTaskAction.Request; import org.elasticsearch.xpack.persistent.CreatePersistentTaskAction.Request;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestPersistentAction; import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.TestRequest; import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.TestRequest;
import org.elasticsearch.test.AbstractStreamableTestCase; import org.elasticsearch.test.AbstractStreamableTestCase;
import java.util.Collections; import java.util.Collections;
@ -39,7 +39,7 @@ public class StartPersistentActionRequestTests extends AbstractStreamableTestCas
@Override @Override
protected NamedWriteableRegistry getNamedWriteableRegistry() { protected NamedWriteableRegistry getNamedWriteableRegistry() {
return new NamedWriteableRegistry(Collections.singletonList( return new NamedWriteableRegistry(Collections.singletonList(
new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new) new Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new)
)); ));
} }
} }

106
plugin/src/test/java/org/elasticsearch/xpack/persistent/TestPersistentActionPlugin.java → plugin/src/test/java/org/elasticsearch/xpack/persistent/TestPersistentTasksPlugin.java
View File

@ -8,7 +8,6 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.action.Action; import org.elasticsearch.action.Action;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.FailedNodeException;
@ -49,7 +48,7 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportResponse.Empty;
import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportService;
import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.watcher.ResourceWatcherService;
import org.elasticsearch.xpack.persistent.PersistentTasks.Assignment; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.Assignment;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@ -72,12 +71,11 @@ import static org.junit.Assert.fail;
/** /**
* A plugin that adds a test persistent task. * A plugin that adds a test persistent task.
*/ */
public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin {
@Override @Override
public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() { public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
return Arrays.asList( return Arrays.asList(
new ActionHandler<>(TestPersistentAction.INSTANCE, TransportTestPersistentAction.class),
new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class), new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class),
new ActionHandler<>(CreatePersistentTaskAction.INSTANCE, CreatePersistentTaskAction.TransportAction.class), new ActionHandler<>(CreatePersistentTaskAction.INSTANCE, CreatePersistentTaskAction.TransportAction.class),
new ActionHandler<>(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class), new ActionHandler<>(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class),
@ -91,24 +89,28 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
ResourceWatcherService resourceWatcherService, ScriptService scriptService, ResourceWatcherService resourceWatcherService, ScriptService scriptService,
NamedXContentRegistry xContentRegistry) { NamedXContentRegistry xContentRegistry) {
PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, client);
PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, threadPool, clusterService, client); TestPersistentTasksExecutor testPersistentAction = new TestPersistentTasksExecutor(Settings.EMPTY, persistentTasksService,
PersistentActionRegistry persistentActionRegistry = new PersistentActionRegistry(Settings.EMPTY); clusterService);
PersistentTasksExecutorRegistry persistentTasksExecutorRegistry = new PersistentTasksExecutorRegistry(Settings.EMPTY,
Collections.singletonList(testPersistentAction));
return Arrays.asList( return Arrays.asList(
persistentActionService, persistentTasksService,
persistentActionRegistry, persistentTasksExecutorRegistry,
new PersistentTaskClusterService(Settings.EMPTY, persistentActionRegistry, clusterService) new PersistentTasksClusterService(Settings.EMPTY, persistentTasksExecutorRegistry, clusterService)
); );
} }
@Override @Override
public List<NamedWriteableRegistry.Entry> getNamedWriteables() { public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
return Arrays.asList( return Arrays.asList(
new NamedWriteableRegistry.Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new), new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new),
new NamedWriteableRegistry.Entry(Task.Status.class, new NamedWriteableRegistry.Entry(Task.Status.class,
PersistentActionCoordinator.Status.NAME, PersistentActionCoordinator.Status::new), PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new),
new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new), new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE,
new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom), PersistentTasksCustomMetaData::new),
new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE,
PersistentTasksCustomMetaData::readDiffFrom),
new NamedWriteableRegistry.Entry(Task.Status.class, Status.NAME, Status::new) new NamedWriteableRegistry.Entry(Task.Status.class, Status.NAME, Status::new)
); );
} }
@ -116,18 +118,18 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
@Override @Override
public List<NamedXContentRegistry.Entry> getNamedXContent() { public List<NamedXContentRegistry.Entry> getNamedXContent() {
return Arrays.asList( return Arrays.asList(
new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasks.TYPE), new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksCustomMetaData.TYPE),
PersistentTasks::fromXContent), PersistentTasksCustomMetaData::fromXContent),
new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(TestPersistentAction.NAME), new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(TestPersistentTasksExecutor.NAME),
TestRequest::fromXContent), TestRequest::fromXContent),
new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent)
); );
} }
public static class TestRequest extends PersistentActionRequest { public static class TestRequest extends PersistentTaskRequest {
public static final ConstructingObjectParser<TestRequest, Void> REQUEST_PARSER = public static final ConstructingObjectParser<TestRequest, Void> REQUEST_PARSER =
new ConstructingObjectParser<>(TestPersistentAction.NAME, args -> new TestRequest((String) args[0])); new ConstructingObjectParser<>(TestPersistentTasksExecutor.NAME, args -> new TestRequest((String) args[0]));
static { static {
REQUEST_PARSER.declareString(constructorArg(), new ParseField("param")); REQUEST_PARSER.declareString(constructorArg(), new ParseField("param"));
@ -158,7 +160,7 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
@Override @Override
public String getWriteableName() { public String getWriteableName() {
return TestPersistentAction.NAME; return TestPersistentTasksExecutor.NAME;
} }
public void setExecutorNodeAttr(String executorNodeAttr) { public void setExecutorNodeAttr(String executorNodeAttr) {
@ -226,53 +228,13 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
} }
} }
public static class TestPersistentTaskRequestBuilder extends
ActionRequestBuilder<TestRequest, PersistentActionResponse, TestPersistentTaskRequestBuilder> {
protected TestPersistentTaskRequestBuilder(ElasticsearchClient client, Action<TestRequest, PersistentActionResponse,
TestPersistentTaskRequestBuilder> action, TestRequest request) {
super(client, action, request);
}
public TestPersistentTaskRequestBuilder testParam(String testParam) {
request.setTestParam(testParam);
return this;
}
public TestPersistentTaskRequestBuilder executorNodeAttr(String targetNode) {
request.setExecutorNodeAttr(targetNode);
return this;
}
}
public static class TestPersistentAction extends Action<TestRequest, PersistentActionResponse, TestPersistentTaskRequestBuilder> {
public static final TestPersistentAction INSTANCE = new TestPersistentAction();
public static final String NAME = "cluster:admin/persistent/test";
private TestPersistentAction() {
super(NAME);
}
@Override
public PersistentActionResponse newResponse() {
return new PersistentActionResponse();
}
@Override
public TestPersistentTaskRequestBuilder newRequestBuilder(ElasticsearchClient client) {
return new TestPersistentTaskRequestBuilder(client, this, new TestRequest());
}
}
public static class Status implements Task.Status { public static class Status implements Task.Status {
public static final String NAME = "test"; public static final String NAME = "test";
private final String phase; private final String phase;
public static final ConstructingObjectParser<Status, Void> STATUS_PARSER = public static final ConstructingObjectParser<Status, Void> STATUS_PARSER =
new ConstructingObjectParser<>(TestPersistentAction.NAME, args -> new Status((String) args[0])); new ConstructingObjectParser<>(TestPersistentTasksExecutor.NAME, args -> new Status((String) args[0]));
static { static {
STATUS_PARSER.declareString(constructorArg(), new ParseField("phase")); STATUS_PARSER.declareString(constructorArg(), new ParseField("phase"));
@ -336,19 +298,15 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
} }
public static class TransportTestPersistentAction extends TransportPersistentAction<TestRequest> { public static class TestPersistentTasksExecutor extends PersistentTasksExecutor<TestRequest> {
private final TransportService transportService; public static final String NAME = "cluster:admin/persistent/test";
private final ClusterService clusterService;
@Inject public TestPersistentTasksExecutor(Settings settings, PersistentTasksService persistentTasksService,
public TransportTestPersistentAction(Settings settings, ThreadPool threadPool, TransportService transportService, ClusterService clusterService) {
PersistentActionService persistentActionService, super(settings, NAME, persistentTasksService, ThreadPool.Names.GENERIC);
PersistentActionRegistry persistentActionRegistry, ActionFilters actionFilters, this.clusterService = clusterService;
IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, TestPersistentAction.NAME, false, threadPool, transportService, persistentActionService,
persistentActionRegistry, actionFilters, indexNameExpressionResolver, TestRequest::new,
ThreadPool.Names.GENERIC);
this.transportService = transportService;
} }
@Override @Override
@ -377,9 +335,9 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin {
// wait for something to happen // wait for something to happen
assertTrue(awaitBusy(() -> testTask.isCancelled() || assertTrue(awaitBusy(() -> testTask.isCancelled() ||
testTask.getOperation() != null || testTask.getOperation() != null ||
transportService.lifecycleState() != Lifecycle.State.STARTED, // speedup finishing on closed nodes clusterService.lifecycleState() != Lifecycle.State.STARTED, // speedup finishing on closed nodes
30, TimeUnit.SECONDS)); // This can take a while during large cluster restart 30, TimeUnit.SECONDS)); // This can take a while during large cluster restart
if (transportService.lifecycleState() != Lifecycle.State.STARTED) { if (clusterService.lifecycleState() != Lifecycle.State.STARTED) {
return; return;
} }
if ("finish".equals(testTask.getOperation())) { if ("finish".equals(testTask.getOperation())) {

2
plugin/src/test/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskRequestTests.java
View File

@ -8,7 +8,7 @@ package org.elasticsearch.xpack.persistent;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.Task;
import org.elasticsearch.test.AbstractStreamableTestCase; import org.elasticsearch.test.AbstractStreamableTestCase;
import org.elasticsearch.xpack.persistent.TestPersistentActionPlugin.Status; import org.elasticsearch.xpack.persistent.TestPersistentTasksPlugin.Status;
import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction.Request; import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction.Request;
import java.util.Collections; import java.util.Collections;

2
plugin/src/test/resources/org/elasticsearch/transport/actions
View File

@ -143,4 +143,4 @@ cluster:admin/persistent/create
cluster:admin/persistent/start cluster:admin/persistent/start
cluster:admin/persistent/completion cluster:admin/persistent/completion
cluster:admin/persistent/update_status cluster:admin/persistent/update_status
cluster:admin/persistent/remove cluster:admin/persistent/remove